diff --git a/datasets/ecmwf-forecast/Dockerfile b/datasets/ecmwf-forecast/Dockerfile index 5c963b147..756ba5fab 100644 --- a/datasets/ecmwf-forecast/Dockerfile +++ b/datasets/ecmwf-forecast/Dockerfile @@ -21,10 +21,10 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 # See https://github.com/mapbox/rasterio/issues/1289 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt -# Install Python 3.10 -RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ - && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \ - && rm -rf "Mambaforge-$(uname)-$(uname -m).sh" +# Install Python 3.11 +RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" \ + && bash "Miniforge3-$(uname)-$(uname -m).sh" -b -p /opt/conda \ + && rm -rf "Miniforge3-$(uname)-$(uname -m).sh" ENV PATH /opt/conda/bin:$PATH ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY ./datasets/ecmwf-forecast/requirements.txt /opt/src/datasets/ecmwf-forecast/requirements.txt RUN python3 -m pip install -r /opt/src/datasets/ecmwf-forecast/requirements.txt diff --git a/datasets/ecmwf-forecast/dataset.yaml b/datasets/ecmwf-forecast/dataset.yaml index d801f93a6..cb2454757 100644 --- a/datasets/ecmwf-forecast/dataset.yaml +++ b/datasets/ecmwf-forecast/dataset.yaml @@ -1,8 +1,8 @@ id: ecmwf_forecast -image: ${{ args.registry }}/pctasks-ecmwf-forecast:2024.6.13.0 +image: ${{ args.registry }}/pctasks-ecmwf-forecast:2026.01.12 args: -- registry + - registry code: src: ${{ local.path(./ecmwf_forecast.py) }} diff --git a/datasets/gbif/Dockerfile b/datasets/gbif/Dockerfile index 21a364821..6c2fef76e 100644 --- a/datasets/gbif/Dockerfile +++ b/datasets/gbif/Dockerfile @@ -22,14 +22,14 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt # Install Python 3.8 -RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ - && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \ - && rm -rf "Mambaforge-$(uname)-$(uname -m).sh" +RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" \ + && bash "Miniforge3-$(uname)-$(uname -m).sh" -b -p /opt/conda \ + && rm -rf "Miniforge3-$(uname)-$(uname -m).sh" ENV PATH /opt/conda/bin:$PATH ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH -RUN mamba install -y -c conda-forge python=3.8 gdal=3.3.3 pip setuptools cython numpy==1.21.5 +RUN mamba install -y -c conda-forge python=3.11 gdal pip setuptools cython numpy RUN python -m pip install --upgrade pip @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY ./datasets/gbif/requirements.txt /opt/src/datasets/gbif/requirements.txt RUN python3 -m pip install -r /opt/src/datasets/gbif/requirements.txt diff --git a/datasets/gbif/dataset.yaml b/datasets/gbif/dataset.yaml index f05ee626b..d390e66f7 100644 --- a/datasets/gbif/dataset.yaml +++ b/datasets/gbif/dataset.yaml @@ -1,8 +1,8 @@ id: gbif -image: ${{ args.registry }}/pctasks-gbif:20230607.1 +image: ${{ args.registry }}/pctasks-gbif:2026.01.12 args: -- registry + - registry code: src: ${{ local.path(./gbif.py) }} @@ -24,4 +24,4 @@ collections: min_depth: 2 max_depth: 2 chunk_storage: - uri: blob://ai4edataeuwest/gbif-etl-data/pctasks-chunks \ No newline at end of file + uri: blob://ai4edataeuwest/gbif-etl-data/pctasks-chunks diff --git a/datasets/goes/goes-cmi/Dockerfile b/datasets/goes/goes-cmi/Dockerfile index 8491689ae..dc5bc427a 100644 --- a/datasets/goes/goes-cmi/Dockerfile +++ b/datasets/goes/goes-cmi/Dockerfile @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY datasets/goes/goes-cmi/requirements.txt /opt/src/datasets/goes/goes-cmi/requirements.txt RUN python3 -m pip install -r /opt/src/datasets/goes/goes-cmi/requirements.txt diff --git a/datasets/goes/goes-cmi/dataset.yaml b/datasets/goes/goes-cmi/dataset.yaml index dd90f7dd5..10f87dbeb 100644 --- a/datasets/goes/goes-cmi/dataset.yaml +++ b/datasets/goes/goes-cmi/dataset.yaml @@ -1,9 +1,9 @@ id: goes_cmi -image: ${{ args.registry }}/pctasks-goes-cmi:2025.04.15.1 +image: ${{ args.registry }}/pctasks-goes-cmi:2026.01.12 args: -- registry -- year-prefix + - registry + - year-prefix code: src: ${{ local.path(./goes_cmi) }} @@ -19,7 +19,7 @@ collections: asset_storage: # The blob storage pattern is # | noaa-goes16 - # | ABI-L2-MCMIPC + # | ABI-L2-MCMIPC # | 2023 # | 170 # day of year # | 23 # hour @@ -27,12 +27,12 @@ collections: - uri: blob://goeseuwest/noaa-goes16/ chunks: splits: - - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }} # CONUS - depth: 3 # daily - - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }} # Mesoscale - depth: 3 # daily - - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }} # Full Disk - depth: 3 # daily + - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }} # CONUS + depth: 3 # daily + - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }} # Mesoscale + depth: 3 # daily + - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }} # Full Disk + depth: 3 # daily options: ends_with: .nc # # GOES-17 is parked @@ -50,25 +50,25 @@ collections: - uri: blob://goeseuwest/noaa-goes18/ chunks: splits: - - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }} # CONUS - depth: 3 # daily - - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }} # Mesoscale - depth: 3 # daily - - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }} # Full Disk - depth: 3 # daily + - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }} # CONUS + depth: 3 # daily + - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }} # Mesoscale + depth: 3 # daily + - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }} # Full Disk + depth: 3 # daily options: ends_with: .nc - uri: blob://goeseuwest/noaa-goes19/ chunks: splits: - - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }} # CONUS - depth: 3 # daily - - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }} # Mesoscale - depth: 3 # daily - - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }} # Full Disk - depth: 3 # daily + - prefix: ABI-L2-MCMIPC/${{ args.year-prefix }} # CONUS + depth: 3 # daily + - prefix: ABI-L2-MCMIPM/${{ args.year-prefix }} # Mesoscale + depth: 3 # daily + - prefix: ABI-L2-MCMIPF/${{ args.year-prefix }} # Full Disk + depth: 3 # daily options: ends_with: .nc chunk_storage: - uri: blob://goeseuwest/noaa-goes-etl-data/pctasks/cmi/ \ No newline at end of file + uri: blob://goeseuwest/noaa-goes-etl-data/pctasks/cmi/ diff --git a/datasets/goes/goes-glm/Dockerfile b/datasets/goes/goes-glm/Dockerfile index 61d8e36d6..3a28dd893 100644 --- a/datasets/goes/goes-glm/Dockerfile +++ b/datasets/goes/goes-glm/Dockerfile @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY datasets/goes/goes-glm /opt/src/datasets/goes-glm RUN python3 -m pip install -r /opt/src/datasets/goes-glm/requirements.txt diff --git a/datasets/goes/goes-glm/dataset.yaml b/datasets/goes/goes-glm/dataset.yaml index c861759a5..72386cbb1 100644 --- a/datasets/goes/goes-glm/dataset.yaml +++ b/datasets/goes/goes-glm/dataset.yaml @@ -1,9 +1,9 @@ id: goes_glm -image: ${{ args.registry }}/pctasks-goes-glm:2025.04.16.0 +image: ${{ args.registry }}/pctasks-goes-glm:2026.01.12 args: -- registry -- year-prefix + - registry + - year-prefix code: src: ${{ local.path(./goes_glm.py) }} @@ -60,4 +60,4 @@ collections: depth: 2 chunk_storage: - uri: blob://goeseuwest/noaa-goes-etl-data/pctasks/glm/ \ No newline at end of file + uri: blob://goeseuwest/noaa-goes-etl-data/pctasks/glm/ diff --git a/datasets/hls2/dataset.yaml b/datasets/hls2/dataset.yaml index 2ede40ccd..10a24fd2a 100644 --- a/datasets/hls2/dataset.yaml +++ b/datasets/hls2/dataset.yaml @@ -1,5 +1,5 @@ id: hls2 -image: ${{ args.registry }}/pctasks-task-base:2025.4.8.2 +image: ${{ args.registry }}/pctasks-task-base:2026.01.12 args: - registry @@ -42,7 +42,6 @@ collections: chunk_length: 20000 chunk_storage: uri: blob://hls2euwest/hls2-l30-info/pctasks-chunks/ - # The blob storage pattern is: # | container # | S30/ or L30/ depending on collection @@ -57,4 +56,4 @@ collections: # | HLS.S30.T56PPQ.2024005T001421.v2.0.B01.tif # | HLS.S30.T56PPQ.2024005T001421.v2.0.B02.tif # | ... -# | eg S30/56/P/PQ/2024/01/05/HLS.S30.T56PPQ.2024005T001421.v2.0/HLS.S30.T56PPQ.2024005T001421.v2.0.B01.tif \ No newline at end of file +# | eg S30/56/P/PQ/2024/01/05/HLS.S30.T56PPQ.2024005T001421.v2.0/HLS.S30.T56PPQ.2024005T001421.v2.0.B01.tif diff --git a/datasets/landsat/Dockerfile b/datasets/landsat/Dockerfile index 6c111966d..72ddf472a 100644 --- a/datasets/landsat/Dockerfile +++ b/datasets/landsat/Dockerfile @@ -21,15 +21,15 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 # See https://github.com/mapbox/rasterio/issues/1289 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt -# Install Python 3.8 -RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ - && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \ - && rm -rf "Mambaforge-$(uname)-$(uname -m).sh" +# Install Python 3.11 +RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" \ + && bash "Miniforge3-$(uname)-$(uname -m).sh" -b -p /opt/conda \ + && rm -rf "Miniforge3-$(uname)-$(uname -m).sh" ENV PATH /opt/conda/bin:$PATH ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH -RUN mamba install -y -c conda-forge python=3.8 gdal=3.3.3 pip setuptools cython numpy==1.21.5 +RUN mamba install -y -c conda-forge python=3.11 gdal pip setuptools cython numpy RUN python -m pip install --upgrade pip @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY ./datasets/landsat/requirements.txt /opt/src/datasets/landsat/requirements.txt RUN python3 -m pip install -r /opt/src/datasets/landsat/requirements.txt diff --git a/datasets/landsat/dataset.yaml b/datasets/landsat/dataset.yaml index c1a3341f2..b3fef29ab 100644 --- a/datasets/landsat/dataset.yaml +++ b/datasets/landsat/dataset.yaml @@ -1,9 +1,9 @@ id: landsat -image: ${{ args.registry }}/pctasks-landsat:20230613.1 +image: ${{ args.registry }}/pctasks-landsat:2026.01.12 args: -- registry -- year-prefix + - registry + - year-prefix code: src: ${{ local.path(./landsat.py) }} @@ -28,7 +28,7 @@ collections: # chunk_storage: # uri: blob://landsateuwest/landsat-c2-etl-data/pctasks/landsat-c2-l1/ - - id: landsat-c2-l2 + - id: landsat-c2-l2 template: ${{ local.path(./collection/landsat-c2-l2) }} class: landsat:LandsatC2Collection asset_storage: @@ -49,7 +49,7 @@ collections: token: ${{ pc.get_token(landsateuwest, landsat-c2) }} chunks: splits: - - depth: 1 # parallelize by Landsat WRS Row + - depth: 1 # parallelize by Landsat WRS Row options: ends_with: _MTL.xml chunk_length: 5000 @@ -57,7 +57,7 @@ collections: token: ${{ pc.get_token(landsateuwest, landsat-c2) }} chunks: splits: - - depth: 1 # parallelize by Landsat WRS Row + - depth: 1 # parallelize by Landsat WRS Row options: ends_with: _MTL.xml chunk_length: 5000 diff --git a/datasets/met-office/collection/met-office-global-deterministic-height/description.md b/datasets/met-office/collection/met-office-global-deterministic-height/description.md new file mode 100644 index 000000000..29a207395 --- /dev/null +++ b/datasets/met-office/collection/met-office-global-deterministic-height/description.md @@ -0,0 +1,56 @@ +This collection offers 1 parameter at 33 available height levels (5m to 6000m) from the Met Office global deterministic 10km forecast. This is a numerical weather prediction forecast for the whole globe, with a resolution of approximately 0.09 degrees i.e. 10km (2,560 x 1,920 grid points). + +The data is available as NetCDF files. It's offered on a free, unsupported basis, so we don't recommend using it for any critical business purposes. + +## Height Levels +Available height levels, in metres (m) above ground, are: +* 5.0 10.0 20.0 30.0 50.0 75.0 100.0 150.0 200.0 250.0 300.0 400.0 500.0 600.0 700.0 800.0 1000.0 1250.0 1500.0 1750.0 2000.0 2250.0 2500.0 2750.0 3000.0 3250.0 3500.0 3750.0 4000.0 4500.0 5000.0 5500.0 6000.0 + +## Timesteps +The following timesteps are available: +* every hour from 0 to 54 hours +* every 3 hours from 57 to 144 hours +* every 6 hours from 150 to 168 hours + +## Update Frequency +The model is run four times each day, with forecast reference times of 00:00, 06:00, 12:00 and 18:00 (UTC). + +The runs at 00:00 and 12:00 provide data for the next 168 hours. The runs at 06:00 and 18:00 provide data for the next 67 hours. + +The forecast reference time represents the nominal data time or start time of a model forecast run, rather than the time when the data is available. + +## Archive length and latency +As of December 2025, the archive contains data from December 2023 onwards. Forecasts will continue to be available for at least two years from their data date. + +The data is typically available 6 hours after the model run time. + +## Technical specs +The data is available as NetCDF files. NetCDF (Network Common Data Form) is an interface for array-orientated data access and a library that supports the interface. It is composed of 3 components: +* Variables store the data +* Dimensions give relevant dimension information for the variables +* Attributes provide auxiliary information about the variables or dataset itself + +NetCDF is used within the atmospheric and oceanic science communities and is network transparent, allowing for it to be accessed by computers that store integers, characters and floating-point numbers. + +Iris supports NetCDF files through reading, writing and handling. Iris implements a model based on the CF conventions, giving a format-agnostic interface for working with data. + +[Find further support on using Iris with NetCDF files.](https://scitools-iris.readthedocs.io/en/stable/) + +## Help us improve the data services we offer +[Join the Met Office research panel](https://forms.office.com/Pages/ResponsePage.aspx?id=YYHxF9cgRkeH_VD-PjtmGdxioYGoFbFIkZuB_q8Fb3VUQkoxRVQzTFdUMzNMVzczWVM5VTc3QTY3MC4u) to help us understand how people interact with weather and climate data, uncover challenges and explore opportunities. + +## How to cite +Met Office global deterministic 10km forecast was accessed on DATE from the Microsoft Planetary Computer (https://zenodo.org/records/7261897). + +## License +British Crown copyright 2023-2025, the Met Office, is licensed under [CC BY-SA](https://creativecommons.org/licenses/by-sa/4.0/deed.en). + +## Providers +[Met Office](https://www.metoffice.gov.uk/) + +See all datasets managed by [Met Office.](https://planetarycomputer.microsoft.com/catalog?filter=met+office) + +## Contact +[servicedesk@metoffice.gov.uk](mailto:servicedesk@metoffice.gov.uk). Service desk is only available Mon – Fri, 09:00 until 17:00 UTC (-1 hour during BST). + +As a non-operational service we aim to respond to any service support enquiries within 3-5 business days. diff --git a/datasets/met-office/collection/met-office-global-deterministic-height/template.json b/datasets/met-office/collection/met-office-global-deterministic-height/template.json new file mode 100644 index 000000000..43a5d2c01 --- /dev/null +++ b/datasets/met-office/collection/met-office-global-deterministic-height/template.json @@ -0,0 +1,271 @@ +{ + "id": "met-office-global-deterministic-height", + "type": "Collection", + "links": [ + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by-sa/4.0/deed.en", + "type": "text/html", + "title": "Creative Commons Attribution-ShareAlike 4.0" + }, + { + "rel": "describedBy", + "href": "https://www.metoffice.gov.uk/services/data/external-data-channels", + "title": "Met Office Dataset Documentation" + } + ], + "title": "Height levels collection Met Office Global 10km deterministic weather forecast", + "assets": { + "thumbnail": { + "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/met-office-global-deterministic-height.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Height levels collection Met Office Global 10km deterministic weather forecast thumbnail" + } + }, + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2023-12-15T00:00:00Z", + null + ] + ] + } + }, + "license": "proprietary", + "keywords": [ + "Met Office", + "Weather", + "Forecast", + "Global", + "Cloud" + ], + "providers": [ + { + "url": "https://www.metoffice.gov.uk/", + "name": "Met Office", + "roles": [ + "producer", + "licensor", + "processor" + ] + }, + { + "url": "https://planetarycomputer.microsoft.com", + "name": "Microsoft", + "roles": [ + "host", + "processor" + ] + } + ], + "stac_version": "1.0.0", + "msft:group_id": "met-office-global-deterministic", + "msft:container": "deterministic", + "stac_extensions": [ + "https://stac-extensions.github.io/forecast/v0.2.0/schema.json", + "https://stac-extensions.github.io/datacube/v2.0.0/schema.json" + ], + "msft:storage_account": "ukmoeuwest", + "msft:short_description": "Met Office Global Deterministic Height Level", + "msft:region": "westeurope", + "item_assets": { + "cloud_amount_on_height_levels": { + "type": "application/netcdf", + "title": "Cloud amount on height levels", + "roles": [ + "data" + ], + "forecast:variable": "cloud_volume_fraction_in_atmosphere_layer" + } + }, + "description": "{{ collection.description }}", + "cube:dimensions": { + "latitude": { + "type": "spatial", + "axis": "y", + "extent": [ + -90, + 90 + ] + }, + "longitude": { + "type": "spatial", + "axis": "x", + "extent": [ + -180, + 180 + ] + }, + "height": { + "type": "spatial", + "values": [ + 5, + 10, + 20, + 30, + 50, + 75, + 100, + 150, + 200, + 250, + 300, + 400, + 500, + 600, + 700, + 800, + 1000, + 1250, + 1500, + 1750, + 2000, + 2250, + 2500, + 2750, + 3000, + 3250, + 3500, + 3750, + 4000, + 4500, + 5000, + 5500, + 6000 + ] + }, + "forecast:reference_datetime": { + "type": "temporal", + "extent": [ + "2023-12-15T00:00:00Z", + null + ] + }, + "forecast:horizon": { + "type": "temporal", + "extent": [ + null, + null + ], + "values": [ + "PT0H", + "PT1H", + "PT2H", + "PT3H", + "PT4H", + "PT5H", + "PT6H", + "PT7H", + "PT8H", + "PT9H", + "PT10H", + "PT11H", + "PT12H", + "PT13H", + "PT14H", + "PT15H", + "PT16H", + "PT17H", + "PT18H", + "PT19H", + "PT20H", + "PT21H", + "PT22H", + "PT23H", + "PT24H", + "PT25H", + "PT26H", + "PT27H", + "PT28H", + "PT29H", + "PT30H", + "PT31H", + "PT32H", + "PT33H", + "PT34H", + "PT35H", + "PT36H", + "PT37H", + "PT38H", + "PT39H", + "PT40H", + "PT41H", + "PT42H", + "PT43H", + "PT44H", + "PT45H", + "PT46H", + "PT47H", + "PT48H", + "PT49H", + "PT50H", + "PT51H", + "PT52H", + "PT53H", + "PT54H", + "PT57H", + "PT60H", + "PT63H", + "PT66H", + "PT69H", + "PT72H", + "PT75H", + "PT78H", + "PT81H", + "PT84H", + "PT87H", + "PT90H", + "PT93H", + "PT96H", + "PT99H", + "PT102H", + "PT105H", + "PT108H", + "PT111H", + "PT114H", + "PT117H", + "PT120H", + "PT123H", + "PT126H", + "PT129H", + "PT132H", + "PT135H", + "PT138H", + "PT141H", + "PT144H", + "PT150H", + "PT156H", + "PT162H", + "PT168H" + ] + } + }, + "cube:variables": { + "cloud_volume_fraction_in_atmosphere_layer": { + "description": "Fraction of horizontal grid square occupied by cloud in layers centred on height levels. The levels are height above ground.", + "dimensions": [ + "latitude", + "longitude", + "height", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + } + } +} \ No newline at end of file diff --git a/datasets/met-office/collection/met-office-global-deterministic-near-surface/description.md b/datasets/met-office/collection/met-office-global-deterministic-near-surface/description.md new file mode 100644 index 000000000..c054b95dd --- /dev/null +++ b/datasets/met-office/collection/met-office-global-deterministic-near-surface/description.md @@ -0,0 +1,89 @@ +This collection offers 48 parameters at near-surface level from the Met Office global deterministic 10km forecast. This is a numerical weather prediction forecast for the whole globe, with a resolution of approximately 0.09 degrees i.e. 10km (2,560 x 1,920 grid points). + +The data is available as NetCDF files. It's offered on a free, unsupported basis, so we don't recommend using it for any critical business purposes. + +## Data collection height +There are 3 forecast heights used within the near-surface this collection: +* Surface: the default collection height +* Screen level: 1.5m above the surface +* Wind parameters: 10m above the surface + +## Timesteps +For most parameters, the following time steps are available, see exceptions below: +* every hour from 0 to 54 hours +* every 3 hours from 57 to 144 hours +* every 6 hours from 150 to 168 hours + +Exceptions (for `accumulation`, `min`, `max` and `mean` parameters): +* Height of orography (height_of_orography) is only available at 0H +* Hourly latent heat flux at surface mean (latent_heat_flux_at_surface_mean-PT01H) is only available every hour from 1 to 54 hours +* 3H latent heat flux at surface mean (latent_heat_flux_at_surface_mean-PT03H) is only available every three hours from 57 to 144 hours +* 6H latent heat flux at surface mean (latent_heat_flux_at_surface_mean-PT06H) is only available every six hours from 150 to 168 hours +* Hourly precipitation accumulation (precipitation_accumulation-PT01H) is only available every hour from 1 to 48 hours +* 3H precipitation accumulation (precipitation_accumulation-PT03H) is only available every three hours from 51 to 144 hours +* 6H precipitation accumulation (precipitation_accumulation-PT06H) is only available every six hours from 150 to 168 hours +* Radiation flux in uv downward at surface (radiation_flux_in_uv_downward_at_surface) has three hourly timesteps from 0 to 144 hours +* Hourly rainfall accumulation (rainfall_accumulation-PT01H) is only available every hour from 1 to 54 hours +* 3H rainfall accumulation (rainfall_accumulation-PT03H) is only available every three hours from 57 to 144 hours +* 6H rainfall accumulation (rainfall_accumulation-PT06H) is only available every six hours from 150 to 168 hours +* Hourly rainfall rate from convection max (rainfall_rate_from_convection_max-PT01H) is only available every hour from 1 to 54 hours +* 3H rainfall rate from convection max (rainfall_rate_from_convection_max-PT03H) is only available every three hours from 57 to 144 hours +* 6H rainfall rate from convection max (rainfall_rate_from_convection_max-PT06H) is only available every six hours from 150 to 168 hours +* Hourly snowfall rate from convection max (snowfall_rate_from_convection_max-PT01H) is only available every hour from 1 to 54 hours +* 3H snowfall rate from convection max (snowfall_rate_from_convection_max-PT03H) is only available every three hours from 57 to 144 hours +* 6H snowfall rate from convection max (snowfall_rate_from_convection_max-PT06H) is only available every six hours from 150 to 168 hours +* Hourly snowfall rate from convection mean (snowfall_rate_from_convection_mean-PT01H) is only available every hour from 1 to 54 hours +* 3H snowfall rate from convection mean (snowfall_rate_from_convection_mean-PT03H) is only available every three hours from 57 to 144 hours +* 6H snowfall rate from convection mean (snowfall_rate_from_convection_mean-PT06H) is only available every six hours from 150 to 168 hours +* Hourly temperature at screen level max (temperature_at_screen_level_max-PT01H) is only available every hour from 1 to 54 hours +* 3H temperature at screen level max (temperature_at_screen_level_max-PT03H) is only available every three hours from 57 to 144 hours +* 6H temperature at screen level max (temperature_at_screen_level_max-PT06H) is only available every six hours from 150 to 168 hours +* Hourly temperature at screen level min (temperature_at_screen_level_min-PT01H) is only available every hour from 1 to 54 hours +* 3H remperature at screen level min (temperature_at_screen_level_min-PT03H) is only available every three hours from 57 to 144 hours +* 6H temperature at screen level min (temperature_at_screen_level_min-PT06H) is only available every six hours from 150 to 168 hours +* Hourly wind gust at 10m max (wind_gust_at_10m_max-PT01H) is only available every hour from 1 to 54 hours +* 3H wind gust at 10m max (wind_gust_at_10m_max-PT03H) is only available every three hours from 57 to 144 hours +* 6H wind gust at 10m max (wind_gust_at_10m_max-PT06H) is only available every six hours from 150 to 168 hours + +## Update Frequency +The model is run four times each day, with forecast reference times of 00:00, 06:00, 12:00 and 18:00 (UTC). + +The runs at 00:00 and 12:00 provide data for the next 168 hours. The runs at 06:00 and 18:00 provide data for the next 67 hours. + +The forecast reference time represents the nominal data time or start time of a model forecast run, rather than the time when the data is available. + +## Archive length and latency +As of December 2025, the archive contains data from December 2023 onwards. Forecasts will continue to be available for at least two years from their data date. + +The data is typically available 6 hours after the model run time. + +## Technical specs +The data is available as NetCDF files. NetCDF (Network Common Data Form) is an interface for array-orientated data access and a library that supports the interface. It is composed of 3 components: +* Variables store the data +* Dimensions give relevant dimension information for the variables +* Attributes provide auxiliary information about the variables or dataset itself + +NetCDF is used within the atmospheric and oceanic science communities and is network transparent, allowing for it to be accessed by computers that store integers, characters and floating-point numbers. + +Iris supports NetCDF files through reading, writing and handling. Iris implements a model based on the CF conventions, giving a format-agnostic interface for working with data. + +[Find further support on using Iris with NetCDF files.](https://scitools-iris.readthedocs.io/en/stable/) + +## Help us improve the data services we offer +[Join the Met Office research panel](https://forms.office.com/Pages/ResponsePage.aspx?id=YYHxF9cgRkeH_VD-PjtmGdxioYGoFbFIkZuB_q8Fb3VUQkoxRVQzTFdUMzNMVzczWVM5VTc3QTY3MC4u) to help us understand how people interact with weather and climate data, uncover challenges and explore opportunities. + +## How to cite +Met Office global deterministic 10km forecast was accessed on DATE from the Microsoft Planetary Computer (https://zenodo.org/records/7261897). + +## License +British Crown copyright 2023-2025, the Met Office, is licensed under [CC BY-SA](https://creativecommons.org/licenses/by-sa/4.0/deed.en). + +## Providers +[Met Office](https://www.metoffice.gov.uk/) + +See all datasets managed by [Met Office.](https://planetarycomputer.microsoft.com/catalog?filter=met+office) + +## Contact +[servicedesk@metoffice.gov.uk](mailto:servicedesk@metoffice.gov.uk). Service desk is only available Mon – Fri, 09:00 until 17:00 UTC (-1 hour during BST). + +As a non-operational service we aim to respond to any service support enquiries within 3-5 business days. diff --git a/datasets/met-office/collection/met-office-global-deterministic-near-surface/template.json b/datasets/met-office/collection/met-office-global-deterministic-near-surface/template.json new file mode 100644 index 000000000..f74ff2662 --- /dev/null +++ b/datasets/met-office/collection/met-office-global-deterministic-near-surface/template.json @@ -0,0 +1,870 @@ +{ + "id": "met-office-global-deterministic-near-surface", + "type": "Collection", + "links": [ + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by-sa/4.0/deed.en", + "type": "text/html", + "title": "Creative Commons Attribution-ShareAlike 4.0" + }, + { + "rel": "describedby", + "href": "https://www.metoffice.gov.uk/services/data/external-data-channels", + "title": "Met Office Dataset Documentation" + } + ], + "title": "Near-surface level collection Met Office global deterministic 10km forecast", + "assets": { + "thumbnail": { + "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/met-office-global-deterministic-near-surface.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Near-surface level collection Met Office global deterministic 10km forecast thumbnail" + } + }, + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2023-12-15T00:00:00Z", + null + ] + ] + } + }, + "license": "proprietary", + "keywords": [ + "Met Office", + "Global", + "Forecast", + "Cloud", + "Fog", + "Heat Flux", + "Precipitation", + "Pressure", + "Radiation", + "Rainfall", + "Humidity", + "Snow", + "Temperature", + "Wind" + ], + "providers": [ + { + "url": "https://www.metoffice.gov.uk/", + "name": "Met Office", + "roles": [ + "producer", + "licensor", + "processor" + ] + }, + { + "url": "https://planetarycomputer.microsoft.com", + "name": "Microsoft", + "roles": [ + "host", + "processor" + ] + } + ], + "stac_version": "1.0.0", + "msft:group_id": "met-office-global-deterministic", + "msft:container": "deterministic", + "stac_extensions": [ + "https://stac-extensions.github.io/forecast/v0.2.0/schema.json", + "https://stac-extensions.github.io/datacube/v2.0.0/schema.json" + ], + "msft:storage_account": "ukmoeuwest", + "msft:short_description": "Met Office Global Deterministic Near Surface Level", + "msft:region": "westeurope", + "item_assets": { + "fog_fraction_at_screen_level": { + "type": "application/netcdf", + "title": "Fog fraction at screen level", + "roles": [ + "data" + ], + "forecast:variable": "fog_area_fraction" + }, + "height_of_orography": { + "type": "application/netcdf", + "title": "Height of orography", + "roles": [ + "data" + ], + "forecast:variable": "surface_altitude" + }, + "latent_heat_flux_at_surface_mean-PT01H": { + "type": "application/netcdf", + "title": "Hourly latent heat flux at surface mean", + "roles": [ + "data" + ], + "forecast:variable": "surface_upward_latent_heat_flux" + }, + "latent_heat_flux_at_surface_mean-PT03H": { + "type": "application/netcdf", + "title": "3H latent heat flux at surface mean", + "roles": [ + "data" + ], + "forecast:variable": "surface_upward_latent_heat_flux" + }, + "latent_heat_flux_at_surface_mean-PT06H": { + "type": "application/netcdf", + "title": "6H latent heat flux at surface mean", + "roles": [ + "data" + ], + "forecast:variable": "surface_upward_latent_heat_flux" + }, + "precipitation_accumulation-PT01H": { + "type": "application/netcdf", + "title": "Hourly precipitation accumulation", + "roles": [ + "data" + ], + "forecast:variable": "lwe_thickness_of_precipitation_amount" + }, + "precipitation_accumulation-PT03H": { + "type": "application/netcdf", + "title": "3H precipitation accumulation", + "roles": [ + "data" + ], + "forecast:variable": "lwe_thickness_of_precipitation_amount" + }, + "precipitation_accumulation-PT06H": { + "type": "application/netcdf", + "title": "6H precipitation accumulation", + "roles": [ + "data" + ], + "forecast:variable": "lwe_thickness_of_precipitation_amount" + }, + "precipitation_rate": { + "type": "application/netcdf", + "title": "Precipitation rate", + "roles": [ + "data" + ], + "forecast:variable": "lwe_precipitation_rate" + }, + "pressure_at_mean_sea_level": { + "type": "application/netcdf", + "title": "Pressure at mean sea level", + "roles": [ + "data" + ], + "forecast:variable": "air_pressure_at_sea_level" + }, + "radiation_flux_in_longwave_downward_at_surface": { + "type": "application/netcdf", + "title": "Radiation flux in longwave downward at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_downwelling_longwave_flux_in_air" + }, + "radiation_flux_in_shortwave_direct_downward_at_surface": { + "type": "application/netcdf", + "title": "Radiation flux in shortwave direct downward at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_direct_downwelling_shortwave_flux_in_air" + }, + "radiation_flux_in_shortwave_total_downward_at_surface": { + "type": "application/netcdf", + "title": "Radiation flux in shortwave total downward at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_downwelling_shortwave_flux_in_air" + }, + "radiation_flux_in_uv_downward_at_surface": { + "type": "application/netcdf", + "title": "Radiation flux in uv downward at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_downwelling_ultraviolet_flux_in_air" + }, + "rainfall_accumulation-PT01H": { + "type": "application/netcdf", + "title": "Hourly rainfall accumulation", + "roles": [ + "data" + ], + "forecast:variable": "thickness_of_rainfall_amount" + }, + "rainfall_accumulation-PT03H": { + "type": "application/netcdf", + "title": "3H rainfall accumulation", + "roles": [ + "data" + ], + "forecast:variable": "thickness_of_rainfall_amount" + }, + "rainfall_accumulation-PT06H": { + "type": "application/netcdf", + "title": "6H rainfall accumulation", + "roles": [ + "data" + ], + "forecast:variable": "thickness_of_rainfall_amount" + }, + "rainfall_rate": { + "type": "application/netcdf", + "title": "Rainfall rate", + "roles": [ + "data" + ], + "forecast:variable": "rainfall_rate" + }, + "rainfall_rate_from_convection": { + "type": "application/netcdf", + "title": "Rainfall rate from convection", + "roles": [ + "data" + ], + "forecast:variable": "convective_rainfall_rate" + }, + "rainfall_rate_from_convection_max-PT01H": { + "type": "application/netcdf", + "title": "Hourly rainfall rate from convection max", + "roles": [ + "data" + ], + "forecast:variable": "convective_rainfall_rate" + }, + "rainfall_rate_from_convection_max-PT03H": { + "type": "application/netcdf", + "title": "3H rainfall rate from convection max", + "roles": [ + "data" + ], + "forecast:variable": "convective_rainfall_rate" + }, + "rainfall_rate_from_convection_max-PT06H": { + "type": "application/netcdf", + "title": "6H rainfall rate from convection max", + "roles": [ + "data" + ], + "forecast:variable": "convective_rainfall_rate" + }, + "relative_humidity_at_screen_level": { + "type": "application/netcdf", + "title": "Relative humidity at screen level", + "roles": [ + "data" + ], + "forecast:variable": "relative_humidity" + }, + "snow_depth_water_equivalent": { + "type": "application/netcdf", + "title": "Snow depth water equivalent", + "roles": [ + "data" + ], + "forecast:variable": "lwe_thickness_of_surface_snow_amount" + }, + "snowfall_rate": { + "type": "application/netcdf", + "title": "Snowfall rate", + "roles": [ + "data" + ], + "forecast:variable": "lwe_snowfall_rate" + }, + "snowfall_rate_from_convection": { + "type": "application/netcdf", + "title": "Snowfall rate from convection", + "roles": [ + "data" + ], + "forecast:variable": "lwe_convective_snowfall_rate" + }, + "snowfall_rate_from_convection_max-PT01H": { + "type": "application/netcdf", + "title": "Hourly snowfall rate from convection max", + "roles": [ + "data" + ], + "forecast:variable": "lwe_convective_snowfall_rate" + }, + "snowfall_rate_from_convection_max-PT03H": { + "type": "application/netcdf", + "title": "3H snowfall rate from convection max", + "roles": [ + "data" + ], + "forecast:variable": "lwe_convective_snowfall_rate" + }, + "snowfall_rate_from_convection_max-PT06H": { + "type": "application/netcdf", + "title": "6H snowfall rate from convection max", + "roles": [ + "data" + ], + "forecast:variable": "lwe_convective_snowfall_rate" + }, + "snowfall_rate_from_convection_mean-PT01H": { + "type": "application/netcdf", + "title": "Hourly snowfall rate from convection mean", + "roles": [ + "data" + ], + "forecast:variable": "lwe_convective_snowfall_rate" + }, + "snowfall_rate_from_convection_mean-PT03H": { + "type": "application/netcdf", + "title": "3H snowfall rate from convection mean", + "roles": [ + "data" + ], + "forecast:variable": "lwe_convective_snowfall_rate" + }, + "snowfall_rate_from_convection_mean-PT06H": { + "type": "application/netcdf", + "title": "6H snowfall rate from convection mean", + "roles": [ + "data" + ], + "forecast:variable": "lwe_convective_snowfall_rate" + }, + "temperature_at_screen_level": { + "type": "application/netcdf", + "title": "Temperature at screen level", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "temperature_at_screen_level_max-PT01H": { + "type": "application/netcdf", + "title": "Hourly temperature at screen level max", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "temperature_at_screen_level_max-PT03H": { + "type": "application/netcdf", + "title": "3H temperature at screen level max", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "temperature_at_screen_level_max-PT06H": { + "type": "application/netcdf", + "title": "6H temperature at screen level max", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "temperature_at_screen_level_min-PT01H": { + "type": "application/netcdf", + "title": "Hourly temperature at screen level min", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "temperature_at_screen_level_min-PT03H": { + "type": "application/netcdf", + "title": "3H remperature at screen level min", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "temperature_at_screen_level_min-PT06H": { + "type": "application/netcdf", + "title": "6H temperature at screen level min", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "temperature_at_surface": { + "type": "application/netcdf", + "title": "Temperature at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_temperature" + }, + "temperature_of_dew_point_at_screen_level": { + "type": "application/netcdf", + "title": "Dew point temperature at screen level", + "roles": [ + "data" + ], + "forecast:variable": "dew_point_temperature" + }, + "visibility_at_screen_level": { + "type": "application/netcdf", + "title": "Visibility at screen level", + "roles": [ + "data" + ], + "forecast:variable": "visibility_in_air" + }, + "wind_direction_at_10m": { + "type": "application/netcdf", + "title": "Wind direction at 10m", + "roles": [ + "data" + ], + "forecast:variable": "wind_from_direction" + }, + "wind_gust_at_10m": { + "type": "application/netcdf", + "title": "Wind gust at 10m", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed_of_gust" + }, + "wind_gust_at_10m_max-PT01H": { + "type": "application/netcdf", + "title": "Hourly wind gust at 10m max", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed_of_gust" + }, + "wind_gust_at_10m_max-PT03H": { + "type": "application/netcdf", + "title": "3H wind gust at 10m max", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed_of_gust" + }, + "wind_gust_at_10m_max-PT06H": { + "type": "application/netcdf", + "title": "6H wind gust at 10m max", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed_of_gust" + }, + "wind_speed_at_10m": { + "type": "application/netcdf", + "title": "Wind speed at 10m", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed" + } + }, + "description": "{{ collection.description }}", + "cube:dimensions": { + "latitude": { + "type": "spatial", + "axis": "y", + "extent": [ + -90, + 90 + ] + }, + "longitude": { + "type": "spatial", + "axis": "x", + "extent": [ + -180, + 180 + ] + }, + "forecast:reference_datetime": { + "type": "temporal", + "extent": [ + "2023-12-15T00:00:00Z", + null + ] + }, + "forecast:horizon": { + "type": "temporal", + "extent": [ + null, + null + ], + "values": [ + "PT0H", + "PT1H", + "PT2H", + "PT3H", + "PT4H", + "PT5H", + "PT6H", + "PT7H", + "PT8H", + "PT9H", + "PT10H", + "PT11H", + "PT12H", + "PT13H", + "PT14H", + "PT15H", + "PT16H", + "PT17H", + "PT18H", + "PT19H", + "PT20H", + "PT21H", + "PT22H", + "PT23H", + "PT24H", + "PT25H", + "PT26H", + "PT27H", + "PT28H", + "PT29H", + "PT30H", + "PT31H", + "PT32H", + "PT33H", + "PT34H", + "PT35H", + "PT36H", + "PT37H", + "PT38H", + "PT39H", + "PT40H", + "PT41H", + "PT42H", + "PT43H", + "PT44H", + "PT45H", + "PT46H", + "PT47H", + "PT48H", + "PT49H", + "PT50H", + "PT51H", + "PT52H", + "PT53H", + "PT54H", + "PT57H", + "PT60H", + "PT63H", + "PT66H", + "PT69H", + "PT72H", + "PT75H", + "PT78H", + "PT81H", + "PT84H", + "PT87H", + "PT90H", + "PT93H", + "PT96H", + "PT99H", + "PT102H", + "PT105H", + "PT108H", + "PT111H", + "PT114H", + "PT117H", + "PT120H", + "PT123H", + "PT126H", + "PT129H", + "PT132H", + "PT135H", + "PT138H", + "PT141H", + "PT144H", + "PT150H", + "PT156H", + "PT162H", + "PT168H" + ] + } + }, + "cube:variables": { + "fog_area_fraction": { + "description": "Here fog means a visibility of 1000 m or lower. The reduction in visibility is caused water droplets or minute ice crystals forming close to the surface. This quantity represents the fraction of horizontal grid square occupied by fog. An alternative interpretation is that this represents the fractional probability of fog being present at any location in the grid square.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "surface_altitude": { + "description": "Altitude or (geometric) height above the geoid of the surface (ground). It is worth remembering that orographic height can be negative, particularly in some of the inland sea areas (e.g. Caspian Sea at -27m and Dead Sea at -430m)", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "surface_upward_latent_heat_flux": { + "description": "The latent heat flux, averaged over a model time step, at the surface.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "lwe_thickness_of_precipitation_amount": { + "description": "Implied depth of the layer of liquid water which has been deposited on the surface in the previous 6 hours. This includes rain, snow and hail, with the ice phase precipitation being considered as a liquid water equivalent (LWE) value. It includes the contribution from the model convection scheme if this is invoked.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "lwe_precipitation_rate": { + "description": "Instantaneous rate at which liquid water (as a depth) was being deposited on the surface. This includes rain, snow and hail with the ice phase precipitation being considered as a liquid water equivalent (LWE) value. It includes the contribution from the model convection scheme if this is invoked.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "air_pressure_at_sea_level": { + "description": "Air pressure at mean sea level which is close to the geoid in sea areas. Air pressure at sea level is the quantity often abbreviated as MSLP or PMSL.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "Pa" + }, + "surface_downwelling_longwave_flux_in_air": { + "description": "Longwave radiation at the surface from above directed at the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "surface_direct_downwelling_shortwave_flux_in_air": { + "description": "Direct component of shortwave radiation at the surface from above directed at the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "surface_downwelling_shortwave_flux_in_air": { + "description": "Total shortwave radiation at the surface from above directed at the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "surface_downwelling_ultraviolet_flux_in_air": { + "description": "Ultaviolet radiation at the surface from above directed at the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "thickness_of_rainfall_amount": { + "description": "Implied depth of the rain produced by the model precipitation scheme which has been deposited on the surface in the previous 6 hours. This excludes the rain produced by the model convection scheme.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "rainfall_rate": { + "description": "Instantaneous rate at which rain (as a depth) was being produced by the model precipitation scheme and deposited on the surface. This excludes the rain produced by the model convection scheme.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "convective_rainfall_rate": { + "description": "Maximum instantaneous rate at which rain (as a depth) has been produced by the model convection scheme and deposited on the surface in the previous 6 hours.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "relative_humidity": { + "description": "Fractional relative humidity (ratio of the partial pressure of water vapour to the equilibrium vapour pressure of water) at screen level (1.5m above the surface.)", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "lwe_thickness_of_surface_snow_amount": { + "description": "Liquid water equivalent (LWE) depth of the snow lying on the surface (ground). Typically water is 10 times as dense as snow, so multiplying by 10 gives an approximate depth of the snow, although wet snow can be significantly denser and powder snow much less dense.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "lwe_snowfall_rate": { + "description": "Instantaneous rate at which liquid water equivalent (LWE) snow (as a depth) was produced by the model precipitation scheme and deposited on the surface. This excludes the snow produced by the the model convection scheme.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "lwe_convective_snowfall_rate": { + "description": "Mean rate at which liquid water equivalent (LWE) snow (as a depth) has been produced by the model convection scheme and deposited on the surface in the previous 6 hours.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "air_temperature": { + "description": "Minimum instantaneous air temperature at screen level (1.5m) in the previous 6 hours.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "surface_temperature": { + "description": "Temperature at the surface interface between the air and the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "dew_point_temperature": { + "description": "Instantaneous dew point temperature (temperature at which a parcel of air reaches saturation upon being cooled at constant pressure and specific humidity) at screen level (1.5m).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "visibility_in_air": { + "description": "Horizontal distance at which something can be seen horizontally from screen level (1.5m).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "wind_from_direction": { + "description": "Wind at 10m above the surface is defined as a two-dimensional (horizontal) air velocity vector with no vertical component. In meteorological reports the direction of the wind vector is given as the direction from which it is blowing. Note: This with wind speed at 10m replaces x wind at 10m and y wind at 10m.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "degrees" + }, + "wind_speed_of_gust": { + "description": "Maximum diagnosed instantaneous wind gust at 10m in the previous 6 hours. This can be considered as the extreme wind speed that might be experienced in this period.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "wind_speed": { + "description": "Wind at 10m above the surface is defined as a two-dimensional (horizontal) air velocity vector with no vertical component. The speed is the magnitude of velocity.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + } + } +} \ No newline at end of file diff --git a/datasets/met-office/collection/met-office-global-deterministic-pressure/description.md b/datasets/met-office/collection/met-office-global-deterministic-pressure/description.md new file mode 100644 index 000000000..bb05ccc7b --- /dev/null +++ b/datasets/met-office/collection/met-office-global-deterministic-pressure/description.md @@ -0,0 +1,56 @@ +This collection offers 8 parameters at 33 available pressure levels (from 100000Pa to 1000Pa) from the Met Office global deterministic 10km forecast. This is a numerical weather prediction forecast for the whole globe, with a resolution of approximately 0.09 degrees i.e. 10km (2,560 x 1,920 grid points). + +The data is available as NetCDF files. It's offered on a free, unsupported basis, so we don't recommend using it for any critical business purposes. + +## Pressure Levels +Available pressure levels, in Pascals (Pa), are: +* 100000.0 97500.0 95000.0 92500.0 90000.0 85000.0 80000.0 75000.0 70000.0 65000.0 60000.0 55000.0 50000.0 45000.0 40000.0 37500.0 35000.0 32500.0 30000.0 27500.0 25000.0 22500.0 20000.0 17500.0 15000.0 12500.0 10000.0 7000.0 5000.0 4000.0 3000.0 2000.0 1000.0 + +## Timesteps +The following timesteps are available: +* every hour from 0 to 54 hours (for most parameters, see parameter table for exceptions) +* every 3 hours from 57 to 144 hours +* every 6 hours from 150 to 168 hours + +## Update Frequency +The model is run four times each day, with forecast reference times of 00:00, 06:00, 12:00 and 18:00 (UTC). + +The runs at 00:00 and 12:00 provide data for the next 168 hours. The runs at 06:00 and 18:00 provide data for the next 67 hours. + +The forecast reference time represents the nominal data time or start time of a model forecast run, rather than the time when the data is available. + +## Archive length and latency +As of December 2025, the archive contains data from December 2023 onwards. Forecasts will continue to be available for at least two years from their data date. + +The data is typically available 6 hours after the model run time. + +## Technical specs +The data is available as NetCDF files. NetCDF (Network Common Data Form) is an interface for array-orientated data access and a library that supports the interface. It is composed of 3 components: +* Variables store the data +* Dimensions give relevant dimension information for the variables +* Attributes provide auxiliary information about the variables or dataset itself + +NetCDF is used within the atmospheric and oceanic science communities and is network transparent, allowing for it to be accessed by computers that store integers, characters and floating-point numbers. + +Iris supports NetCDF files through reading, writing and handling. Iris implements a model based on the CF conventions, giving a format-agnostic interface for working with data. + +[Find further support on using Iris with NetCDF files.](https://scitools-iris.readthedocs.io/en/stable/) + +## Help us improve the data services we offer +[Join the Met Office research panel](https://forms.office.com/Pages/ResponsePage.aspx?id=YYHxF9cgRkeH_VD-PjtmGdxioYGoFbFIkZuB_q8Fb3VUQkoxRVQzTFdUMzNMVzczWVM5VTc3QTY3MC4u) to help us understand how people interact with weather and climate data, uncover challenges and explore opportunities. + +## How to cite +Met Office global deterministic 10km forecast was accessed on DATE from the Microsoft Planetary Computer (https://zenodo.org/records/7261897). + +## License +British Crown copyright 2023-2025, the Met Office, is licensed under [CC BY-SA](https://creativecommons.org/licenses/by-sa/4.0/deed.en). + +## Providers +[Met Office](https://www.metoffice.gov.uk/) + +See all datasets managed by [Met Office.](https://planetarycomputer.microsoft.com/catalog?filter=met+office) + +## Contact +[servicedesk@metoffice.gov.uk](mailto:servicedesk@metoffice.gov.uk). Service desk is only available Mon – Fri, 09:00 until 17:00 UTC (-1 hour during BST). + +As a non-operational service we aim to respond to any service support enquiries within 3-5 business days. diff --git a/datasets/met-office/collection/met-office-global-deterministic-pressure/template.json b/datasets/met-office/collection/met-office-global-deterministic-pressure/template.json new file mode 100644 index 000000000..ae76ccb28 --- /dev/null +++ b/datasets/met-office/collection/met-office-global-deterministic-pressure/template.json @@ -0,0 +1,397 @@ +{ + "id": "met-office-global-deterministic-pressure", + "type": "Collection", + "links": [ + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by-sa/4.0/deed.en", + "type": "text/html", + "title": "Creative Commons Attribution-ShareAlike 4.0" + }, + { + "rel": "describedBy", + "href": "https://www.metoffice.gov.uk/services/data/external-data-channels", + "title": "Met Office Dataset Documentation" + } + ], + "title": "Pressure levels collection Met Office Global 10km deterministic weather forecast", + "assets": { + "thumbnail": { + "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/met-office-global-deterministic-pressure.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Pressure levels collection Met Office Global 10km deterministic weather forecast thumbnail" + } + }, + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2023-12-15T00:00:00Z", + null + ] + ] + } + }, + "license": "proprietary", + "keywords": [ + "MetOffice", + "Global", + "Cloud" + ], + "providers": [ + { + "url": "https://www.metoffice.gov.uk/", + "name": "Met Office", + "roles": [ + "producer", + "licensor", + "processor" + ] + }, + { + "url": "https://planetarycomputer.microsoft.com", + "name": "Microsoft", + "roles": [ + "host", + "processor" + ] + } + ], + "stac_version": "1.0.0", + "msft:group_id": "met-office-global-deterministic", + "msft:container": "deterministic", + "stac_extensions": [ + "https://stac-extensions.github.io/forecast/v0.2.0/schema.json", + "https://stac-extensions.github.io/datacube/v2.0.0/schema.json" + ], + "msft:storage_account": "ukmoeuwest", + "msft:short_description": "Met Office Global Deterministic Pressure Level", + "msft:region": "westeurope", + "item_assets": { + "geopotential_height_on_pressure_levels": { + "type": "application/netcdf", + "title": "Geopotential height on pressure levels", + "roles": [ + "data" + ], + "forecast:variable": "geopotential_height" + }, + "height_ASL_on_pressure_levels": { + "type": "application/netcdf", + "title": "Height Above Sea Level on pressure levels", + "roles": [ + "data" + ], + "forecast:variable": "geopotential_height" + }, + "relative_humidity_on_pressure_levels": { + "type": "application/netcdf", + "title": "Relative humidity on pressure levels", + "roles": [ + "data" + ], + "forecast:variable": "relative_humidity" + }, + "temperature_on_pressure_levels": { + "type": "application/netcdf", + "title": "Temperature on pressure levels", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "wet_bulb_potential_temperature_on_pressure_levels": { + "type": "application/netcdf", + "title": "Wet bulb potential temperature on pressure levels", + "roles": [ + "data" + ], + "forecast:variable": "wet_bulb_potential_temperature" + }, + "wind_direction_on_pressure_levels": { + "type": "application/netcdf", + "title": "Wind direction on pressure levels", + "roles": [ + "data" + ], + "forecast:variable": "wind_from_direction" + }, + "wind_speed_on_pressure_levels": { + "type": "application/netcdf", + "title": "Wind speed on pressure levels", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed" + }, + "wind_vertical_velocity_on_pressure_levels": { + "type": "application/netcdf", + "title": "Wind vertical velocity on pressure levels", + "roles": [ + "data" + ], + "forecast:variable": "upward_air_velocity" + } + }, + "description": "{{ collection.description }}", + "cube:dimensions": { + "latitude": { + "type": "spatial", + "axis": "y", + "extent": [ + -90, + 90 + ] + }, + "longitude": { + "type": "spatial", + "axis": "x", + "extent": [ + -180, + 180 + ] + }, + "pressure_levels": { + "type": "spatial", + "values": [ + 100000, + 97500, + 95000, + 92500, + 90000, + 85000, + 80000, + 75000, + 70000, + 6500, + 60000, + 55000, + 50000, + 45000, + 40000, + 37500, + 35000, + 32500, + 30000, + 27500, + 25000, + 22500, + 20000, + 17500, + 15000, + 12500, + 10000, + 7000, + 5000, + 4000, + 3000, + 2000, + 1000 + ] + }, + "forecast:reference_datetime": { + "type": "temporal", + "extent": [ + "2023-12-15T00:00:00Z", + null + ] + }, + "forecast:horizon": { + "type": "temporal", + "extent": [ + null, + null + ], + "values": [ + "PT0H", + "PT1H", + "PT2H", + "PT3H", + "PT4H", + "PT5H", + "PT6H", + "PT7H", + "PT8H", + "PT9H", + "PT10H", + "PT11H", + "PT12H", + "PT13H", + "PT14H", + "PT15H", + "PT16H", + "PT17H", + "PT18H", + "PT19H", + "PT20H", + "PT21H", + "PT22H", + "PT23H", + "PT24H", + "PT25H", + "PT26H", + "PT27H", + "PT28H", + "PT29H", + "PT30H", + "PT31H", + "PT32H", + "PT33H", + "PT34H", + "PT35H", + "PT36H", + "PT37H", + "PT38H", + "PT39H", + "PT40H", + "PT41H", + "PT42H", + "PT43H", + "PT44H", + "PT45H", + "PT46H", + "PT47H", + "PT48H", + "PT49H", + "PT50H", + "PT51H", + "PT52H", + "PT53H", + "PT54H", + "PT57H", + "PT60H", + "PT63H", + "PT66H", + "PT69H", + "PT72H", + "PT75H", + "PT78H", + "PT81H", + "PT84H", + "PT87H", + "PT90H", + "PT93H", + "PT96H", + "PT99H", + "PT102H", + "PT105H", + "PT108H", + "PT111H", + "PT114H", + "PT117H", + "PT120H", + "PT123H", + "PT126H", + "PT129H", + "PT132H", + "PT135H", + "PT138H", + "PT141H", + "PT144H", + "PT150H", + "PT156H", + "PT162H", + "PT168H" + ] + } + }, + "cube:variables": { + "geopotential_height": { + "description": "Height above sea level of the pressure levels. Note: from January 2026, this parameter will be replaced by geopotential_height_on_pressure_levels", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "relative_humidity": { + "description": "Fractional relative humidity (ratio of the partial pressure of water vapour to the equilibrium vapour pressure of water) on pressure levels", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "air_temperature": { + "description": "Air temperature on pressure levels", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "wet_bulb_potential_temperature": { + "description": "Wet bulb potential temperature on pressure levels. Wet bulb potential temperature is defined as the temperature that a parcel of air would have if it were brought down to the standard pressure of 1000hPa following a saturated adiabatic lapse rate.", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "wind_from_direction": { + "description": "Wind direction on pressure levels is defined as a two-dimensional (horizontal) air velocity vector with no vertical component. In meteorological reports the direction of the wind vector is given as the direction from which it is blowing.", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "degrees" + }, + "wind_speed": { + "description": "Wind speed on pressure levels is defined as a two-dimensional (horizontal) air velocity vector with no vertical component. The speed is the magnitude of velocity.", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "upward_air_velocity": { + "description": "Speed of the vertical component of the air motion at a pressure levels. Upwards is positive and downwards is negative.", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + } + } +} \ No newline at end of file diff --git a/datasets/met-office/collection/met-office-global-deterministic-whole-atmosphere/description.md b/datasets/met-office/collection/met-office-global-deterministic-whole-atmosphere/description.md new file mode 100644 index 000000000..7064bb84b --- /dev/null +++ b/datasets/met-office/collection/met-office-global-deterministic-whole-atmosphere/description.md @@ -0,0 +1,52 @@ +This collection offers 14 whole-atmosphere parameters from the Met Office global deterministic 10km forecast. This is a numerical weather prediction forecast for the whole globe, with a resolution of approximately 0.09 degrees i.e. 10km (2,560 x 1,920 grid points). + +The data is available as NetCDF files. It's offered on a free, unsupported basis, so we don't recommend using it for any critical business purposes. + +## Timesteps +The following timesteps are available: +* every hour from 0 to 54 hours (for most parameters, see parameter table for exceptions) +* every 3 hours from 57 to 144 hours +* every 6 hours from 150 to 168 hours + +## Update Frequency +The model is run four times each day, with forecast reference times of 00:00, 06:00, 12:00 and 18:00 (UTC). + +The runs at 00:00 and 12:00 provide data for the next 168 hours. The runs at 06:00 and 18:00 provide data for the next 67 hours. + +The forecast reference time represents the nominal data time or start time of a model forecast run, rather than the time when the data is available. + +## Archive length and latency +As of December 2025, the archive contains data from December 2023 onwards. Forecasts will continue to be available for at least two years from their data date. + +The data is typically available 6 hours after the model run time. + +## Technical specs +The data is available as NetCDF files. NetCDF (Network Common Data Form) is an interface for array-orientated data access and a library that supports the interface. It is composed of 3 components: +* Variables store the data +* Dimensions give relevant dimension information for the variables +* Attributes provide auxiliary information about the variables or dataset itself + +NetCDF is used within the atmospheric and oceanic science communities and is network transparent, allowing for it to be accessed by computers that store integers, characters and floating-point numbers. + +Iris supports NetCDF files through reading, writing and handling. Iris implements a model based on the CF conventions, giving a format-agnostic interface for working with data. + +[Find further support on using Iris with NetCDF files.](https://scitools-iris.readthedocs.io/en/stable/) + +## Help us improve the data services we offer +[Join the Met Office research panel](https://forms.office.com/Pages/ResponsePage.aspx?id=YYHxF9cgRkeH_VD-PjtmGdxioYGoFbFIkZuB_q8Fb3VUQkoxRVQzTFdUMzNMVzczWVM5VTc3QTY3MC4u) to help us understand how people interact with weather and climate data, uncover challenges and explore opportunities. + +## How to cite +Met Office global deterministic 10km forecast was accessed on DATE from the Microsoft Planetary Computer (https://zenodo.org/records/7261897). + +## License +British Crown copyright 2023-2025, the Met Office, is licensed under [CC BY-SA](https://creativecommons.org/licenses/by-sa/4.0/deed.en). + +## Providers +[Met Office](https://www.metoffice.gov.uk/) + +See all datasets managed by [Met Office.](https://planetarycomputer.microsoft.com/catalog?filter=met+office) + +## Contact +[servicedesk@metoffice.gov.uk](mailto:servicedesk@metoffice.gov.uk). Service desk is only available Mon – Fri, 09:00 until 17:00 UTC (-1 hour during BST). + +As a non-operational service we aim to respond to any service support enquiries within 3-5 business days. diff --git a/datasets/met-office/collection/met-office-global-deterministic-whole-atmosphere/template.json b/datasets/met-office/collection/met-office-global-deterministic-whole-atmosphere/template.json new file mode 100644 index 000000000..7d1382d5a --- /dev/null +++ b/datasets/met-office/collection/met-office-global-deterministic-whole-atmosphere/template.json @@ -0,0 +1,455 @@ +{ + "id": "met-office-global-deterministic-whole-atmosphere", + "type": "Collection", + "links": [ + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by-sa/4.0/deed.en", + "type": "text/html", + "title": "Creative Commons Attribution-ShareAlike 4.0" + }, + { + "rel": "describedBy", + "href": "https://www.metoffice.gov.uk/services/data/external-data-channels", + "title": "Met Office Dataset Documentation" + } + ], + "title": "Whole Atmosphere collection Met Office Global 10km deterministic weather forecast", + "assets": { + "thumbnail": { + "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/met-office-global-deterministic-whole-atmosphere.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Whole Atmosphere collection Met Office Global 10km deterministic weather forecast thumbnail" + } + }, + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2023-12-15T00:00:00Z", + null + ] + ] + } + }, + "license": "proprietary", + "keywords": [ + "MetOffice", + "Global", + "Cloud" + ], + "providers": [ + { + "url": "https://www.metoffice.gov.uk/", + "name": "Met Office", + "roles": [ + "producer", + "licensor", + "processor" + ] + }, + { + "url": "https://planetarycomputer.microsoft.com", + "name": "Microsoft", + "roles": [ + "host", + "processor" + ] + } + ], + "stac_version": "1.0.0", + "msft:group_id": "met-office-global-deterministic", + "msft:container": "deterministic", + "stac_extensions": [ + "https://stac-extensions.github.io/forecast/v0.2.0/schema.json", + "https://stac-extensions.github.io/datacube/v2.0.0/schema.json" + ], + "msft:storage_account": "ukmoeuwest", + "msft:short_description": "Met Office Global Deterministic Whole Atmosphere Level", + "msft:region": "westeurope", + "item_assets": { + "CAPE_mixed_layer_lowest_500m": { + "type": "application/netcdf", + "title": "CAPE mixed layer lowest 500m", + "roles": [ + "data" + ], + "forecast:variable": "atmosphere_convective_available_potential_energy" + }, + "CAPE_most_unstable_below_500hPa": { + "type": "application/netcdf", + "title": "CAPE most unstable below 500hPa", + "roles": [ + "data" + ], + "forecast:variable": "atmosphere_convective_available_potential_energy" + }, + "CAPE_surface": { + "type": "application/netcdf", + "title": "CAPE surface", + "roles": [ + "data" + ], + "forecast:variable": "atmosphere_convective_available_potential_energy_wrt_surface" + }, + "CIN_mixed_layer_lowest_500m": { + "type": "application/netcdf", + "title": "CIN mixed layer lowest 500m", + "roles": [ + "data" + ], + "forecast:variable": "atmosphere_convective_inhibition" + }, + "CIN_most_unstable_below_500hPa": { + "type": "application/netcdf", + "title": "CIN most unstable below 500hPa", + "roles": [ + "data" + ], + "forecast:variable": "atmosphere_convective_inhibition" + }, + "CIN_surface": { + "type": "application/netcdf", + "title": "CIN surface", + "roles": [ + "data" + ], + "forecast:variable": "atmosphere_convective_inhibition_wrt_surface" + }, + "cloud_amount_below_1000ft_ASL": { + "type": "application/netcdf", + "title": "Cloud amount below 1000ft ASL", + "roles": [ + "data" + ], + "forecast:variable": "cloud_area_fraction_assuming_only_consider_surface_to_1000_feet_asl" + }, + "cloud_amount_of_high_cloud": { + "type": "application/netcdf", + "title": "Cloud amount of high cloud", + "roles": [ + "data" + ], + "forecast:variable": "high_type_cloud_area_fraction" + }, + "cloud_amount_of_low_cloud": { + "type": "application/netcdf", + "title": "Cloud amount of low cloud", + "roles": [ + "data" + ], + "forecast:variable": "low_type_cloud_area_fraction" + }, + "cloud_amount_of_medium_cloud": { + "type": "application/netcdf", + "title": "Cloud amount of medium cloud", + "roles": [ + "data" + ], + "forecast:variable": "medium_type_cloud_area_fraction" + }, + "cloud_amount_of_total_cloud": { + "type": "application/netcdf", + "title": "Cloud amount of total cloud", + "roles": [ + "data" + ], + "forecast:variable": "cloud_area_fraction" + }, + "cloud_amount_of_total_convective_cloud": { + "type": "application/netcdf", + "title": "Cloud amount of total convective cloud", + "roles": [ + "data" + ], + "forecast:variable": "convective_cloud_area_fraction" + }, + "pressure_at_tropopause": { + "type": "application/netcdf", + "title": "Pressure at tropopause", + "roles": [ + "data" + ], + "forecast:variable": "tropopause_air_pressure" + }, + "temperature_at_tropopause": { + "type": "application/netcdf", + "title": "Temperature at tropopause", + "roles": [ + "data" + ], + "forecast:variable": "tropopause_air_temperature" + } + }, + "description": "{{ collection.description }}", + "cube:dimensions": { + "latitude": { + "type": "spatial", + "axis": "y", + "extent": [ + -90, + 90 + ] + }, + "longitude": { + "type": "spatial", + "axis": "x", + "extent": [ + -180, + 180 + ] + }, + "forecast:reference_datetime": { + "type": "temporal", + "extent": [ + "2023-12-15T00:00:00Z", + null + ] + }, + "forecast:horizon": { + "type": "temporal", + "extent": [ + null, + null + ], + "values": [ + "PT0H", + "PT1H", + "PT2H", + "PT3H", + "PT4H", + "PT5H", + "PT6H", + "PT7H", + "PT8H", + "PT9H", + "PT10H", + "PT11H", + "PT12H", + "PT13H", + "PT14H", + "PT15H", + "PT16H", + "PT17H", + "PT18H", + "PT19H", + "PT20H", + "PT21H", + "PT22H", + "PT23H", + "PT24H", + "PT25H", + "PT26H", + "PT27H", + "PT28H", + "PT29H", + "PT30H", + "PT31H", + "PT32H", + "PT33H", + "PT34H", + "PT35H", + "PT36H", + "PT37H", + "PT38H", + "PT39H", + "PT40H", + "PT41H", + "PT42H", + "PT43H", + "PT44H", + "PT45H", + "PT46H", + "PT47H", + "PT48H", + "PT49H", + "PT50H", + "PT51H", + "PT52H", + "PT53H", + "PT54H", + "PT57H", + "PT60H", + "PT63H", + "PT66H", + "PT69H", + "PT72H", + "PT75H", + "PT78H", + "PT81H", + "PT84H", + "PT87H", + "PT90H", + "PT93H", + "PT96H", + "PT99H", + "PT102H", + "PT105H", + "PT108H", + "PT111H", + "PT114H", + "PT117H", + "PT120H", + "PT123H", + "PT126H", + "PT129H", + "PT132H", + "PT135H", + "PT138H", + "PT141H", + "PT144H", + "PT150H", + "PT156H", + "PT162H", + "PT168H" + ] + } + }, + "cube:variables": { + "atmosphere_convective_available_potential_energy": { + "description": "CAPE (Convective Available Potential Energy) calculated for the most unstable parcel where the most unstable parcel is defined as the parcel with the highest fixed level CAPE launched from any level (including screen-level=1.5m) within 500hPa of the surface pressure.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "J/kg" + }, + "atmosphere_convective_available_potential_energy_wrt_surface": { + "description": "Value of CAPE (Convective Available Potential Energy) calculated for a surface based parcel where a surface based parcel is defined as a parcel initiated with thermodynamic properties at screen level height (1.5m) i.e. the parcel is launched from screen level.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "J/kg" + }, + "atmosphere_convective_inhibition": { + "description": "Convective Inhibition calculated as the additional energy required to lift the most unstable parcel to its level of free convection. Where most unstable parcel is defined as the parcel with the highest fixed-level CAPE launched from any level (including screen-level) within 500 hPa of the surface pressure.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "J/kg" + }, + "atmosphere_convective_inhibition_wrt_surface": { + "description": "Convective Inhibition calculated as the additional energy required to lift a surface based parcel (i.e. a parcel launched from screen-level (1.5m)) to its level of free convection.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "J/kg" + }, + "cloud_area_fraction_assuming_only_consider_surface_to_1000_feet_asl": { + "description": "Fraction of horizontal grid square occupied by cloud below 1000 feet above sea level.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "high_type_cloud_area_fraction": { + "description": "Fraction of horizontal grid square occupied by cloud in the high-level cloud height range (from the lowest model layer containing the 5574m height level up to but excluding the lowest model layer containing 13608m height level).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "low_type_cloud_area_fraction": { + "description": "Fraction of horizontal grid square occupied by cloud in the low-level cloud height range (from the lowest model layer containing the 111m height level up to but excluding the lowest model layer containing 1949m height level).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "medium_type_cloud_area_fraction": { + "description": "Fraction of horizontal grid square occupied by cloud in the mid-level cloud height range (from the lowest model layer containing the 1949m height level up to but excluding the lowest model layer containing 5574m height level).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "cloud_area_fraction": { + "description": "Fraction of horizontal grid square occupied by cloud as diagnosed by the model cloud scheme. This is for the whole atmosphere column as seen from the surface or the top of the atmosphere.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "convective_cloud_area_fraction": { + "description": "Fraction of horizontal grid square occupied by convective cloud as diagnosed by the model convection scheme. This is for the whole atmosphere column as seen from the surface or the top of the atmosphere.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "tropopause_air_pressure": { + "description": "Air pressure at the tropopause.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "Pa" + }, + "tropopause_air_temperature": { + "description": "Air temperature at the tropopause.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + } + } +} \ No newline at end of file diff --git a/datasets/met-office/collection/met-office-uk-deterministic-height/description.md b/datasets/met-office/collection/met-office-uk-deterministic-height/description.md new file mode 100644 index 000000000..02041569a --- /dev/null +++ b/datasets/met-office/collection/met-office-uk-deterministic-height/description.md @@ -0,0 +1,63 @@ +This collection offers 4 parameters at 33 available height levels (5m to 6000m) from the Met Office UKV 2km deterministic forecast. This is a high-resolution gridded weather forecast for the UK, with a resolution of 0.018 degrees, projected on to a 2km horizontal grid. The data is available as NetCDF files. + +## Height Levels +Available height levels, in metres (m) above ground, are: +* 5.0 10.0 20.0 30.0 50.0 75.0 100.0 150.0 200.0 250.0 300.0 400.0 500.0 600.0 700.0 800.0 1000.0 1250.0 1500.0 1750.0 2000.0 2250.0 2500.0 2750.0 3000.0 3250.0 3500.0 3750.0 4000.0 4500.0 5000.0 5500.0 6000.0 + +## Coverage area +The forecast covers the UK and Ireland, with the following latitude and longitude coordinates for each corner of the included area: +* Southwest: 48.8643°N, 10.6734°W +* Northwest: 61.3322°N, 13.7254°W +* Northeast: 61.6102°N, 4.3408°E +* Southeast: 49.0594°N, 2.4654°E + +## Timesteps +The following time steps are available: +* every hour from 0 to 54 hours +* every 3 hours from 57 to 120 hours + +## Update frequency +There are three lengths of model run, each with its own update frequency: +* Nowcast: forecasts the next 12 hours and are at 0100, 0200, 0400, 0500, 0700, 0800, 1000, 1100, 1300, 1400, 1600, 1700, 1900, 2000, 2200 and 2300 UTC. +* Short: forecasts the next 54 hours and are at 0000, 0600, 0900, 1200, 1800 and 2100 UTC. +* Medium: forecasts the next 120 hours and are at 0300 and 1500 UTC. + +## Archive length and latency + +As of December 2025, the archive contains data from December 2023 onwards. Forecasts will continue to be available for at least two years from their data date. + +The data is typically available 3-6 hours after the model run time. + +## Technical specs +The data is available as NetCDF files. NetCDF (Network Common Data Form) is an interface for array-orientated data access and a library that supports the interface. It is composed of 3 components: +* variables store the data +* dimensions give relevant dimension information for the variables +* attributes provide auxiliary information about the variables or dataset itself + +NetCDF is used within the atmospheric and oceanic science communities and is network transparent, allowing for it to be accessed by computers that store integers, characters and floating-point numbers. + +Iris supports NetCDF files through reading, writing and handling. Iris implements a model based on the CF conventions, giving a format-agnostic interface for working with data. + +[Find further support on using Iris with NetCDF files.](https://scitools-iris.readthedocs.io/en/stable/) + +## Help us improve the data services we offer + +[Join the Met Office research panel](https://forms.office.com/Pages/ResponsePage.aspx?id=YYHxF9cgRkeH_VD-PjtmGdxioYGoFbFIkZuB_q8Fb3VUQkoxRVQzTFdUMzNMVzczWVM5VTc3QTY3MC4u) to help us understand how people interact with weather and climate data, uncover challenges and explore opportunities. + +## How to cite + +UKV 2km deterministic forecast was accessed on DATE from the Microsoft Planetary Computer (https://zenodo.org/records/7261897). + +## License + +British Crown copyright 2023-2025, the Met Office, is licensed under [CC BY-SA](https://creativecommons.org/licenses/by-sa/4.0/deed.en). + +## Providers +[Met Office](https://www.metoffice.gov.uk/) + +See all datasets managed by [Met Office.](https://planetarycomputer.microsoft.com/catalog?filter=met+office) + +## Contact +[servicedesk@metoffice.gov.uk](mailto:servicedesk@metoffice.gov.uk). Service desk is only available Mon – Fri, 09:00 until 17:00 UTC (-1 hour during BST). + +As a non-operational service we aim to respond to any service support enquiries within 3-5 business days. diff --git a/datasets/met-office/collection/met-office-uk-deterministic-height/template.json b/datasets/met-office/collection/met-office-uk-deterministic-height/template.json new file mode 100644 index 000000000..7db4829bf --- /dev/null +++ b/datasets/met-office/collection/met-office-uk-deterministic-height/template.json @@ -0,0 +1,322 @@ +{ + "id": "met-office-uk-deterministic-height", + "type": "Collection", + "links": [ + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by-sa/4.0/deed.en", + "type": "text/html", + "title": "Creative Commons Attribution-ShareAlike 4.0" + }, + { + "rel": "describedby", + "href": "https://www.metoffice.gov.uk/services/data/external-data-channels", + "title": "Met Office Dataset Documentation" + } + ], + "title": "Height levels collection Met Office UKV 2km deterministic forecast", + "assets": { + "thumbnail": { + "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/met-office-uk-deterministic-height.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Height levels collection Met Office UKV 2km deterministic forecast thumbnail" + } + }, + "extent": { + "spatial": { + "bbox": [ + [ + -13.7254, + 48.8643, + 4.3408, + 61.6102 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2023-12-15T00:00:00Z", + null + ] + ] + } + }, + "license": "proprietary", + "keywords": [ + "Met Office", + "Weather", + "Forecast", + "UK", + "Cloud", + "Temperature", + "Wind", + "Height" + ], + "providers": [ + { + "url": "https://www.metoffice.gov.uk/", + "name": "Met Office", + "roles": [ + "producer", + "licensor", + "processor" + ] + }, + { + "url": "https://planetarycomputer.microsoft.com", + "name": "Microsoft", + "roles": [ + "host", + "processor" + ] + } + ], + "stac_version": "1.0.0", + "msft:group_id": "met-office-uk-deterministic", + "msft:container": "deterministic", + "stac_extensions": [ + "https://stac-extensions.github.io/forecast/v0.2.0/schema.json", + "https://stac-extensions.github.io/datacube/v2.0.0/schema.json" + ], + "msft:storage_account": "ukmoeuwest", + "msft:short_description": "Met Office UK Deterministic Height Level", + "msft:region": "westeurope", + "item_assets": { + "cloud_amount_on_height_levels": { + "type": "application/netcdf", + "title": "Cloud amount on height levels", + "roles": [ + "data" + ], + "forecast:variable": "cloud_volume_fraction_in_atmosphere_layer" + }, + "temperature_on_height_levels": { + "type": "application/netcdf", + "title": "Temperature on height levels", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "wind_direction_on_height_levels": { + "type": "application/netcdf", + "title": "Wind direction on height levels", + "roles": [ + "data" + ], + "forecast:variable": "wind_from_direction" + }, + "wind_speed_on_height_levels": { + "type": "application/netcdf", + "title": "Wind speed on height levels", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed" + } + }, + "description": "{{ collection.description }}", + "cube:dimensions": { + "latitude": { + "type": "spatial", + "axis": "y", + "extent": [ + 50, + 62 + ] + }, + "longitude": { + "type": "spatial", + "axis": "x", + "extent": [ + 5, + 15 + ] + }, + "height": { + "type": "spatial", + "values": [ + 5, + 10, + 20, + 30, + 50, + 75, + 100, + 150, + 200, + 250, + 300, + 400, + 500, + 600, + 700, + 800, + 1000, + 1250, + 1500, + 1750, + 2000, + 2250, + 2500, + 2750, + 3000, + 3250, + 3500, + 3750, + 4000, + 4500, + 5000, + 5500, + 6000 + ] + }, + "forecast:reference_datetime": { + "type": "temporal", + "extent": [ + "2023-12-15T00:00:00Z", + null + ] + }, + "forecast:horizon": { + "type": "temporal", + "extent": [ + null, + null + ], + "values": [ + "PT0H", + "PT1H", + "PT2H", + "PT3H", + "PT4H", + "PT5H", + "PT6H", + "PT7H", + "PT8H", + "PT9H", + "PT10H", + "PT11H", + "PT12H", + "PT13H", + "PT14H", + "PT15H", + "PT16H", + "PT17H", + "PT18H", + "PT19H", + "PT20H", + "PT21H", + "PT22H", + "PT23H", + "PT24H", + "PT25H", + "PT26H", + "PT27H", + "PT28H", + "PT29H", + "PT30H", + "PT31H", + "PT32H", + "PT33H", + "PT34H", + "PT35H", + "PT36H", + "PT37H", + "PT38H", + "PT39H", + "PT40H", + "PT41H", + "PT42H", + "PT43H", + "PT44H", + "PT45H", + "PT46H", + "PT47H", + "PT48H", + "PT49H", + "PT50H", + "PT51H", + "PT52H", + "PT53H", + "PT54H", + "PT57H", + "PT60H", + "PT63H", + "PT66H", + "PT69H", + "PT72H", + "PT75H", + "PT78H", + "PT81H", + "PT84H", + "PT87H", + "PT90H", + "PT93H", + "PT96H", + "PT99H", + "PT102H", + "PT105H", + "PT108H", + "PT111H", + "PT114H", + "PT117H", + "PT120H" + ] + } + }, + "cube:variables": { + "cloud_volume_fraction_in_atmosphere_layer": { + "description": "Fraction of horizontal grid square occupied by cloud in layers centred on height levels.", + "dimensions": [ + "latitude", + "longitude", + "height", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "air_temperature": { + "description": "Air temperature on height levels.", + "dimensions": [ + "latitude", + "longitude", + "height", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "wind_from_direction": { + "description": "Wind direction on height levels is defined as a two-dimensional (horizontal) air velocity vector with no vertical component. In meteorological reports the direction of the wind vector is given as the direction from which it is blowing.", + "dimensions": [ + "latitude", + "longitude", + "height", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "degrees" + }, + "wind_speed": { + "description": "Wind speed on height levels is defined as a two-dimensional (horizontal) air velocity vector with no vertical component. The speed is the magnitude of velocity.", + "dimensions": [ + "latitude", + "longitude", + "height", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + } + } +} \ No newline at end of file diff --git a/datasets/met-office/collection/met-office-uk-deterministic-near-surface/description.md b/datasets/met-office/collection/met-office-uk-deterministic-near-surface/description.md new file mode 100644 index 000000000..cea150898 --- /dev/null +++ b/datasets/met-office/collection/met-office-uk-deterministic-near-surface/description.md @@ -0,0 +1,80 @@ +This collection offers 35 parameters at near-surface level from the Met Office UKV 2km deterministic forecast. This is a high-resolution gridded weather forecast for the UK, with a resolution of 0.018 degrees, projected on to a 2km horizontal grid. + +The data is available as NetCDF files. It's offered on a free, unsupported basis, so we don't recommend using it for any critical business purposes. + +## Coverage area +The forecast covers the UK and Ireland, with the following latitude and longitude coordinates for each corner of the included area: +* Southwest: 48.8643°N, 10.6734°W +* Northwest: 61.3322°N, 13.7254°W +* Northeast: 61.6102°N, 4.3408°E +* Southeast: 49.0594°N, 2.4654°E + +## Data collection height +There are 3 forecast heights used within the near-surface this collection: +* Surface: the default collection height +* Screen level: 1.5m above the surface +* Wind parameters: 10m above the surface + +## Timesteps +For most parameters, the following time steps are available, see exceptions below: +* every hour from 0 to 54 hours +* every 3 hours from 57 to 120 hours + +Exceptions (for `rate`, `accumulation`, `min` and `max` parameters): +* Hourly hail fall accumulation (hail_fall_accumulation-PT01H) is only available every hour from 1 to 54 hours +* Hail fall rate (hail_fall_rate) has 15 minutely timesteps from 0 to 54 hours +* Hourly precipitation accumulation (precipitation_accumulation-PT01H) is only available every hour from 1 to 54 hours +* 3H precipitation accumulation (precipitation_accumulation-PT03H) is only available every three hours from 57 to 120 hours +* Precipitation rate (precipitation_rate) has 15 minutely timesteps from 0 to 54 hours +* Hourly rainfall accumulation (rainfall_accumulation-PT01H) is only available every hour from 1 to 54 hours +* 3H rainfall accumulation (rainfall_accumulation-PT03H) is only available every three hours from 57 to 120 hours +* Rainfall rate (rainfall_rate) has 15 minutely timesteps from 0 to 54 hours +* Hourly snowfall accumulation (snowfall_accumulation-PT01H) is only available every hour from 1 to 54 hours +* 3H snowfall accumulation (snowfall_accumulation-PT03H) is only available every three hours from 57 to 120 hours +* Snowfall rate (snowfall_rate) has 15 minutely timesteps from 0 to 54 hours +* Hourly temperature at screen level maximum (temperature_at_screen_level_max-PT01H) is only available every hour from 1 to 120 hours +* Hourly temperature at screen level minimum (temperature_at_screen_level_min-PT01H) is only available every hour from 1 to 120 hours +* Hourly wind gust at 10m maximum (wind_gust_at_10m_max-PT01H) is only available every hour from 1 to 54 hours +* 3H wind gust at 10m maximum(wind_gust_at_10m_max-PT03H) is only available every three hours from 57 to 120 hours + +## Update frequency +There are three lengths of model run, each with its own update frequency: +* Nowcast: forecasts the next 12 hours and are at 0100, 0200, 0400, 0500, 0700, 0800, 1000, 1100, 1300, 1400, 1600, 1700, 1900, 2000, 2200 and 2300 UTC. +* Short: forecasts the next 54 hours and are at 0000, 0600, 0900, 1200, 1800 and 2100 UTC. +* Medium: forecasts the next 120 hours and are at 0300 and 1500 UTC. + +## Archive length and latency +As of December 2025, the archive contains data from December 2023 onwards. Forecasts will continue to be available for at least two years from their data date. + +The data is typically available 3-6 hours after the model run time. + +## Technical specs +The data is available as NetCDF files. NetCDF (Network Common Data Form) is an interface for array-orientated data access and a library that supports the interface. It is composed of 3 components: +* variables store the data +* dimensions give relevant dimension information for the variables +* attributes provide auxiliary information about the variables or dataset itself + +NetCDF is used within the atmospheric and oceanic science communities and is network transparent, allowing for it to be accessed by computers that store integers, characters and floating-point numbers. + +Iris supports NetCDF files through reading, writing and handling. Iris implements a model based on the CF conventions, giving a format-agnostic interface for working with data. + +[Find further support on using Iris with NetCDF files.](https://scitools-iris.readthedocs.io/en/stable/) + +## Help us improve the data services we offer +[Join the Met Office research panel](https://forms.office.com/Pages/ResponsePage.aspx?id=YYHxF9cgRkeH_VD-PjtmGdxioYGoFbFIkZuB_q8Fb3VUQkoxRVQzTFdUMzNMVzczWVM5VTc3QTY3MC4u) to help us understand how people interact with weather and climate data, uncover challenges and explore opportunities. + +## How to cite +UKV 2km deterministic forecast was accessed on DATE from the Microsoft Planetary Computer (https://zenodo.org/records/7261897). + +## License +British Crown copyright 2023-2025, the Met Office, is licensed under [CC BY-SA](https://creativecommons.org/licenses/by-sa/4.0/deed.en). + +## Providers +[Met Office](https://www.metoffice.gov.uk/) + +See all datasets managed by [Met Office.](https://planetarycomputer.microsoft.com/catalog?filter=met+office) + +## Contact +[servicedesk@metoffice.gov.uk](mailto:servicedesk@metoffice.gov.uk). Service desk is only available Mon – Fri, 09:00 until 17:00 UTC (-1 hour during BST). + +As a non-operational service we aim to respond to any service support enquiries within 3-5 business days. diff --git a/datasets/met-office/collection/met-office-uk-deterministic-near-surface/template.json b/datasets/met-office/collection/met-office-uk-deterministic-near-surface/template.json new file mode 100644 index 000000000..844dafc0e --- /dev/null +++ b/datasets/met-office/collection/met-office-uk-deterministic-near-surface/template.json @@ -0,0 +1,793 @@ +{ + "id": "met-office-uk-deterministic-near-surface", + "type": "Collection", + "links": [ + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by-sa/4.0/deed.en", + "type": "text/html", + "title": "Creative Commons Attribution-ShareAlike 4.0" + }, + { + "rel": "describedby", + "href": "https://www.metoffice.gov.uk/services/data/external-data-channels", + "title": "Met Office Dataset Documentation" + } + ], + "title": "Near-surface level collection Met Office UKV 2km deterministic forecast", + "assets": { + "thumbnail": { + "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/met-office-uk-deterministic-near-surface.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Near-surface level collection Met Office UKV 2km deterministic forecast Thumbnail" + } + }, + "extent": { + "spatial": { + "bbox": [ + [ + -13.7254, + 48.8643, + 4.3408, + 61.6102 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2023-12-15T00:00:00Z", + null + ] + ] + } + }, + "license": "proprietary", + "keywords": [ + "Met Office", + "Weather", + "Forecast", + "UK", + "Precipitation", + "Temperature", + "Wind", + "Pressure", + "Humidity" + ], + "providers": [ + { + "url": "https://www.metoffice.gov.uk/", + "name": "Met Office", + "roles": [ + "producer", + "licensor", + "processor" + ] + }, + { + "url": "https://planetarycomputer.microsoft.com", + "name": "Microsoft", + "roles": [ + "host", + "processor" + ] + } + ], + "stac_version": "1.0.0", + "msft:group_id": "met-office-uk-deterministic", + "msft:container": "deterministic", + "stac_extensions": [ + "https://stac-extensions.github.io/forecast/v0.2.0/schema.json", + "https://stac-extensions.github.io/datacube/v2.0.0/schema.json" + ], + "msft:storage_account": "ukmoeuwest", + "msft:short_description": "Met Office UK Deterministic Near Surface Level", + "msft:region": "westeurope", + "item_assets": { + "fog_fraction_at_screen_level": { + "type": "application/netcdf", + "title": "Fog fraction at screen level", + "roles": [ + "data" + ], + "forecast:variable": "fog_area_fraction" + }, + "hail_fall_accumulation-PT01H": { + "type": "application/netcdf", + "title": "Hourly hail fall accumulation", + "roles": [ + "data" + ], + "forecast:variable": "lwe_thickness_of_graupel_and_hail_fall_amount" + }, + "hail_fall_rate": { + "type": "application/netcdf", + "title": "Hail fall rate", + "roles": [ + "data" + ], + "forecast:variable": "lwe_graupel_and_hail_fall_rate" + }, + "height_of_orography": { + "type": "application/netcdf", + "title": "Height of orography", + "roles": [ + "data" + ], + "forecast:variable": "surface_altitude" + }, + "landsea_mask": { + "type": "application/netcdf", + "title": "Landsea mask", + "roles": [ + "data" + ], + "forecast:variable": "land_binary_mask" + }, + "precipitation_accumulation-PT01H": { + "type": "application/netcdf", + "title": "Hourly precipitation accumulation", + "roles": [ + "data" + ], + "forecast:variable": "lwe_thickness_of_precipitation_amount" + }, + "precipitation_accumulation-PT03H": { + "type": "application/netcdf", + "title": "3H precipitation accumulation", + "roles": [ + "data" + ], + "forecast:variable": "lwe_thickness_of_precipitation_amount" + }, + "precipitation_rate": { + "type": "application/netcdf", + "title": "Precipitation rate", + "roles": [ + "data" + ], + "forecast:variable": "lwe_precipitation_rate" + }, + "pressure_at_mean_sea_level": { + "type": "application/netcdf", + "title": "Pressure at mean sea level", + "roles": [ + "data" + ], + "forecast:variable": "air_pressure_at_sea_level" + }, + "pressure_at_surface": { + "type": "application/netcdf", + "title": "Pressure at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_air_pressure" + }, + "radiation_flux_in_longwave_downward_at_surface": { + "type": "application/netcdf", + "title": "Radiation flux in longwave downward at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_downwelling_longwave_flux_in_air" + }, + "radiation_flux_in_shortwave_diffuse_downward_at_surface": { + "type": "application/netcdf", + "title": "Radiation flux in shortwave diffuse downward at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_diffusive_downwelling_shortwave_flux_in_air" + }, + "radiation_flux_in_shortwave_direct_downward_at_surface": { + "type": "application/netcdf", + "title": "Radiation flux in shortwave direct downward at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_direct_downwelling_shortwave_flux_in_air" + }, + "radiation_flux_in_shortwave_total_downward_at_surface": { + "type": "application/netcdf", + "title": "Radiation flux in shortwave total downward at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_downwelling_shortwave_flux_in_air" + }, + "radiation_flux_in_uv_downward_at_surface": { + "type": "application/netcdf", + "title": "Radiation flux in uv downward at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_downwelling_ultraviolet_flux_in_air" + }, + "rainfall_accumulation-PT01H": { + "type": "application/netcdf", + "title": "Hourly rainfall accumulation", + "roles": [ + "data" + ], + "forecast:variable": "thickness_of_rainfall_amount" + }, + "rainfall_accumulation-PT03H": { + "type": "application/netcdf", + "title": "3H rainfall accumulation", + "roles": [ + "data" + ], + "forecast:variable": "thickness_of_rainfall_amount" + }, + "rainfall_rate": { + "type": "application/netcdf", + "title": "Rainfall rate", + "roles": [ + "data" + ], + "forecast:variable": "rainfall_rate" + }, + "relative_humidity_at_screen_level": { + "type": "application/netcdf", + "title": "Relative humidity at screen level", + "roles": [ + "data" + ], + "forecast:variable": "relative_humidity" + }, + "sensible_heat_flux_at_surface": { + "type": "application/netcdf", + "title": "Sensible heat flux at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_upward_sensible_heat_flux" + }, + "snow_depth_water_equivalent": { + "type": "application/netcdf", + "title": "Snow depth water equivalent", + "roles": [ + "data" + ], + "forecast:variable": "lwe_thickness_of_surface_snow_amount" + }, + "snowfall_accumulation-PT01H": { + "type": "application/netcdf", + "title": "Hourly snowfall accumulation", + "roles": [ + "data" + ], + "forecast:variable": "lwe_thickness_of_snowfall_amount" + }, + "snowfall_accumulation-PT03H": { + "type": "application/netcdf", + "title": "3H snowfall accumulation", + "roles": [ + "data" + ], + "forecast:variable": "lwe_thickness_of_snowfall_amount" + }, + "snowfall_rate": { + "type": "application/netcdf", + "title": "Snowfall rate", + "roles": [ + "data" + ], + "forecast:variable": "lwe_snowfall_rate" + }, + "temperature_at_screen_level": { + "type": "application/netcdf", + "title": "Temperature at screen level", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "temperature_at_screen_level_max-PT01H": { + "type": "application/netcdf", + "title": "Hourly temperature at screen level maximum", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "temperature_at_screen_level_min-PT01H": { + "type": "application/netcdf", + "title": "Hourly temperature at screen level minimum", + "roles": [ + "data" + ], + "forecast:variable": "air_temperature" + }, + "temperature_at_surface": { + "type": "application/netcdf", + "title": "Temperature at surface", + "roles": [ + "data" + ], + "forecast:variable": "surface_temperature" + }, + "temperature_of_dew_point_at_screen_level": { + "type": "application/netcdf", + "title": "Dew point temperature at screen level", + "roles": [ + "data" + ], + "forecast:variable": "dew_point_temperature" + }, + "visibility_at_screen_level": { + "type": "application/netcdf", + "title": "Visibility at screen level", + "roles": [ + "data" + ], + "forecast:variable": "visibility_in_air" + }, + "wind_direction_at_10m": { + "type": "application/netcdf", + "title": "Wind direction at 10m", + "roles": [ + "data" + ], + "forecast:variable": "wind_from_direction" + }, + "wind_gust_at_10m": { + "type": "application/netcdf", + "title": "Wind gust at 10m", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed_of_gust" + }, + "wind_gust_at_10m_max-PT01H": { + "type": "application/netcdf", + "title": "Hourly wind gust at 10m maximum", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed_of_gust" + }, + "wind_gust_at_10m_max-PT03H": { + "type": "application/netcdf", + "title": "3H wind gust at 10m maximum", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed_of_gust" + }, + "wind_speed_at_10m": { + "type": "application/netcdf", + "title": "Wind speed at 10m", + "roles": [ + "data" + ], + "forecast:variable": "wind_speed" + } + }, + "description": "{{ collection.description }}", + "cube:dimensions": { + "latitude": { + "type": "spatial", + "axis": "y", + "extent": [ + 50, + 62 + ] + }, + "longitude": { + "type": "spatial", + "axis": "x", + "extent": [ + 5, + 15 + ] + }, + "forecast:reference_datetime": { + "type": "temporal", + "extent": [ + "2023-12-15T00:00:00Z", + null + ] + }, + "forecast:horizon": { + "type": "temporal", + "extent": [ + null, + null + ], + "values": [ + "PT0H", + "PT1H", + "PT2H", + "PT3H", + "PT4H", + "PT5H", + "PT6H", + "PT7H", + "PT8H", + "PT9H", + "PT10H", + "PT11H", + "PT12H", + "PT13H", + "PT14H", + "PT15H", + "PT16H", + "PT17H", + "PT18H", + "PT19H", + "PT20H", + "PT21H", + "PT22H", + "PT23H", + "PT24H", + "PT25H", + "PT26H", + "PT27H", + "PT28H", + "PT29H", + "PT30H", + "PT31H", + "PT32H", + "PT33H", + "PT34H", + "PT35H", + "PT36H", + "PT37H", + "PT38H", + "PT39H", + "PT40H", + "PT41H", + "PT42H", + "PT43H", + "PT44H", + "PT45H", + "PT46H", + "PT47H", + "PT48H", + "PT49H", + "PT50H", + "PT51H", + "PT52H", + "PT53H", + "PT54H", + "PT57H", + "PT60H", + "PT63H", + "PT66H", + "PT69H", + "PT72H", + "PT75H", + "PT78H", + "PT81H", + "PT84H", + "PT87H", + "PT90H", + "PT93H", + "PT96H", + "PT99H", + "PT102H", + "PT105H", + "PT108H", + "PT111H", + "PT114H", + "PT117H", + "PT120H" + ] + } + }, + "cube:variables": { + "fog_area_fraction": { + "description": "Here fog means a visibility of 1000 m or lower. The reduction in visibility is caused by water droplets or minute ice crystals forming close to the surface. This quantity represents the fraction of horizontal grid square occupied by fog. An alternative interpretation is that this represents the fractional probability of fog being present at any location in the grid square.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "lwe_thickness_of_graupel_and_hail_fall_amount": { + "description": "Implied depth of the layer of liquid water equivalent (LWE) hail produced by the model precipitation scheme and deposited on the surface in the previous hour.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "lwe_graupel_and_hail_fall_rate": { + "description": "Instantaneous rate at which liquid water equivalent (LWE) hail (as a depth) has been produced by the model precipitation scheme and deposited on the surface.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "surface_altitude": { + "description": "Altitude or (geometric) height above the geoid of the surface (ground). It is worth remembering that orographic height can be negative, particularly in some of the inland sea areas (e.g. Caspian Sea at -27m and Dead Sea at -430m).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "land_binary_mask": { + "description": "Binary indicator of whether a point is considered land (value = 1) or sea (value = 0). This is useful to identify the coastline and characterise points for correct interpretation of forecast data by users.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "lwe_thickness_of_precipitation_amount": { + "description": "Implied depth of the layer of liquid water which has been deposited on the surface in the previous 3 hours. This includes rain, snow and hail, with the ice phase precipitation being considered as a liquid water equivalent (LWE) value. It includes the contribution from the model convection scheme if this is invoked.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "lwe_precipitation_rate": { + "description": "Instantaneous rate at which liquid water (as a depth) was being deposited on the surface. This includes rain, snow and hail with the ice phase precipitation being considered as a liquid water equivalent (LWE) value. It includes the contribution from the model convection scheme if this is invoked.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "air_pressure_at_sea_level": { + "description": "Air pressure at mean sea level which is close to the geoid in sea areas. Air pressure at sea level is the quantity often abbreviated as MSLP or PMSL.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "Pa" + }, + "surface_air_pressure": { + "description": "Air pressure at the surface (lower boundary of the atmosphere).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "Pa" + }, + "surface_downwelling_longwave_flux_in_air": { + "description": "Longwave radiation at the surface from above directed at the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "surface_diffusive_downwelling_shortwave_flux_in_air": { + "description": "Diffuse component of shortwave radiation at the surface from above directed at the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "surface_direct_downwelling_shortwave_flux_in_air": { + "description": "Direct component of shortwave radiation at the surface from above directed at the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "surface_downwelling_shortwave_flux_in_air": { + "description": "Total shortwave radiation at the surface from above directed at the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "surface_downwelling_ultraviolet_flux_in_air": { + "description": "Ultraviolet radiation at the surface from above directed at the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "thickness_of_rainfall_amount": { + "description": "Implied depth of the rain produced by the model precipitation scheme which has been deposited on the surface in the previous 3 hours. This excludes the rain produced by the model convection scheme.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "rainfall_rate": { + "description": "Instantaneous rate at which rain (as a depth) was being produced by the model precipitation scheme and deposited on the surface. This excludes the rain produced by the model convection scheme.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "relative_humidity": { + "description": "Fractional relative humidity (ratio of the partial pressure of water vapour to the equilibrium vapour pressure of water) at screen level (1.5m above the surface.)", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "surface_upward_sensible_heat_flux": { + "description": "Exchange of heat between the surface and the air by motion of air, also called turbulent heat flux.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "W/m2" + }, + "lwe_thickness_of_surface_snow_amount": { + "description": "Liquid water equivalent (LWE) depth of the snow lying on the surface (ground). Typically water is 10 times as dense as snow, so multiplying by 10 gives an approximate depth of the snow, although wet snow can be significantly denser and powder snow much less dense.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "lwe_thickness_of_snowfall_amount": { + "description": "Implied depth of the (LWE) snow (as a depth) produced by the model precipitation scheme and deposited on the surface in the previous 3 hours. This excludes the snow produced by the model convection scheme.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "lwe_snowfall_rate": { + "description": "Instantaneous rate at which liquid water equivalent (LWE) snow (as a depth) was produced by the model precipitation scheme and deposited on the surface. This excludes the snow produced by the model convection scheme.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "air_temperature": { + "description": "Minimum instantaneous air temperature at screen level (1.5m) in the previous hour.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "surface_temperature": { + "description": "Temperature at the surface interface between the air and the ground.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "dew_point_temperature": { + "description": "Instantaneous dew point temperature (temperature at which a parcel of air reaches saturation upon being cooled at constant pressure and specific humidity) at screen level (1.5m).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "visibility_in_air": { + "description": "Horizontal distance at which something can be seen horizontally from screen level (1.5m).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "wind_from_direction": { + "description": "Wind at 10m above the surface is defined as a two-dimensional (horizontal) air velocity vector with no vertical component. In meteorological reports the direction of the wind vector is given as the direction from which it is blowing. Note: This with wind speed at 10m replaces x wind at 10m and y wind at 10m.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "degrees" + }, + "wind_speed_of_gust": { + "description": "Maximum diagnosed instantaneous wind gust at 10m in the previous 3 hours. This can be considered as the extreme wind speed that might be experienced in this period.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + }, + "wind_speed": { + "description": "Wind at 10m above the surface is defined as a two-dimensional (horizontal) air velocity vector with no vertical component. The speed is the magnitude of velocity.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + } + } +} \ No newline at end of file diff --git a/datasets/met-office/collection/met-office-uk-deterministic-pressure/description.md b/datasets/met-office/collection/met-office-uk-deterministic-pressure/description.md new file mode 100644 index 000000000..b360bdf33 --- /dev/null +++ b/datasets/met-office/collection/met-office-uk-deterministic-pressure/description.md @@ -0,0 +1,66 @@ +This collection offers 7 parameters at 33 available pressure levels (from 100000Pa to 1000Pa) from the Met Office UKV 2km deterministic forecast. This is a high-resolution gridded weather forecast for the UK, with a resolution of 0.018 degrees, projected on to a 2km horizontal grid. + +The data is available as NetCDF files. It's offered on a free, unsupported basis, so we don't recommend using it for any critical business purposes. + +## Pressure Levels +Available pressure levels, in Pascals (Pa), are: +* 100000.0 97500.0 95000.0 92500.0 90000.0 85000.0 80000.0 75000.0 70000.0 65000.0 60000.0 55000.0 50000.0 45000.0 40000.0 37500.0 35000.0 32500.0 30000.0 27500.0 25000.0 22500.0 20000.0 17500.0 15000.0 12500.0 10000.0 7000.0 5000.0 4000.0 3000.0 2000.0 1000.0 + +## Coverage area +The forecast covers the UK and Ireland, with the following latitude and longitude coordinates for each corner of the included area: +* Southwest: 48.8643°N, 10.6734°W +* Northwest: 61.3322°N, 13.7254°W +* Northeast: 61.6102°N, 4.3408°E +* Southeast: 49.0594°N, 2.4654°E + +## Timesteps +The following time steps are available: +* every hour from 0 to 54 hours (for most parameters, see parameter table for exceptions) +* every 3 hours from 57 to 120 hours + +## Update frequency +There are three lengths of model run, each with its own update frequency: +* Nowcast: forecasts the next 12 hours and are at 0100, 0200, 0400, 0500, 0700, 0800, 1000, 1100, 1300, 1400, 1600, 1700, 1900, 2000, 2200 and 2300 UTC. +* Short: forecasts the next 54 hours and are at 0000, 0600, 0900, 1200, 1800 and 2100 UTC. +* Medium: forecasts the next 120 hours and are at 0300 and 1500 UTC. + +## Archive length and latency + +As of December 2025, the archive contains data from December 2023 onwards. Forecasts will continue to be available for at least two years from their data date. + +The data is typically available 3-6 hours after the model run time. + + +## Technical specs +The data is available as NetCDF files. NetCDF (Network Common Data Form) is an interface for array-orientated data access and a library that supports the interface. It is composed of 3 components: +* variables store the data +* dimensions give relevant dimension information for the variables +* attributes provide auxiliary information about the variables or dataset itself + +NetCDF is used within the atmospheric and oceanic science communities and is network transparent, allowing for it to be accessed by computers that store integers, characters and floating-point numbers. + +Iris supports NetCDF files through reading, writing and handling. Iris implements a model based on the CF conventions, giving a format-agnostic interface for working with data. + +[Find further support on using Iris with NetCDF files.](https://scitools-iris.readthedocs.io/en/stable/) + +## Help us improve the data services we offer + +[Join the Met Office research panel](https://forms.office.com/Pages/ResponsePage.aspx?id=YYHxF9cgRkeH_VD-PjtmGdxioYGoFbFIkZuB_q8Fb3VUQkoxRVQzTFdUMzNMVzczWVM5VTc3QTY3MC4u) to help us understand how people interact with weather and climate data, uncover challenges and explore opportunities. + +## How to cite + +UKV 2km deterministic forecast was accessed on DATE from the Microsoft Planetary Computer (https://zenodo.org/records/7261897). + +## License + +British Crown copyright 2023-2025, the Met Office, is licensed under [CC BY-SA](https://creativecommons.org/licenses/by-sa/4.0/deed.en). + +## Providers +[Met Office](https://www.metoffice.gov.uk/) + +See all datasets managed by [Met Office.](https://planetarycomputer.microsoft.com/catalog?filter=met+office) + +## Contact +[servicedesk@metoffice.gov.uk](mailto:servicedesk@metoffice.gov.uk). Service desk is only available Mon – Fri, 09:00 until 17:00 UTC (-1 hour during BST). + +As a non-operational service we aim to respond to any service support enquiries within 3-5 business days. diff --git a/datasets/met-office/collection/met-office-uk-deterministic-pressure/template.json b/datasets/met-office/collection/met-office-uk-deterministic-pressure/template.json new file mode 100644 index 000000000..7de4a6c12 --- /dev/null +++ b/datasets/met-office/collection/met-office-uk-deterministic-pressure/template.json @@ -0,0 +1,215 @@ +{ + "id": "met-office-uk-deterministic-pressure", + "type": "Collection", + "description": "{{ collection.description }}", + "links": [ + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by-sa/4.0/deed.en", + "type": "text/html", + "title": "Creative Commons Attribution-ShareAlike 4.0" + }, + { + "rel": "describedby", + "href": "https://www.metoffice.gov.uk/services/data/external-data-channels", + "title": "Met Office Dataset Documentation" + } + ], + "title": "Pressure levels collection Met Office UKV 2km deterministic forecast", + "assets": { + "thumbnail": { + "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/met-office-uk-deterministic-pressure.jpg", + "type": "image/jpeg", + "roles": ["thumbnail"], + "title": "Pressure levels collection Met Office UKV 2km deterministic forecast Thumbnail" + } + }, + "extent": { + "spatial": { + "bbox": [[-13.7254, 48.8643, 4.3408, 61.6102]] + }, + "temporal": { + "interval": [["2023-12-15T00:00:00Z", null]] + } + }, + "license": "proprietary", + "keywords": [ + "Met Office", + "Weather", + "Forecast", + "UK", + "Temperature", + "Wind", + "Pressure", + "Humidity" + ], + "providers": [ + { + "url": "https://www.metoffice.gov.uk/", + "name": "Met Office", + "roles": ["producer", "licensor", "processor"] + }, + { + "url": "https://planetarycomputer.microsoft.com", + "name": "Microsoft", + "roles": ["host", "processor"] + } + ], + "stac_version": "1.0.0", + "msft:group_id": "met-office-uk-deterministic", + "msft:container": "deterministic", + "stac_extensions": [ + "https://stac-extensions.github.io/forecast/v0.2.0/schema.json", + "https://stac-extensions.github.io/datacube/v2.0.0/schema.json" + ], + "msft:storage_account": "ukmoeuwest", + "msft:short_description": "Met Office UK Deterministic Pressure Level", + "msft:region": "westeurope", + "item_assets": { + "geopotential_height_on_pressure_levels": { + "type": "application/netcdf", + "title": "Geopotential height on pressure levels", + "roles": ["data"], + "forecast:variable": "geopotential_height" + }, + "height_ASL_on_pressure_levels": { + "type": "application/netcdf", + "title": "Height Above Sea Level on pressure levels", + "roles": ["data"], + "forecast:variable": "geopotential_height" + }, + "relative_humidity_on_pressure_levels": { + "type": "application/netcdf", + "title": "Relative humidity on pressure levels", + "roles": ["data"], + "forecast:variable": "relative_humidity" + }, + "temperature_on_pressure_levels": { + "type": "application/netcdf", + "title": "Temperature on pressure levels", + "roles": ["data"], + "forecast:variable": "air_temperature" + }, + "wet_bulb_potential_temperature_on_pressure_levels": { + "type": "application/netcdf", + "title": "Wet bulb potential temperature on pressure levels", + "roles": ["data"], + "forecast:variable": "wet_bulb_potential_temperature" + }, + "wind_direction_on_pressure_levels": { + "type": "application/netcdf", + "title": "Wind direction on pressure levels", + "roles": ["data"], + "forecast:variable": "wind_from_direction" + }, + "wind_speed_on_pressure_levels": { + "type": "application/netcdf", + "title": "Wind speed on pressure levels", + "roles": ["data"], + "forecast:variable": "wind_speed" + } + }, + "cube:dimensions": { + "latitude": { + "type": "spatial", + "axis": "y", + "extent": [-90, 90] + }, + "longitude": { + "type": "spatial", + "axis": "x", + "extent": [-180, 180] + }, + "pressure_levels": { + "type": "spatial", + "values": [ + 100000, 97500, 95000, 92500, 90000, 85000, 80000, 75000, 70000, 6500, + 60000, 55000, 50000, 45000, 40000, 37500, 35000, 32500, 30000, 27500, + 25000, 22500, 20000, 17500, 15000, 12500, 10000, 7000, 5000, 4000, 3000, + 2000, 1000 + ] + }, + "forecast:reference_datetime": { + "type": "temporal", + "extent": ["2023-12-15T00:00:00Z", null] + }, + "forecast:horizon": { + "type": "temporal", + "extent": [null, null] + } + }, + "cube:variables": { + "geopotential_height": { + "description": "Note: from January 2026, this parameter will replace height_ASL_on_pressure_levels.", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "unit": "m", + "type": "data" + }, + "relative_humidity": { + "description": "Ratio of the partial pressure of water vapour to the equilibrium vapour pressure of water.", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "unit": "1", + "type": "data" + }, + "air_temperature": { + "description": "Air temperature on pressure levels", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "wet_bulb_potential_temperature": { + "description": "Wet bulb potential temperature on pressure levels. Wet bulb potential temperature is defined as the temperature that a parcel of air would have if it were brought down to the standard pressure of 1000hPa following a saturated adiabatic lapse rate.", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "K" + }, + "wind_from_direction": { + "description": "Wind direction on pressure levels is defined as a two-dimensional (horizontal) air velocity vector with no vertical component. In meteorological reports the direction of the wind vector is given as the direction from which it is blowing.", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "degrees" + }, + "wind_speed": { + "description": "Wind speed on pressure levels is defined as a two-dimensional (horizontal) air velocity vector with no vertical component. The speed is the magnitude of velocity.", + "dimensions": [ + "latitude", + "longitude", + "pressure_levels", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m/s" + } + } +} diff --git a/datasets/met-office/collection/met-office-uk-deterministic-whole-atmosphere/description.md b/datasets/met-office/collection/met-office-uk-deterministic-whole-atmosphere/description.md new file mode 100644 index 000000000..30e54550b --- /dev/null +++ b/datasets/met-office/collection/met-office-uk-deterministic-whole-atmosphere/description.md @@ -0,0 +1,63 @@ +This collection offers 11 whole-atmosphere parameters from the Met Office UKV 2km deterministic forecast. This is a high-resolution gridded weather forecast for the UK, with a resolution of 0.018 degrees, projected on to a 2km horizontal grid. The data is available as NetCDF files. + +## Coverage area +The forecast covers the UK and Ireland, with the following latitude and longitude coordinates for each corner of the included area: +* Southwest: 48.8643°N, 10.6734°W +* Northwest: 61.3322°N, 13.7254°W +* Northeast: 61.6102°N, 4.3408°E +* Southeast: 49.0594°N, 2.4654°E + +## Timesteps +The following time steps are available: +* every hour from 0 to 54 hours +* every 3 hours from 57 to 120 hours + +Exceptions: +* Hourly lightning flash accumulation (lightning_flash_accumulation-PT01H) is only available every hour from 1 to 54 hours + +## Update frequency +There are three lengths of model run, each with its own update frequency: +There are three lengths of model run, each with its own update frequency: +* Nowcast: forecasts the next 12 hours and are at 0100, 0200, 0400, 0500, 0700, 0800, 1000, 1100, 1300, 1400, 1600, 1700, 1900, 2000, 2200 and 2300 UTC. +* Short: forecasts the next 54 hours and are at 0000, 0600, 0900, 1200, 1800 and 2100 UTC. +* Medium: forecasts the next 120 hours and are at 0300 and 1500 UTC. + +## Archive length and latency + +As of December 2025, the archive contains data from December 2023 onwards. Forecasts will continue to be available for at least two years from their data date. + +The data is typically available 3-6 hours after the model run time. + +## Technical specs +The data is available as NetCDF files. NetCDF (Network Common Data Form) is an interface for array-orientated data access and a library that supports the interface. It is composed of 3 components: +* variables store the data +* dimensions give relevant dimension information for the variables +* attributes provide auxiliary information about the variables or dataset itself + +NetCDF is used within the atmospheric and oceanic science communities and is network transparent, allowing for it to be accessed by computers that store integers, characters and floating-point numbers. + +Iris supports NetCDF files through reading, writing and handling. Iris implements a model based on the CF conventions, giving a format-agnostic interface for working with data. + +[Find further support on using Iris with NetCDF files.](https://scitools-iris.readthedocs.io/en/stable/) + +## Help us improve the data services we offer + +[Join the Met Office research panel](https://forms.office.com/Pages/ResponsePage.aspx?id=YYHxF9cgRkeH_VD-PjtmGdxioYGoFbFIkZuB_q8Fb3VUQkoxRVQzTFdUMzNMVzczWVM5VTc3QTY3MC4u) to help us understand how people interact with weather and climate data, uncover challenges and explore opportunities. + +## How to cite + +UKV 2km deterministic forecast was accessed on DATE from the Microsoft Planetary Computer (https://zenodo.org/records/7261897). + +## License + +British Crown copyright 2023-2025, the Met Office, is licensed under [CC BY-SA](https://creativecommons.org/licenses/by-sa/4.0/deed.en). + +## Providers +[Met Office](https://www.metoffice.gov.uk/) + +See all datasets managed by [Met Office.](https://planetarycomputer.microsoft.com/catalog?filter=met+office) + +## Contact +[servicedesk@metoffice.gov.uk](mailto:servicedesk@metoffice.gov.uk). Service desk is only available Mon – Fri, 09:00 until 17:00 UTC (-1 hour during BST). + +As a non-operational service we aim to respond to any service support enquiries within 3-5 business days. diff --git a/datasets/met-office/collection/met-office-uk-deterministic-whole-atmosphere/template.json b/datasets/met-office/collection/met-office-uk-deterministic-whole-atmosphere/template.json new file mode 100644 index 000000000..8fcd7e403 --- /dev/null +++ b/datasets/met-office/collection/met-office-uk-deterministic-whole-atmosphere/template.json @@ -0,0 +1,414 @@ +{ + "id": "met-office-uk-deterministic-whole-atmosphere", + "type": "Collection", + "links": [ + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by-sa/4.0/deed.en", + "type": "text/html", + "title": "Creative Commons Attribution-ShareAlike 4.0" + }, + { + "rel": "describedby", + "href": "https://www.metoffice.gov.uk/services/data/external-data-channels", + "title": "Met Office Dataset Documentation" + } + ], + "title": "Whole Atmosphere collection Met Office UKV 2km deterministic forecast", + "assets": { + "thumbnail": { + "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/met-office-uk-deterministic-whole-atmosphere.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Whole Atmosphere collection Met Office UKV 2km deterministic forecast thumbnail" + } + }, + "extent": { + "spatial": { + "bbox": [ + [ + -13.7254, + 48.8643, + 4.3408, + 61.6102 + ] + ] + }, + "temporal": { + "interval": [ + [ + "2023-12-15T00:00:00Z", + null + ] + ] + } + }, + "license": "proprietary", + "keywords": [ + "Met Office", + "Weather", + "Forecast", + "UK", + "CAPE", + "Cloud", + "Freezing", + "Wet Bulb", + "Lightning" + ], + "providers": [ + { + "url": "https://www.metoffice.gov.uk/", + "name": "Met Office", + "roles": [ + "producer", + "licensor", + "processor" + ] + }, + { + "url": "https://planetarycomputer.microsoft.com", + "name": "Microsoft", + "roles": [ + "host", + "processor" + ] + } + ], + "stac_version": "1.0.0", + "msft:group_id": "met-office-uk-deterministic", + "msft:container": "deterministic", + "stac_extensions": [ + "https://stac-extensions.github.io/forecast/v0.2.0/schema.json", + "https://stac-extensions.github.io/datacube/v2.0.0/schema.json" + ], + "msft:storage_account": "ukmoeuwest", + "msft:short_description": "Met Office UK Deterministic Whole Atmosphere Level", + "msft:region": "westeurope", + "item_assets": { + "CAPE_most_unstable_below_500hPa": { + "type": "application/netcdf", + "title": "CAPE most unstable below 500hPa", + "roles": [ + "data" + ], + "forecast:variable": "atmosphere_convective_available_potential_energy" + }, + "CAPE_surface": { + "type": "application/netcdf", + "title": "CAPE surface", + "roles": [ + "data" + ], + "forecast:variable": "atmosphere_convective_available_potential_energy_wrt_surface" + }, + "cloud_amount_below_1000ft_ASL": { + "type": "application/netcdf", + "title": "Cloud amount below 1000ft ASL", + "roles": [ + "data" + ], + "forecast:variable": "cloud_area_fraction_assuming_only_consider_surface_to_1000_feet_asl" + }, + "cloud_amount_of_high_cloud": { + "type": "application/netcdf", + "title": "Cloud amount of high cloud", + "roles": [ + "data" + ], + "forecast:variable": "high_type_cloud_area_fraction" + }, + "cloud_amount_of_low_cloud": { + "type": "application/netcdf", + "title": "Cloud amount of low cloud", + "roles": [ + "data" + ], + "forecast:variable": "low_type_cloud_area_fraction" + }, + "cloud_amount_of_medium_cloud": { + "type": "application/netcdf", + "title": "Cloud amount of medium cloud", + "roles": [ + "data" + ], + "forecast:variable": "medium_type_cloud_area_fraction" + }, + "cloud_amount_of_total_cloud": { + "type": "application/netcdf", + "title": "Cloud amount of total cloud", + "roles": [ + "data" + ], + "forecast:variable": "cloud_area_fraction" + }, + "height_AGL_at_cloud_base_where_cloud_cover_2p5_oktas": { + "type": "application/netcdf", + "title": "Height AGL at cloud base where cloud cover 2p5 oktas", + "roles": [ + "data" + ], + "forecast:variable": "cloud_base_height_2p5_oktas" + }, + "height_AGL_at_freezing_level": { + "type": "application/netcdf", + "title": "Height AGL at freezing level", + "roles": [ + "data" + ], + "forecast:variable": "freezing_level_height" + }, + "height_AGL_at_wet_bulb_freezing_level": { + "type": "application/netcdf", + "title": "Height AGL at wet bulb freezing level", + "roles": [ + "data" + ], + "forecast:variable": "wet_bulb_freezing_level_height" + }, + "lightning_flash_accumulation-PT01H": { + "type": "application/netcdf", + "title": "Hourly lightning flash accumulation", + "roles": [ + "data" + ], + "forecast:variable": "number_of_lightning_flashes_per_unit_area" + } + }, + "description": "{{ collection.description }}", + "cube:dimensions": { + "latitude": { + "type": "spatial", + "axis": "y", + "extent": [ + 50, + 62 + ] + }, + "longitude": { + "type": "spatial", + "axis": "x", + "extent": [ + 5, + 15 + ] + }, + "forecast:reference_datetime": { + "type": "temporal", + "extent": [ + "2023-12-15T00:00:00Z", + null + ] + }, + "forecast:horizon": { + "type": "temporal", + "extent": [ + null, + null + ], + "values": [ + "PT0H", + "PT1H", + "PT2H", + "PT3H", + "PT4H", + "PT5H", + "PT6H", + "PT7H", + "PT8H", + "PT9H", + "PT10H", + "PT11H", + "PT12H", + "PT13H", + "PT14H", + "PT15H", + "PT16H", + "PT17H", + "PT18H", + "PT19H", + "PT20H", + "PT21H", + "PT22H", + "PT23H", + "PT24H", + "PT25H", + "PT26H", + "PT27H", + "PT28H", + "PT29H", + "PT30H", + "PT31H", + "PT32H", + "PT33H", + "PT34H", + "PT35H", + "PT36H", + "PT37H", + "PT38H", + "PT39H", + "PT40H", + "PT41H", + "PT42H", + "PT43H", + "PT44H", + "PT45H", + "PT46H", + "PT47H", + "PT48H", + "PT49H", + "PT50H", + "PT51H", + "PT52H", + "PT53H", + "PT54H", + "PT57H", + "PT60H", + "PT63H", + "PT66H", + "PT69H", + "PT72H", + "PT75H", + "PT78H", + "PT81H", + "PT84H", + "PT87H", + "PT90H", + "PT93H", + "PT96H", + "PT99H", + "PT102H", + "PT105H", + "PT108H", + "PT111H", + "PT114H", + "PT117H", + "PT120H" + ] + } + }, + "cube:variables": { + "atmosphere_convective_available_potential_energy": { + "description": "CAPE (Convective Available Potential Energy) calculated for the most unstable parcel where the most unstable parcel is defined as the parcel with the highest fixed level CAPE launched from any level (including screen-level=1.5m) within 500hPa of the surface pressure.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "J/kg" + }, + "atmosphere_convective_available_potential_energy_wrt_surface": { + "description": "Value of CAPE (Convective Available Potential Energy) calculated for a surface based parcel where a surface based parcel is defined as a parcel initiated with thermodynamic properties at screen level height (1.5m) i.e. the parcel is launched from screen level.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "J/kg" + }, + "cloud_area_fraction_assuming_only_consider_surface_to_1000_feet_asl": { + "description": "Fraction of horizontal grid square occupied by cloud below 1000 feet above sea level.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "high_type_cloud_area_fraction": { + "description": "Fraction of horizontal grid square occupied by cloud in the high-level cloud height range (from the lowest model layer containing the 5574m height level up to but excluding the lowest model layer containing 13608m height level).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "low_type_cloud_area_fraction": { + "description": "Fraction of horizontal grid square occupied by cloud in the low-level cloud height range (from the lowest model layer containing the 111m height level up to but excluding the lowest model layer containing 1949m height level).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "medium_type_cloud_area_fraction": { + "description": "Fraction of horizontal grid square occupied by cloud in the mid-level cloud height range (from the lowest model layer containing the 1949m height level up to but excluding the lowest model layer containing 5574m height level).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "cloud_area_fraction": { + "description": "Fraction of horizontal grid square occupied by cloud as diagnosed by the model cloud scheme. This is for the whole atmosphere column as seen from the surface or the top of the atmosphere.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "1" + }, + "cloud_base_height_2p5_oktas": { + "description": "Height of the base of the lowest cloud above the surface (ground) where there is at least 2.5 oktas (eighths) of cloud cover. This corresponds to scattered cloud.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "freezing_level_height": { + "description": "Height of the 0 degrees celsius isotherm (freezing level) above the surface (ground).", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "wet_bulb_freezing_level_height": { + "description": "Height of the wet bulb freezing level (i.e. where the wet bulb temperature is 0oC) above sea level. This is also referred to as the altitude of the wet bulb freezing level or (geometric) height above the geoid, which is the reference geopotential surface. Wet bulb temperature is defined as the temperature of a parcel of air cooled to saturation (100% relative humidity) by the evaporation of water into it with the latent heat supplied by the parcel.", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "m" + }, + "number_of_lightning_flashes_per_unit_area": { + "description": "This represents the total (cloud-to-cloud and cloud-to-ground) lightning flashes per square metre within the previous hour", + "dimensions": [ + "latitude", + "longitude", + "forecast:reference_datetime", + "forecast:horizon" + ], + "type": "data", + "unit": "count/m2" + } + } +} \ No newline at end of file diff --git a/datasets/met-office/dataset.yaml b/datasets/met-office/dataset.yaml new file mode 100644 index 000000000..c9d3fc859 --- /dev/null +++ b/datasets/met-office/dataset.yaml @@ -0,0 +1,143 @@ +id: met-office +image: ${{ args.registry }}/pctasks-task-base:2026.01.12 + +args: + - registry + +code: + src: ${{ local.path(./met_office.py) }} + requirements: ${{ local.path(./requirements.txt) }} + +environment: + APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.task-application-insights-connection-string }} + +collections: + - id: met-office-global-deterministic-height + template: ${{ local.path(./collection/met-office-global-deterministic-height) }} + class: met_office:MetOfficeGlobalHeightCollection + asset_storage: + # "global/whole-atmosphere/update/2025/10/23/1200Z/20251023T1200Z-PT0000H00M.updated" + - uri: blob://ukmoeuwest/deterministic/global/height/update + chunks: + splits: + - depth: 3 + options: + min_depth: 1 + max_depth: 1 + extensions: + - .updated + chunk_length: 20000 + chunk_storage: + uri: blob://ukmoeuwest/chunks + - id: met-office-global-deterministic-near-surface + template: ${{ local.path(./collection/met-office-global-deterministic-near-surface) }} + class: met_office:MetOfficeGlobalNearSurfaceCollection + asset_storage: + - uri: blob://ukmoeuwest/deterministic/global/near-surface/update + chunks: + splits: + - depth: 3 + options: + min_depth: 1 + max_depth: 1 + extensions: + - .updated + chunk_length: 20000 + chunk_storage: + uri: blob://ukmoeuwest/chunks + - id: met-office-global-deterministic-pressure + template: ${{ local.path(./collection/met-office-global-deterministic-pressure) }} + class: met_office:MetOfficeGlobalPressureCollection + asset_storage: + - uri: blob://ukmoeuwest/deterministic/global/pressure/update + chunks: + splits: + - depth: 3 + options: + min_depth: 1 + max_depth: 1 + extensions: + - .updated + chunk_length: 20000 + chunk_storage: + uri: blob://ukmoeuwest/chunks + - id: met-office-global-deterministic-whole-atmosphere + template: ${{ local.path(./collection/met-office-global-deterministic-whole-atmosphere) }} + class: met_office:MetOfficeGlobalWholeAtmosphereCollection + asset_storage: + - uri: blob://ukmoeuwest/deterministic/global/whole-atmosphere/update + chunks: + splits: + - depth: 3 + options: + min_depth: 1 + max_depth: 1 + extensions: + - .updated + chunk_length: 20000 + chunk_storage: + uri: blob://ukmoeuwest/chunks + - id: met-office-uk-deterministic-height + template: ${{ local.path(./collection/met-office-uk-deterministic-height) }} + class: met_office:MetOfficeUkHeightCollection + asset_storage: + - uri: blob://ukmoeuwest/deterministic/uk/height/update + chunks: + splits: + - depth: 3 + options: + min_depth: 1 + max_depth: 1 + extensions: + - .updated + chunk_length: 20000 + chunk_storage: + uri: blob://ukmoeuwest/chunks + - id: met-office-uk-deterministic-near-surface + template: ${{ local.path(./collection/met-office-uk-deterministic-near-surface) }} + class: met_office:MetOfficeUkNearSurfaceCollection + asset_storage: + - uri: blob://ukmoeuwest/deterministic/uk/near-surface/update + chunks: + splits: + - depth: 3 + options: + min_depth: 1 + max_depth: 1 + extensions: + - .updated + chunk_length: 20000 + chunk_storage: + uri: blob://ukmoeuwest/chunks + - id: met-office-uk-deterministic-pressure + template: ${{ local.path(./collection/met-office-uk-deterministic-pressure) }} + class: met_office:MetOfficeUkPressureCollection + asset_storage: + - uri: blob://ukmoeuwest/deterministic/uk/pressure/update + chunks: + splits: + - depth: 3 + options: + min_depth: 1 + max_depth: 1 + extensions: + - .updated + chunk_length: 20000 + chunk_storage: + uri: blob://ukmoeuwest/chunks + - id: met-office-uk-deterministic-whole-atmosphere + template: ${{ local.path(./collection/met-office-uk-deterministic-whole-atmosphere) }} + class: met_office:MetOfficeUkWholeAtmosphereCollection + asset_storage: + - uri: blob://ukmoeuwest/deterministic/uk/whole-atmosphere/update + chunks: + splits: + - depth: 3 + options: + min_depth: 1 + max_depth: 1 + extensions: + - .updated + chunk_length: 20000 + chunk_storage: + uri: blob://ukmoeuwest/chunks diff --git a/datasets/met-office/met_office.py b/datasets/met-office/met_office.py new file mode 100644 index 000000000..cfc992dd8 --- /dev/null +++ b/datasets/met-office/met_office.py @@ -0,0 +1,85 @@ +import logging +from typing import Union + +from pctasks.core.models.task import WaitTaskResult +from pctasks.core.storage import StorageFactory +from pctasks.dataset.collection import Collection +from pystac import Item +from stactools.met_office_deterministic import stac +from stactools.met_office_deterministic.constants import Model, Theme + +handler = logging.StreamHandler() +handler.setFormatter(logging.Formatter("[%(levelname)s]:%(asctime)s: %(message)s")) +handler.setLevel(logging.INFO) +logger = logging.getLogger(__name__) +logger.addHandler(handler) +logger.setLevel(logging.INFO) + + +class MetOfficeCollection(Collection): + model: Model + theme: Theme + + @classmethod + def create_item( + cls, asset_uri: str, storage_factory: StorageFactory + ) -> Union[list[Item], WaitTaskResult]: + logger.info(f"Found sentinel file: {asset_uri}") + parts = asset_uri.split("/") + parent = "".join(parts[7:10]) + "T" + parts[10] + storage_uri = "/".join(parts[0:6] + [parent]) + item_id = parts[-1].split(".")[0] + storage = storage_factory.get_storage(storage_uri) + logger.info(f"Listing {storage_uri} for item {item_id}") + hrefs = list( + storage.get_url(path) + for path in storage.list_files(name_starts_with=item_id, extensions=[".nc"]) + ) + logger.info(f"Found {len(hrefs)} hrefs") + # Should only create one + items = stac.create_items(hrefs, model=cls.model, theme=cls.theme) + if len(items) > 1: + logger.warning( + f"Expected to only create 1 item, but created {len(items)} instead. Ids: {', '.join(item.id for item in items)}" + ) + return items + + +class MetOfficeGlobalHeightCollection(MetOfficeCollection): + model = Model.global_ + theme = Theme.height + + +class MetOfficeGlobalNearSurfaceCollection(MetOfficeCollection): + model = Model.global_ + theme = Theme.near_surface + + +class MetOfficeGlobalPressureCollection(MetOfficeCollection): + model = Model.global_ + theme = Theme.pressure_level + + +class MetOfficeGlobalWholeAtmosphereCollection(MetOfficeCollection): + model = Model.global_ + theme = Theme.whole_atmosphere + + +class MetOfficeUkHeightCollection(MetOfficeCollection): + model = Model.uk + theme = Theme.height + + +class MetOfficeUkNearSurfaceCollection(MetOfficeCollection): + model = Model.uk + theme = Theme.near_surface + + +class MetOfficeUkPressureCollection(MetOfficeCollection): + model = Model.uk + theme = Theme.pressure_level + + +class MetOfficeUkWholeAtmosphereCollection(MetOfficeCollection): + model = Model.uk + theme = Theme.whole_atmosphere diff --git a/datasets/met-office/requirements.txt b/datasets/met-office/requirements.txt new file mode 100644 index 000000000..a58084fc5 --- /dev/null +++ b/datasets/met-office/requirements.txt @@ -0,0 +1 @@ +stactools-met-office-deterministic==0.4.0 diff --git a/datasets/met-office/scripts/ingest-collections.sh b/datasets/met-office/scripts/ingest-collections.sh new file mode 100755 index 000000000..8f11302a4 --- /dev/null +++ b/datasets/met-office/scripts/ingest-collections.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env sh + +set -e + +COLLECTION_TYPE="${1:-all}" + +case "$COLLECTION_TYPE" in + global) + COLLECTIONS=" + met-office-global-deterministic-height + met-office-global-deterministic-near-surface + met-office-global-deterministic-pressure + met-office-global-deterministic-whole-atmosphere + " + ;; + uk) + COLLECTIONS=" + met-office-uk-deterministic-height + met-office-uk-deterministic-near-surface + met-office-uk-deterministic-pressure + met-office-uk-deterministic-whole-atmosphere + " + ;; + all) + COLLECTIONS=" + met-office-global-deterministic-height + met-office-global-deterministic-near-surface + met-office-global-deterministic-pressure + met-office-global-deterministic-whole-atmosphere + met-office-uk-deterministic-height + met-office-uk-deterministic-near-surface + met-office-uk-deterministic-pressure + met-office-uk-deterministic-whole-atmosphere + " + ;; + *) + echo "Usage: $0 [global|uk|all]" + echo " global - Ingest only global collections" + echo " uk - Ingest only UK collections" + echo " all - Ingest all collections (default)" + exit 1 + ;; +esac + +for collection in $COLLECTIONS +do + pctasks dataset ingest-collection \ + -d dataset.yaml \ + -a registry pccomponents \ + -c "$collection" \ + --confirm --submit +done diff --git a/datasets/met-office/scripts/ingest-items.sh b/datasets/met-office/scripts/ingest-items.sh new file mode 100755 index 000000000..e6eca3804 --- /dev/null +++ b/datasets/met-office/scripts/ingest-items.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env sh + +set -e + +COLLECTION_TYPE="all" +SINCE="" + +usage() { + echo "Usage: $0 [--collection global|uk|all] [--since datetime]" + echo " --collection - Collection type: global, uk, or all (default: all)" + echo " --since - Optional datetime to filter items (e.g., 2024-01-01)" + exit 1 +} + +while [ $# -gt 0 ]; do + case "$1" in + --collection) + COLLECTION_TYPE="$2" + shift 2 + ;; + --since) + SINCE="$2" + shift 2 + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" + usage + ;; + esac +done + +case "$COLLECTION_TYPE" in + global) + COLLECTIONS=" + met-office-global-deterministic-height + met-office-global-deterministic-near-surface + met-office-global-deterministic-pressure + met-office-global-deterministic-whole-atmosphere + " + ;; + uk) + COLLECTIONS=" + met-office-uk-deterministic-height + met-office-uk-deterministic-near-surface + met-office-uk-deterministic-pressure + met-office-uk-deterministic-whole-atmosphere + " + ;; + all) + COLLECTIONS=" + met-office-global-deterministic-height + met-office-global-deterministic-near-surface + met-office-global-deterministic-pressure + met-office-global-deterministic-whole-atmosphere + met-office-uk-deterministic-height + met-office-uk-deterministic-near-surface + met-office-uk-deterministic-pressure + met-office-uk-deterministic-whole-atmosphere + " + ;; + *) + echo "Invalid collection type: $COLLECTION_TYPE" + usage + ;; +esac + +SINCE_ARG="" +SUBMIT_MODE="--upsert" +if [ -n "$SINCE" ]; then + SINCE_ARG="-a since $SINCE" + SUBMIT_MODE="--submit" +fi + +for collection in $COLLECTIONS +do + pctasks dataset process-items \ + -d dataset.yaml \ + -a registry pccomponents.azurecr.io \ + --is-update-workflow \ + --workflow-id met-office-$collection-process-items \ + -c "$collection" ingest \ + --confirm $SUBMIT_MODE $SINCE_ARG +done diff --git a/datasets/met-office/test_met_office.py b/datasets/met-office/test_met_office.py new file mode 100644 index 000000000..dc79a8d7a --- /dev/null +++ b/datasets/met-office/test_met_office.py @@ -0,0 +1,96 @@ +import pytest +from met_office import ( + MetOfficeCollection, + MetOfficeGlobalHeightCollection, + MetOfficeGlobalNearSurfaceCollection, + MetOfficeGlobalPressureCollection, + MetOfficeGlobalWholeAtmosphereCollection, + MetOfficeUkHeightCollection, + MetOfficeUkNearSurfaceCollection, + MetOfficeUkPressureCollection, + MetOfficeUkWholeAtmosphereCollection, +) +from pctasks.core.storage import StorageFactory + +test_storage_account = "ukmoeuwest" +test_container = "deterministic" + + +@pytest.mark.parametrize( + "asset_uri,collection_class", + [ + ( + f"blob://{test_storage_account}/{test_container}/global/height/update/2026/01/06/1200Z/20260113T1200Z-PT0168H00M.updated", + MetOfficeGlobalHeightCollection, + ), + ( + f"blob://{test_storage_account}/{test_container}/global/near-surface/update/2026/01/06/1200Z/20260113T1200Z-PT0168H00M.updated", + MetOfficeGlobalNearSurfaceCollection, + ), + ( + f"blob://{test_storage_account}/{test_container}/global/pressure/update/2026/01/06/1200Z/20260113T1200Z-PT0168H00M.updated", + MetOfficeGlobalPressureCollection, + ), + ( + f"blob://{test_storage_account}/{test_container}/global/whole-atmosphere/update/2026/01/06/1200Z/20260113T1200Z-PT0168H00M.updated", + MetOfficeGlobalWholeAtmosphereCollection, + ), + ( + f"blob://{test_storage_account}/{test_container}/uk/height/update/2024/06/26/0700Z/20240626T1900Z-PT0012H00M.updated", + MetOfficeUkHeightCollection, + ), + ( + f"blob://{test_storage_account}/{test_container}/uk/near-surface/update/2024/11/29/0600Z/20241201T1200Z-PT0054H00M.updated", + MetOfficeUkNearSurfaceCollection, + ), + ( + f"blob://{test_storage_account}/{test_container}/uk/pressure/update/2024/10/25/1700Z/20241026T0500Z-PT0012H00M.updated", + MetOfficeUkPressureCollection, + ), + ( + f"blob://{test_storage_account}/{test_container}/uk/whole-atmosphere/update/2024/12/13/1100Z/20241213T2300Z-PT0012H00M.updated", + MetOfficeUkWholeAtmosphereCollection, + ), + ( + f"blob://{test_storage_account}/{test_container}/global/near-surface/update/2025/12/30/1200Z/20251230T1200Z-PT0000H00M.updated", + MetOfficeGlobalNearSurfaceCollection, + ) + ], +) +def test_met_office_collection( + asset_uri: str, + collection_class: type[MetOfficeCollection], +) -> None: + class MockBlobClient: + def __init__(self, real_client): + self._real_client = real_client + + def delete_file(self, path): + # Mock delete to avoid modifying storage + pass + + def __getattr__(self, name): + # Delegate all other methods to the real client + return getattr(self._real_client, name) + + class MockStorageFactory: + def __init__(self, real_factory): + self._real_factory = real_factory + + def get_storage(self, uri): + # Use real storage for reading + return self._real_factory.get_storage(uri) + + def get_storage_for_file(self, uri): + # Wrap the real client with our mock to intercept delete + real_client, path = self._real_factory.get_storage_for_file(uri) + return MockBlobClient(real_client), path + + real_storage_factory = StorageFactory() + storage_factory = MockStorageFactory(real_storage_factory) + result = collection_class.create_item(asset_uri, storage_factory) # pyright: ignore[reportArgumentType] + + assert result is not None + assert isinstance(result, list) + assert len(result) == 1 + result[0].validate() diff --git a/datasets/modis/Dockerfile b/datasets/modis/Dockerfile index 76f9fe5dd..4954703df 100644 --- a/datasets/modis/Dockerfile +++ b/datasets/modis/Dockerfile @@ -44,27 +44,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY ./datasets/modis/requirements.txt /opt/src/datasets/modis/requirements.txt RUN python3 -m pip install -r /opt/src/datasets/modis/requirements.txt diff --git a/datasets/modis/dataset.yaml b/datasets/modis/dataset.yaml index f31a44bab..3b4066034 100644 --- a/datasets/modis/dataset.yaml +++ b/datasets/modis/dataset.yaml @@ -1,8 +1,8 @@ id: modis -image: ${{ args.registry }}/pctasks-modis:2025.12.15.0 +image: ${{ args.registry }}/pctasks-modis:2026.01.12 args: -- registry + - registry code: src: ${{ local.path(./modis.py) }} @@ -13,7 +13,7 @@ environment: collections: - id: modis-09A1-061 template: ${{ local.path(./collection/modis-09A1-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: # The blob storage pattern is # | modis-09A1-061 @@ -37,7 +37,7 @@ collections: - id: modis-09Q1-061 template: ${{ local.path(./collection/modis-09Q1-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD09Q1/ chunks: @@ -52,7 +52,7 @@ collections: - id: modis-10A1-061 template: ${{ local.path(./collection/modis-10A1-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD10A1/ chunks: @@ -67,7 +67,7 @@ collections: - id: modis-10A2-061 template: ${{ local.path(./collection/modis-10A2-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD10A2/ chunks: @@ -82,7 +82,7 @@ collections: - id: modis-11A1-061 template: ${{ local.path(./collection/modis-11A1-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD11A1/ chunks: @@ -97,7 +97,7 @@ collections: - id: modis-11A2-061 template: ${{ local.path(./collection/modis-11A2-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD11A2/ chunks: @@ -112,7 +112,7 @@ collections: - id: modis-13A1-061 template: ${{ local.path(./collection/modis-13A1-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD13A1/ chunks: @@ -127,7 +127,7 @@ collections: - id: modis-13Q1-061 template: ${{ local.path(./collection/modis-13Q1-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD13Q1/ chunks: @@ -142,7 +142,7 @@ collections: - id: modis-14A1-061 template: ${{ local.path(./collection/modis-14A1-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD14A1/ chunks: @@ -157,7 +157,7 @@ collections: - id: modis-14A2-061 template: ${{ local.path(./collection/modis-14A2-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD14A2/ chunks: @@ -172,7 +172,7 @@ collections: - id: modis-15A2H-061 template: ${{ local.path(./collection/modis-15A2H-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD15A2H/ chunks: @@ -191,7 +191,7 @@ collections: - id: modis-15A3H-061 template: ${{ local.path(./collection/modis-15A3H-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MCD15A3H/ chunks: @@ -202,7 +202,7 @@ collections: - id: modis-16A3GF-061 template: ${{ local.path(./collection/modis-16A3GF-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD16A3GF/ chunks: @@ -217,7 +217,7 @@ collections: - id: modis-17A2H-061 template: ${{ local.path(./collection/modis-17A2H-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD17A2H/ chunks: @@ -232,7 +232,7 @@ collections: - id: modis-17A2HGF-061 template: ${{ local.path(./collection/modis-17A2HGF-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD17A2HGF/ chunks: @@ -247,7 +247,7 @@ collections: - id: modis-17A3HGF-061 template: ${{ local.path(./collection/modis-17A3HGF-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD17A3HGF/ chunks: @@ -262,7 +262,7 @@ collections: - id: modis-21A2-061 template: ${{ local.path(./collection/modis-21A2-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MOD21A2/ chunks: @@ -277,7 +277,7 @@ collections: - id: modis-43A4-061 template: ${{ local.path(./collection/modis-43A4-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MCD43A4/ chunks: @@ -288,11 +288,11 @@ collections: - id: modis-64A1-061 template: ${{ local.path(./collection/modis-64A1-061/) }} - class: modis:MODISCollection + class: modis:MODISCollection asset_storage: - uri: blob://modiseuwest/modis-061/MCD64A1/ chunks: options: extensions: [.hdf] chunk_storage: - uri: blob://modiseuwest/modis-061-etl-data/pctasks-chunks/64A1/ \ No newline at end of file + uri: blob://modiseuwest/modis-061-etl-data/pctasks-chunks/64A1/ diff --git a/datasets/noaa-cdr/Dockerfile b/datasets/noaa-cdr/Dockerfile index c09a582b4..5d30e7a7b 100644 --- a/datasets/noaa-cdr/Dockerfile +++ b/datasets/noaa-cdr/Dockerfile @@ -22,9 +22,9 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt # Install Python 3.11 -RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ - && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \ - && rm -rf "Mambaforge-$(uname)-$(uname -m).sh" +RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" \ + && bash "Miniforge3-$(uname)-$(uname -m).sh" -b -p /opt/conda \ + && rm -rf "Miniforge3-$(uname)-$(uname -m).sh" ENV PATH /opt/conda/bin:$PATH ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY ./datasets/noaa-cdr/requirements.txt /opt/src/datasets/noaa-cdr/requirements.txt RUN python3 -m pip install -r /opt/src/datasets/noaa-cdr/requirements.txt diff --git a/datasets/noaa-cdr/dataset.yaml b/datasets/noaa-cdr/dataset.yaml index 1b3fcb3db..6f6308d47 100644 --- a/datasets/noaa-cdr/dataset.yaml +++ b/datasets/noaa-cdr/dataset.yaml @@ -1,5 +1,5 @@ id: noaa-cdr -image: ${{ args.registry }}/pctasks-noaa-cdr:2023.4.28.0 +image: ${{ args.registry }}/pctasks-noaa-cdr:2026.01.12 args: - registry code: diff --git a/datasets/noaa-mrms-qpe/Dockerfile b/datasets/noaa-mrms-qpe/Dockerfile index 54ea82a30..338e33ee9 100644 --- a/datasets/noaa-mrms-qpe/Dockerfile +++ b/datasets/noaa-mrms-qpe/Dockerfile @@ -1,5 +1,5 @@ ARG registry -FROM ${registry}/pctasks-task-base:latest +FROM ${registry}/pctasks-task-base:2026.01.12 COPY datasets/noaa-mrms-qpe /opt/src/datasets/noaa-mrms-qpe RUN python3 -m pip install -r /opt/src/datasets/noaa-mrms-qpe/requirements.txt diff --git a/datasets/sentinel-1-grd/Dockerfile b/datasets/sentinel-1-grd/Dockerfile index 1bb85d582..6f672fdfb 100644 --- a/datasets/sentinel-1-grd/Dockerfile +++ b/datasets/sentinel-1-grd/Dockerfile @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY ./datasets/sentinel-1-grd/requirements.txt /opt/src/datasets/sentinel-1-grd/requirements.txt diff --git a/datasets/sentinel-1-grd/dataset.yaml b/datasets/sentinel-1-grd/dataset.yaml index f33651816..06eff20ee 100644 --- a/datasets/sentinel-1-grd/dataset.yaml +++ b/datasets/sentinel-1-grd/dataset.yaml @@ -1,9 +1,9 @@ id: sentinel-1-grd -image: ${{ args.registry }}/pctasks-sentinel-1-grd:20250708.1 +image: ${{ args.registry }}/pctasks-sentinel-1-grd:2026.01.12 args: -- registry -- year-prefix + - registry + - year-prefix code: src: ${{ local.path(./s1grd.py) }} @@ -32,6 +32,6 @@ collections: - depth: 2 # daily options: ends_with: manifest.safe - chunk_length: 100 # daily number of files is ~600-800 + chunk_length: 100 # daily number of files is ~600-800 chunk_storage: uri: blob://sentinel1euwest/s1-grd-etl-data/pctasks-chunks/ diff --git a/datasets/sentinel-1-rtc/Dockerfile b/datasets/sentinel-1-rtc/Dockerfile index cae17dfb6..a4685f2ea 100644 --- a/datasets/sentinel-1-rtc/Dockerfile +++ b/datasets/sentinel-1-rtc/Dockerfile @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY ./datasets/sentinel-1-rtc/requirements.txt /opt/src/datasets/sentinel-1-rtc/requirements.txt RUN python3 -m pip install -r /opt/src/datasets/sentinel-1-rtc/requirements.txt diff --git a/datasets/sentinel-1-rtc/dataset.yaml b/datasets/sentinel-1-rtc/dataset.yaml index 2b60e3217..574d34096 100644 --- a/datasets/sentinel-1-rtc/dataset.yaml +++ b/datasets/sentinel-1-rtc/dataset.yaml @@ -1,9 +1,9 @@ id: sentinel-1-rtc -image: ${{ args.registry }}/pctasks-sentinel-1-rtc:20230613.1 +image: ${{ args.registry }}/pctasks-sentinel-1-rtc:2026.01.12 args: -- registry -- year-prefix + - registry + - year-prefix code: src: ${{ local.path(./s1rtc.py) }} @@ -29,9 +29,9 @@ collections: - uri: blob://sentinel1euwestrtc/sentinel1-grd-rtc-stac/GRD/${{ args.year-prefix }} chunks: splits: - - depth: 2 # daily + - depth: 2 # daily options: ends_with: _rtc.json - chunk_length: 2000 # daily number of files is ~600-800 + chunk_length: 2000 # daily number of files is ~600-800 chunk_storage: uri: blob://sentinel1euwestrtc/sentinel1-grd-rtc-etl-data/pctasks-chunks/ diff --git a/datasets/sentinel-2/Dockerfile b/datasets/sentinel-2/Dockerfile index 05eb2c193..47d9e8a93 100644 --- a/datasets/sentinel-2/Dockerfile +++ b/datasets/sentinel-2/Dockerfile @@ -21,13 +21,13 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 # See https://github.com/mapbox/rasterio/issues/1289 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt -# Install Python 3.8 -RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ - && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \ - && rm -rf "Mambaforge-$(uname)-$(uname -m).sh" +# Install Python 3.11 +RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" \ + && bash "Miniforge3-$(uname)-$(uname -m).sh" -b -p /opt/conda \ + && rm -rf "Miniforge3-$(uname)-$(uname -m).sh" -ENV PATH /opt/conda/bin:$PATH -ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH +ENV PATH=/opt/conda/bin:$PATH +ENV LD_LIBRARY_PATH=/opt/conda/lib/:$LD_LIBRARY_PATH RUN mamba install -y -c conda-forge python=3.11 gdal pip setuptools cython numpy @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY ./datasets/sentinel-2/requirements.txt /opt/src/datasets/sentinel-2/requirements.txt RUN python3 -m pip install -r /opt/src/datasets/sentinel-2/requirements.txt diff --git a/datasets/sentinel-3/Dockerfile b/datasets/sentinel-3/Dockerfile index 7bdffadfb..02e0d206f 100644 --- a/datasets/sentinel-3/Dockerfile +++ b/datasets/sentinel-3/Dockerfile @@ -22,14 +22,14 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt # Install Python 3.8 -RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ - && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \ - && rm -rf "Mambaforge-$(uname)-$(uname -m).sh" +RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" \ + && bash "Miniforge3-$(uname)-$(uname -m).sh" -b -p /opt/conda \ + && rm -rf "Miniforge3-$(uname)-$(uname -m).sh" ENV PATH /opt/conda/bin:$PATH ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH -RUN mamba install -y -c conda-forge python=3.8 gdal=3.3.3 pip setuptools cython numpy==1.21.5 +RUN mamba install -y -c conda-forge python=3.11 gdal pip setuptools cython numpy RUN python -m pip install --upgrade pip @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY ./datasets/sentinel-3/requirements.txt /opt/src/datasets/sentinel-3/requirements.txt RUN python3 -m pip install -r /opt/src/datasets/sentinel-3/requirements.txt diff --git a/datasets/sentinel-3/dataset.yaml b/datasets/sentinel-3/dataset.yaml index 7261428e1..7098ef77e 100644 --- a/datasets/sentinel-3/dataset.yaml +++ b/datasets/sentinel-3/dataset.yaml @@ -1,5 +1,5 @@ id: sentinel-3 -image: ${{ args.registry }}/pctasks-sentinel-3:20230630.1 +image: ${{ args.registry }}/pctasks-sentinel-3:2026.01.12 args: - registry @@ -30,10 +30,10 @@ collections: - uri: blob://sentinel3euwest/sentinel-3/OLCI/OL_2_LFR___/${{ args.year-prefix }}/ chunks: splits: - - depth: 1 # split by month, about 15 minutes for all xfdumanifest.xml files + - depth: 1 # split by month, about 15 minutes for all xfdumanifest.xml files options: ends_with: xfdumanifest.xml - chunk_length: 200 # takes about 15 minutes to create items for ~2 days of data + chunk_length: 200 # takes about 15 minutes to create items for ~2 days of data chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-olci-lfr-l2-netcdf @@ -189,4 +189,4 @@ collections: ends_with: xfdumanifest.xml chunk_length: 200 chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-slstr-wst-l2-netcdf \ No newline at end of file + uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-slstr-wst-l2-netcdf diff --git a/datasets/sentinel-5p/Dockerfile b/datasets/sentinel-5p/Dockerfile index 828da64ba..5d2bb7f7e 100644 --- a/datasets/sentinel-5p/Dockerfile +++ b/datasets/sentinel-5p/Dockerfile @@ -22,14 +22,14 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt # Install Python 3.8 -RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ - && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \ - && rm -rf "Mambaforge-$(uname)-$(uname -m).sh" +RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" \ + && bash "Miniforge3-$(uname)-$(uname -m).sh" -b -p /opt/conda \ + && rm -rf "Miniforge3-$(uname)-$(uname -m).sh" ENV PATH /opt/conda/bin:$PATH ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH -RUN mamba install -y -c conda-forge python=3.8 gdal=3.3.3 pip setuptools cython numpy==1.21.5 +RUN mamba install -y -c conda-forge python=3.11 gdal pip setuptools cython numpy RUN python -m pip install --upgrade pip @@ -43,27 +43,33 @@ RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/cli /opt/src/pctasks/cli RUN cd /opt/src/pctasks/cli && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/task /opt/src/pctasks/task RUN cd /opt/src/pctasks/task && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/client /opt/src/pctasks/client RUN cd /opt/src/pctasks/client && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest RUN cd /opt/src/pctasks/ingest && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset RUN cd /opt/src/pctasks/dataset && \ - pip install . + pip install -r requirements.txt && \ + pip install --no-deps . COPY ./datasets/sentinel-5p/requirements.txt /opt/src/datasets/sentinel-5p/requirements.txt RUN python3 -m pip install -r /opt/src/datasets/sentinel-5p/requirements.txt diff --git a/datasets/sentinel-5p/dataset.yaml b/datasets/sentinel-5p/dataset.yaml index dbdeb57eb..12cf5f333 100644 --- a/datasets/sentinel-5p/dataset.yaml +++ b/datasets/sentinel-5p/dataset.yaml @@ -1,5 +1,5 @@ id: sentinel_5p -image: ${{ args.registry }}/pctasks-sentinel-5p:20230630.3 +image: ${{ args.registry }}/pctasks-sentinel-5p:2026.01.12 args: - registry diff --git a/docker-compose.yml b/docker-compose.yml index 864c4cc72..7957126d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,8 @@ services: container_name: pctasks-azurite image: mcr.microsoft.com/azure-storage/azurite:3.35.0 hostname: azurite - command: "azurite --silent --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost + command: + "azurite --silent --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 -l /workspace" ports: - "10000:10000" # Blob @@ -153,7 +154,7 @@ services: "--port", "8512", "--reload", - "--proxy-headers" + "--proxy-headers", ] # Used to let kind cluster pull local docker images @@ -166,42 +167,6 @@ services: volumes: - ~/.azure:/root/.azure - functions: - container_name: pctasks-functions - image: pctasks-functions - build: - context: . - dockerfile: pctasks_funcs/Dockerfile - volumes: - - ./pctasks:/home/site/pctasks - - ./pctasks_funcs:/home/site/wwwroot - - ~/.azure:/home/.azure - ports: - - "7071:7071" # Functions - environment: - - AzureWebJobsStorage=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1; - - WEBSITE_HOSTNAME=functions:7071 - - # Must use IP address to avoid SSL errors - - PCTASKS_COSMOSDB__URL=${PCTASKS_COSMOSDB__URL:-https://172.16.238.246:8081/} - - PCTASKS_COSMOSDB__accountEndpoint=${PCTASKS_COSMOSDB__URL} - # Set trigger app setting separately to avoid issues with __ in env var names - - FUNC_COSMOSDB_CONN_STR=AccountEndpoint=${PCTASKS_COSMOSDB__URL:-https://172.16.238.246:8081/};AccountKey=${PCTASKS_COSMOSDB__KEY:-C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==}; - - FUNC_STORAGE_ACCOUNT_CONN_STR=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1; - - FUNC_STORAGE_QUEUE_ACCOUNT_URL=http://azurite:10001/devstoreaccount1 - - FUNC_STORAGE_ACCOUNT_NAME=devstoreaccount1 - - FUNC_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== - # Isolate Change Feed functions for unit tests. - - PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX=${PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX-} - # Used in the various function.json to dynmically set the Cosmos DB container to watch. - - FUNC_WORKFLOWS_COLLECTION_NAME=workflows${PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX-} - - FUNC_WORKFLOW_RUNS_COLLECTION_NAME=workflow-runs${PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX-} - - FUNC_STORAGE_EVENTS_COLLECTION_NAME=storage-events${PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX-} - - FUNC_ITEMS_COLLECTION_NAME=items${PCTASKS_COSMOSDB__TEST_CONTAINER_SUFFIX-} - - COSMOSDB_EMULATOR_HOST=172.16.238.246 - - PCTASKS_DISPATCH__TEST_COLLECTION__QUEUE_NAME=test-collection - - PCTASKS_DISPATCH__TEST_COLLECTION__PREFIX=http://azurite:10000/devstoreaccount1/ - networks: default: # Network created during scripts/setup diff --git a/pctasks/ingest/pctasks/ingest/models.py b/pctasks/ingest/pctasks/ingest/models.py index 51c10a2a7..0acf3cd1f 100644 --- a/pctasks/ingest/pctasks/ingest/models.py +++ b/pctasks/ingest/pctasks/ingest/models.py @@ -53,8 +53,13 @@ def _get_discriminator_tag(v: Any) -> str: return NDJSON_MESSAGE_TYPE elif isinstance(v, IngestCollectionsInput): return COLLECTIONS_MESSAGE_TYPE - else: - return "Any" + elif isinstance(v, dict): + type_value = v.get("type") + if type_value == NDJSON_MESSAGE_TYPE: + return NDJSON_MESSAGE_TYPE + elif type_value == COLLECTIONS_MESSAGE_TYPE: + return COLLECTIONS_MESSAGE_TYPE + return "Any" class IngestTaskInput(PCBaseModel): diff --git a/pctasks/ingest/tests/test_models.py b/pctasks/ingest/tests/test_models.py new file mode 100644 index 000000000..af44947e4 --- /dev/null +++ b/pctasks/ingest/tests/test_models.py @@ -0,0 +1,33 @@ +import pytest +import yaml + +from pctasks.ingest.models import ( + IngestNdjsonInput, + IngestTaskInput, +) + + +def test_ingest_task_input_model_validate_from_yaml() -> None: + """Test model_validate from YAML input like glm.yaml ingest-items task.""" + yaml_content = """ +content: + type: Ndjson + uris: + - ${{tasks.create-items.output.ndjson_uri}} +options: + insert_group_size: 5000 + insert_only: false +""" + data = yaml.safe_load(yaml_content) + input = IngestTaskInput.model_validate(data) + + assert isinstance(input.content, IngestNdjsonInput) + assert input.content.type == "Ndjson" + assert input.content.uris == ["${{tasks.create-items.output.ndjson_uri}}"] + assert input.options.insert_group_size == 5000 + assert input.options.insert_only is False + + +def test_ingest_ndjson_input_requires_uris_or_folder() -> None: + with pytest.raises(ValueError, match="Either ndjson_folder or uris must be"): + IngestNdjsonInput(type="Ndjson") diff --git a/pctasks/ingest/tests/test_settings.py b/pctasks/ingest/tests/test_settings.py index d37b025dc..c8fd3993b 100644 --- a/pctasks/ingest/tests/test_settings.py +++ b/pctasks/ingest/tests/test_settings.py @@ -2,7 +2,7 @@ from pctasks.ingest.settings import SECTION_NAME, IngestSettings -def test_image_keys(): +def test_image_keys() -> None: yaml = """ submit: account_name: pctrxetlrobrxetlsa diff --git a/pctasks/ingest_task/Dockerfile b/pctasks/ingest_task/Dockerfile index 8df85a7c3..df7d93a81 100644 --- a/pctasks/ingest_task/Dockerfile +++ b/pctasks/ingest_task/Dockerfile @@ -1,15 +1,13 @@ -FROM python:3.9-slim +FROM mcr.microsoft.com/azurelinux/base/python:3.12 -# Setup timezone info ENV TZ=UTC - -ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 +RUN \ + tdnf update -y && \ + tdnf install ca-certificates azure-cli -y \ + && tdnf clean all ENV PIP_NO_CACHE_DIR=1 RUN pip install "setuptools>=65.5.1" -RUN python -m pip install --upgrade pip - # # Copy and install packages # diff --git a/pctasks/ingest_task/pyproject.toml b/pctasks/ingest_task/pyproject.toml index e8f1e5cf2..3c275d6c9 100644 --- a/pctasks/ingest_task/pyproject.toml +++ b/pctasks/ingest_task/pyproject.toml @@ -24,12 +24,12 @@ classifiers = [ "Programming Language :: Python :: 3.8", ] dependencies = [ - "fire==0.4.0", + "fire>=0.7.0", "orjson>=3.5.2", "pctasks.ingest @ {root:parent:uri}/ingest", "pctasks.task @ {root:parent:uri}/task", - "plpygis==0.2.0", - "pypgstac[psycopg]>=0.8.5,<0.9", + "plpygis>=0.5.0", + "pypgstac[psycopg]==0.9.6", "pystac==1.10.1", ] diff --git a/pctasks/ingest_task/requirements.txt b/pctasks/ingest_task/requirements.txt index 8eec19065..34e8c2a76 100644 --- a/pctasks/ingest_task/requirements.txt +++ b/pctasks/ingest_task/requirements.txt @@ -72,7 +72,7 @@ cryptography==45.0.6 # msal # pyjwt # from https://pypi.org/simple -fire==0.4.0 +fire==0.7.1 # via # pctasks-ingest-task (./pctasks/ingest_task/pyproject.toml) # pypgstac @@ -91,6 +91,9 @@ google-auth==2.40.3 googleapis-common-protos==1.70.0 # via google-api-core # from https://pypi.org/simple +hydraters==0.1.3 + # via pypgstac + # from https://pypi.org/simple idna==3.10 # via # pctasks-core @@ -181,7 +184,7 @@ packaging==25.0 planetary-computer==1.0.0 # via pctasks-core # from https://pypi.org/simple -plpygis==0.2.0 +plpygis==0.6.1 # via # pctasks-ingest-task (./pctasks/ingest_task/pyproject.toml) # pypgstac @@ -243,7 +246,7 @@ pygments==2.19.2 pyjwt==2.10.1 # via msal # from https://pypi.org/simple -pypgstac==0.8.6 +pypgstac==0.9.6 # via pctasks-ingest-task (./pctasks/ingest_task/pyproject.toml) # from https://pypi.org/simple pystac==1.10.1 @@ -304,7 +307,6 @@ rsa==4.9.1 six==1.17.0 # via # azure-core - # fire # opencensus # python-dateutil # from https://pypi.org/simple diff --git a/pctasks_funcs/Dockerfile b/pctasks_funcs/Dockerfile deleted file mode 100644 index 853428660..000000000 --- a/pctasks_funcs/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM mcr.microsoft.com/azure-functions/python:4-python3.10 - -ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ - AzureFunctionsJobHost__Logging__Console__IsEnabled=true - -RUN apt update && apt install -y azure-cli - -# Copy pctasks packages -COPY pctasks /home/site/pctasks - -# Copy function app -COPY pctasks_funcs /home/site/wwwroot - -WORKDIR /home/site/wwwroot - -RUN pip install -r /home/site/wwwroot/requirements.txt - -CMD [ "/home/site/wwwroot/start.sh" ] diff --git a/scripts/publish-ingestion-images b/scripts/publish-ingestion-images new file mode 100755 index 000000000..641db2c1f --- /dev/null +++ b/scripts/publish-ingestion-images @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +TAG=2026.01.12 +REGISTRY=pccomponents.azurecr.io + +images=( + "pctasks-goes-glm:datasets/goes/goes-glm/Dockerfile" + "pctasks-goes-cmi:datasets/goes/goes-cmi/Dockerfile" + "pctasks-landsat:datasets/landsat/Dockerfile" + "pctasks-sentinel-2:datasets/sentinel-2/Dockerfile" + "pctasks-modis:datasets/modis/Dockerfile" + "pctasks-noaa-cdr:datasets/noaa-cdr/Dockerfile" + "pctasks-sentinel-1-grd:datasets/sentinel-1-grd/Dockerfile" + "pctasks-sentinel-1-rtc:datasets/sentinel-1-rtc/Dockerfile" + "pctasks-sentinel-3:datasets/sentinel-3/Dockerfile" + "pctasks-sentinel-5p:datasets/sentinel-5p/Dockerfile" + "pctasks-gbif:datasets/gbif/Dockerfile" + "pctasks-ecmwf-forecast:datasets/ecmwf-forecast/Dockerfile" +) + +for entry in "${images[@]}"; do + name="${entry%%:*}" + dockerfile="${entry#*:}" + image="$REGISTRY/$name:$TAG" + echo "Building $image..." + docker build -t "$image" -f "$dockerfile" . + docker push "$image" +done + +# Build image with build-arg +image="$REGISTRY/pctasks-noaa-mrms-qpe:$TAG" +echo "Building $image..." +docker build -t "$image" -f datasets/noaa-mrms-qpe/Dockerfile --build-arg registry=$REGISTRY . +docker push "$image" \ No newline at end of file