diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de8e7abb9..906efd859 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,11 +26,11 @@ repos: # name: isort (python) # args: [ "--profile", "black", "--filter-files" ] - repo: https://github.com/psf/black - rev: 24.4.0 + rev: 24.4.2 hooks: - id: black - - repo: https://github.com/pre-commit/mirrors-pylint - rev: 'v3.0.0a5' # Use the sha / tag you want to point at + - repo: https://github.com/PyCQA/pylint + rev: 'v3.2.3' # Use the sha / tag you want to point at hooks: - id: pylint - repo: https://github.com/PyCQA/flake8 diff --git a/apps/cloud/odc/apps/cloud/azure_to_tar.py b/apps/cloud/odc/apps/cloud/azure_to_tar.py index da1fc9bee..3702c441f 100644 --- a/apps/cloud/odc/apps/cloud/azure_to_tar.py +++ b/apps/cloud/odc/apps/cloud/azure_to_tar.py @@ -35,9 +35,10 @@ def cli( url_prefix = (account_url + "/" + container_name + "/")[len("https://") :] # jam it all in a tar - tar_opts = dict( - name=outfile, mode="w" + tar_mode(gzip=True, xz=True, is_pipe=False) - ) + tar_opts = { + "name": outfile, + "mode": "w" + tar_mode(gzip=True, xz=True, is_pipe=False), + } with tarfile.open(**tar_opts) as tar: for yaml in yamls: add_txt_file(tar=tar, content=yaml[0], fname=url_prefix + yaml[1]) diff --git a/apps/cloud/odc/apps/cloud/s3_find.py b/apps/cloud/odc/apps/cloud/s3_find.py index 673d2c4c1..b835def35 100755 --- a/apps/cloud/odc/apps/cloud/s3_find.py +++ b/apps/cloud/odc/apps/cloud/s3_find.py @@ -52,7 +52,9 @@ def cli(uri, skip_check, no_sign_request=None, request_payer=False): try: stream = s3_find_glob(uri, skip_check=skip_check, s3=s3, **opts) for i, o in enumerate(stream): - print(o.url, flush=(i % flush_freq == 0)) + print( + o.url, flush=(i % flush_freq == 0) + ) # pylint:disable=superfluous-parens except ValueError as ve: click.echo(str(ve), err=True) sys.exit(1) diff --git a/apps/cloud/odc/apps/cloud/s3_to_tar.py b/apps/cloud/odc/apps/cloud/s3_to_tar.py index 688ff885c..af18ea340 100755 --- a/apps/cloud/odc/apps/cloud/s3_to_tar.py +++ b/apps/cloud/odc/apps/cloud/s3_to_tar.py @@ -71,7 +71,7 @@ def dump_to_tar(data_stream, tar): fetcher = S3Fetcher(nconcurrent=nconnections, aws_unsigned=no_sign_request) is_pipe = outfile == "-" - tar_opts = dict(mode="w" + tar_mode(gzip=gzip, xz=xz, is_pipe=is_pipe)) + tar_opts = {"mode": "w" + tar_mode(gzip=gzip, xz=xz, is_pipe=is_pipe)} if is_pipe: if stdout.isatty(): click.echo("Will not write to a terminal", err=True) diff --git a/apps/cloud/odc/apps/cloud/thredds_to_tar.py b/apps/cloud/odc/apps/cloud/thredds_to_tar.py index ea4c4a83c..cb1f47da3 100644 --- a/apps/cloud/odc/apps/cloud/thredds_to_tar.py +++ b/apps/cloud/odc/apps/cloud/thredds_to_tar.py @@ -60,9 +60,10 @@ def cli(thredds_catalogue, skips, select, workers, outfile): yamls = download_yamls(urls, workers) # jam it all in a tar - tar_opts = dict( - name=outfile, mode="w" + tar_mode(gzip=True, xz=True, is_pipe=False) - ) + tar_opts = { + "name": outfile, + "mode": "w" + tar_mode(gzip=True, xz=True, is_pipe=False), + } with tarfile.open(**tar_opts) as tar: for yaml in yamls: add_txt_file(tar=tar, content=yaml[0], fname=yaml[1]) diff --git a/apps/dc_tools/odc/apps/dc_tools/_stac.py b/apps/dc_tools/odc/apps/dc_tools/_stac.py index d01e297bf..ad9375639 100644 --- a/apps/dc_tools/odc/apps/dc_tools/_stac.py +++ b/apps/dc_tools/odc/apps/dc_tools/_stac.py @@ -13,7 +13,7 @@ from eodatasets3.serialise import from_doc from eodatasets3.stac import to_stac_item from toolz import get_in -from urlpath import URL +from urllib.parse import urlparse from ._docs import odc_uuid @@ -29,6 +29,7 @@ "sentinel_s2_l2a_cogs", "sentinel-s2-l2a-cogs", "sentinel-2-l2a", + "s2_l2a_c1", ] # Mapping between EO3 field names and STAC properties object field names @@ -118,7 +119,10 @@ def _stac_product_lookup( dataset_id = properties.get("sentinel:product_id") or properties.get( "s2:granule_id", dataset_id ) - product_name = "s2_l2a" + if collection == "s2_l2a_c1": + product_name = "s2_l2a_c1" + else: + product_name = "s2_l2a" if region_code is None: # Let's try two options, and throw an exception if we still don't get it try: @@ -179,10 +183,27 @@ def _find_self_href(item: Document) -> str: return self_uri[0] +def _get_relative_path(asset_href, self_link): + if self_link is None: + return asset_href + self_parts = urlparse(self_link) + href_parts = urlparse(asset_href) + + if self_parts.netloc.split(".")[0] != href_parts.netloc.split(".")[0]: + # files are not stored with same hostname (e.g. different buckets) + # therefore use the absolute path + return asset_href + + try: + return str(Path(href_parts.path).relative_to(Path(self_parts.path).parent)) + except ValueError: + # if it's not relative, keep as an absolute link + return asset_href + + def _get_stac_bands( item: Document, default_grid: str, - relative: bool = False, proj_shape: Optional[str] = None, proj_transform: Optional[str] = None, ) -> Tuple[Document, Document, Document]: @@ -195,27 +216,6 @@ def _get_stac_bands( assets = item.get("assets", {}) - def _get_path(asset, force_relative=False): - path = URL(asset["href"]) - if relative: - try: - if self_link is None: - raise ValueError - path = path.relative_to(URL(self_link).parent) - # Value error is raised if the path is not relative to the parent - # or if the self link cannot be found. - except ValueError: - # If the path is not relative to the parent force_relative - # is still used for data assets, due to a historical assumption. - # TODO: Implement rewrite_assets (like in stac_to_dc) in all - # tools so that this is no longer necessary. - if force_relative: - path = path.name - else: - pass - - return str(path) - for asset_name, asset in assets.items(): image_types = ["jp2", "geotiff"] # If something's not in image_types, make it an accessory @@ -223,7 +223,9 @@ def _get_path(asset, force_relative=False): if not any( t in asset.get("type", []) for t in image_types ) or "thumbnail" in asset.get("roles", []): - accessories[asset_name] = {"path": _get_path(asset)} + accessories[asset_name] = { + "path": _get_relative_path(asset["href"], self_link) + } continue # If transform specified here in the asset it should override @@ -240,7 +242,7 @@ def _get_path(asset, force_relative=False): "transform": transform, } - path = _get_path(asset, force_relative=True) + path = _get_relative_path(asset["href"], self_link) band_index = asset.get("band", None) band_info = {"path": path} @@ -275,10 +277,6 @@ def round_coords(c1, c2): return None -def stac_transform_absolute(input_stac): - return stac_transform(input_stac, relative=False) - - def _convert_value_to_eo3_type(key: str, value): """ Convert return type as per EO3 specification. @@ -332,7 +330,7 @@ def _check_valid_uuid(uuid_string: str) -> bool: return False -def stac_transform(input_stac: Document, relative: bool = True) -> Document: +def stac_transform(input_stac: Document) -> Document: """Takes in a raw STAC 1.0 dictionary and returns an ODC dictionary""" # pylint: disable=too-many-locals @@ -371,7 +369,6 @@ def stac_transform(input_stac: Document, relative: bool = True) -> Document: bands, grids, accessories = _get_stac_bands( input_stac, default_grid, - relative=relative, proj_shape=proj_shape, proj_transform=proj_transform, ) diff --git a/apps/dc_tools/odc/apps/dc_tools/_version.py b/apps/dc_tools/odc/apps/dc_tools/_version.py index 699eb888d..927de2582 100644 --- a/apps/dc_tools/odc/apps/dc_tools/_version.py +++ b/apps/dc_tools/odc/apps/dc_tools/_version.py @@ -1 +1 @@ -__version__ = "0.2.16" +__version__ = "0.2.18" diff --git a/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py index 8474bc410..733515e81 100644 --- a/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py @@ -38,7 +38,7 @@ "{algo}/{year}/map/ESA_WorldCover_10m_{year}_{algo}_{ns}{ew}_Map.tif" ) -map_version = dict(algo="v100", year="2020") +map_version = {"algo": "v100", "year": "2020"} def _unpack_bbox(bounding_box: str) -> Tuple[int, int, int, int]: diff --git a/apps/dc_tools/odc/apps/dc_tools/index_from_tar.py b/apps/dc_tools/odc/apps/dc_tools/index_from_tar.py index 253c2f96f..5cc1c437d 100644 --- a/apps/dc_tools/odc/apps/dc_tools/index_from_tar.py +++ b/apps/dc_tools/odc/apps/dc_tools/index_from_tar.py @@ -126,13 +126,13 @@ def cli( if ignore_lineage: auto_add_lineage = False - ds_resolve_args = dict( - products=product_names, - exclude_products=exclude_product_names, - fail_on_missing_lineage=not auto_add_lineage, - verify_lineage=verify_lineage, - skip_lineage=ignore_lineage, - ) + ds_resolve_args = { + "products": product_names, + "exclude_products": exclude_product_names, + "fail_on_missing_lineage": not auto_add_lineage, + "verify_lineage": verify_lineage, + "skip_lineage": ignore_lineage, + } allowed_changes = {(): allow_any} diff --git a/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py index 2e4f0bd7a..cf3dc0fab 100755 --- a/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py @@ -11,7 +11,7 @@ from datacube import Datacube from datacube.index.hl import Doc2Dataset from odc.apps.dc_tools._docs import parse_doc_stream -from odc.apps.dc_tools._stac import stac_transform, stac_transform_absolute +from odc.apps.dc_tools._stac import stac_transform from odc.apps.dc_tools.utils import ( IndexingException, SkippedException, @@ -26,7 +26,6 @@ statsd_gauge_reporting, statsd_setting, transform_stac, - transform_stac_absolute, update_flag, update_if_exists_flag, verify_lineage, @@ -59,12 +58,15 @@ def dump_to_odc( uris_docs = parse_doc_stream( ((doc.url, doc.data) for doc in document_stream), on_error=doc_error, - transform=transform, ) found_docs = False for uri, metadata in uris_docs: found_docs = True + stac_doc = None + if transform: + stac_doc = metadata + metadata = stac_transform(metadata) try: index_update_dataset( metadata, @@ -76,6 +78,7 @@ def dump_to_odc( allow_unsafe=allow_unsafe, archive_less_mature=archive_less_mature, publish_action=publish_action, + stac_doc=stac_doc, ) ds_added += 1 except IndexingException: @@ -103,7 +106,6 @@ def dump_to_odc( @fail_on_missing_lineage @verify_lineage @transform_stac -@transform_stac_absolute @update_flag @update_if_exists_flag @allow_unsafe @@ -121,7 +123,6 @@ def cli( fail_on_missing_lineage, verify_lineage, stac, - absolute, update, update_if_exists, allow_unsafe, @@ -151,13 +152,6 @@ def cli( datefmt="%m/%d/%Y %I:%M:%S", ) - transform = None - if stac: - if absolute: - transform = stac_transform_absolute - else: - transform = stac_transform - opts = {} if request_payer: opts["RequestPayer"] = "requester" @@ -215,7 +209,7 @@ def cli( skip_lineage=skip_lineage, fail_on_missing_lineage=fail_on_missing_lineage, verify_lineage=verify_lineage, - transform=transform, + transform=stac, update=update, update_if_exists=update_if_exists, allow_unsafe=allow_unsafe, diff --git a/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py index 6ddb321e1..a6c945eec 100644 --- a/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py @@ -33,14 +33,13 @@ statsd_setting, statsd_gauge_reporting, transform_stac, - transform_stac_absolute, archive_less_mature, update_flag, update_if_exists_flag, verify_lineage, publish_action, ) -from ._stac import stac_transform, stac_transform_absolute, ds_to_stac +from ._stac import stac_transform, ds_to_stac # Added log handler logging.basicConfig(level=logging.WARNING, handlers=[logging.StreamHandler()]) @@ -72,7 +71,7 @@ def extract_action_from_message(message): return None -def handle_json_message(metadata, transform, odc_metadata_link): +def handle_json_message(metadata, odc_metadata_link): odc_yaml_uri = None uri = None @@ -100,9 +99,6 @@ def handle_json_message(metadata, transform, odc_metadata_link): # if no odc_metadata_link provided, it will look for metadata dict "href" value with "rel==self" uri = get_uri(metadata, "self") - if transform: - metadata = transform(metadata) - return metadata, uri @@ -250,13 +246,12 @@ def queue_to_odc( do_archiving(metadata, dc, publish_action) else: if not record_path: - if transform: - stac_doc = metadata # Extract metadata and URI from a STAC or similar # json structure for indexing - metadata, uri = handle_json_message( - metadata, transform, odc_metadata_link - ) + metadata, uri = handle_json_message(metadata, odc_metadata_link) + if transform: + stac_doc = metadata + metadata = stac_transform(metadata) else: # Extract metadata from an S3 bucket notification # or similar for indexing @@ -315,7 +310,6 @@ def queue_to_odc( @fail_on_missing_lineage @verify_lineage @transform_stac -@transform_stac_absolute @update_flag @update_if_exists_flag @allow_unsafe @@ -352,7 +346,6 @@ def cli( fail_on_missing_lineage, verify_lineage, stac, - absolute, update, update_if_exists, allow_unsafe, @@ -370,13 +363,6 @@ def cli( ): """Iterate through messages on an SQS queue and add them to datacube""" - transform = None - if stac: - if absolute: - transform = stac_transform_absolute - else: - transform = stac_transform - candidate_products = product.split() sqs = boto3.resource("sqs") @@ -391,7 +377,7 @@ def cli( skip_lineage=skip_lineage, fail_on_missing_lineage=fail_on_missing_lineage, verify_lineage=verify_lineage, - transform=transform, + transform=stac, limit=limit, update=update, no_sign_request=no_sign_request, diff --git a/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py index b16c2379a..1b228b219 100644 --- a/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py @@ -4,15 +4,13 @@ import concurrent import json import logging -import os import sys from typing import Any, Dict, Generator, Optional, Tuple import click -import pystac from datacube import Datacube from datacube.index.hl import Doc2Dataset -from odc.apps.dc_tools._stac import stac_transform, stac_transform_absolute +from odc.apps.dc_tools._stac import stac_transform from odc.apps.dc_tools.utils import ( SkippedException, allow_unsafe, @@ -63,57 +61,24 @@ def _parse_options(options: Optional[str]) -> Dict[str, Any]: return parsed_options -def _guess_location( - item: pystac.Item, rewrite: Optional[Tuple[str, str]] = None -) -> Tuple[str, bool]: - self_link = None - asset_link = None - relative = True - +def item_to_meta_uri( + item: Item, + rename_product: Optional[str] = None, +) -> Generator[Tuple[dict, str, bool], None, None]: for link in item.links: if link.rel == "self": - self_link = link.target + uri = link.target # Override self with canonical if link.rel == "canonical": - self_link = link.target + uri = link.target break - for _, asset in item.assets.items(): - if "geotiff" in asset.media_type: - asset_link = os.path.dirname(asset.href) - break - - if rewrite is not None: - for _, asset in item.assets.items(): - if "geotiff" in asset.media_type: - asset.href = asset.href.replace(rewrite[0], rewrite[1]) - - # If the metadata and the document are not on the same path, - # we need to use absolute links and not relative ones. - if (self_link and asset_link) and os.path.dirname(self_link) != os.path.dirname( - asset_link - ): - relative = False - - return self_link, relative - - -def item_to_meta_uri( - item: Item, - rewrite: Optional[Tuple[str, str]] = None, - rename_product: Optional[str] = None, -) -> Generator[Tuple[dict, str, bool], None, None]: - uri, relative = _guess_location(item, rewrite) metadata = item.to_dict() if rename_product is not None: metadata["properties"]["odc:product"] = rename_product stac = metadata - - if relative: - metadata = stac_transform(metadata) - else: - metadata = stac_transform_absolute(metadata) + metadata = stac_transform(metadata) return (metadata, uri, stac) @@ -124,12 +89,18 @@ def process_item( doc2ds: Doc2Dataset, update_if_exists: bool, allow_unsafe: bool, - rewrite: Optional[Tuple[str, str]] = None, rename_product: Optional[str] = None, archive_less_mature: int = None, publish_action: bool = False, + asset_name_mapping: Optional[str] = None, ): - meta, uri, stac = item_to_meta_uri(item, rewrite, rename_product) + if asset_name_mapping: + for mapping in asset_name_mapping.split(','): + name_from, name_to = mapping.split('/') + if item.assets[name_from] is not None: + item.assets[name_to] = item.assets[name_from] + del item.assets[name_from] + meta, uri, stac = item_to_meta_uri(item, rename_product) index_update_dataset( meta, uri, @@ -149,10 +120,10 @@ def stac_api_to_odc( config: dict, catalog_href: str, allow_unsafe: bool = True, - rewrite: Optional[Tuple[str, str]] = None, rename_product: Optional[str] = None, archive_less_mature: int = None, publish_action: Optional[str] = None, + asset_name_mapping: Optional[str] = None, ) -> Tuple[int, int, int]: doc2ds = Doc2Dataset(dc.index) client = Client.open(catalog_href) @@ -182,10 +153,10 @@ def stac_api_to_odc( doc2ds, update_if_exists=update_if_exists, allow_unsafe=allow_unsafe, - rewrite=rewrite, rename_product=rename_product, archive_less_mature=archive_less_mature, publish_action=publish_action, + asset_name_mapping=asset_name_mapping, ): item.id for item in search.get_all_items() } @@ -236,12 +207,11 @@ def stac_api_to_odc( help="Other search terms, as a # separated list, i.e., --options=cloud_cover=0,100#sky=green", ) @click.option( - "--rewrite-assets", + "--asset-name-mapping", type=str, default=None, help=( - "Rewrite asset hrefs, for example, to change from " - "HTTPS to S3 URIs, --rewrite-assets=https://example.com/,s3://" + "Rewrite asset names, e.g. from band numbers to common names: --asset-name-mapping=B02/blue,B03/green,B04/red" ), ) @rename_product @@ -257,17 +227,16 @@ def cli( bbox, datetime, options, - rewrite_assets, rename_product, statsd_setting, archive_less_mature, publish_action, + asset_name_mapping ): """ Iterate through STAC items from a STAC API and add them to datacube. """ config = _parse_options(options) - rewrite = None # Format the search terms if bbox: @@ -283,13 +252,6 @@ def cli( # number if max_items is not None. config["max_items"] = limit - if rewrite_assets is not None: - rewrite = list(rewrite_assets.split(",")) - if len(rewrite) != 2: - raise ValueError( - "Rewrite assets argument needs to be two strings split by ','" - ) - # Do the thing dc = Datacube() added, failed, skipped = stac_api_to_odc( @@ -298,10 +260,10 @@ def cli( config, catalog_href, allow_unsafe=allow_unsafe, - rewrite=rewrite, rename_product=rename_product, archive_less_mature=archive_less_mature, publish_action=publish_action, + asset_name_mapping=asset_name_mapping, ) print( diff --git a/apps/dc_tools/setup.cfg b/apps/dc_tools/setup.cfg index b9d13d21a..8ebf2c74f 100644 --- a/apps/dc_tools/setup.cfg +++ b/apps/dc_tools/setup.cfg @@ -20,6 +20,7 @@ python_requires = >= 3.9 tests_require = pytest deepdiff + docker install_requires = click @@ -41,6 +42,7 @@ install_requires = tests = pytest deepdiff + docker AZURE = odc-cloud[AZURE] THREDDS = odc-cloud[THREDDS] diff --git a/apps/dc_tools/tests/conftest.py b/apps/dc_tools/tests/conftest.py index 671010cab..6db93f3fd 100644 --- a/apps/dc_tools/tests/conftest.py +++ b/apps/dc_tools/tests/conftest.py @@ -26,6 +26,12 @@ LANDSAT_ODC_NRT: str = "ga_ls8c_ard_3-1-0_088080_2020-05-25_nrt.odc-metadata.yaml" SENTINEL_STAC_OLD: str = "S2A_28QCH_20200714_0_L2A_old.json" SENTINEL_ODC: str = "S2A_28QCH_20200714_0_L2A.odc-metadata.json" +SENTINEL_STAC_C1: str = "S2B_T37MGP_20240710T073012_L2A_C1.json" +SENTINEL_ODC_C1: str = "S2B_T37MGP_20240710T073012_L2A_C1-odc-metadata.yaml" +SENTINEL_STAC_C1_REL: str = "S2B_T37MGP_20240710T073012_L2A_C1_rel_links.json" +SENTINEL_ODC_C1_REL: str = ( + "S2B_T37MGP_20240710T073012_L2A_C1_rel_links-odc-metadata.yaml" +) USGS_LANDSAT_STAC: str = "LC08_L2SR_081119_20200101_20200823_02_T2.json" LIDAR_STAC: str = "lidar_dem.json" MATURITY_PRODUCT: str = "ga_ls5t_gm_product.yaml" @@ -166,6 +172,40 @@ def sentinel_odc(): return metadata +@pytest.fixture +def sentinel_c1_stac(): + """Return an example of new collection Sentinel 2 L2A C1 STAC metadata.""" + with TEST_DATA_FOLDER.joinpath(SENTINEL_STAC_C1).open("r", encoding="utf8") as f: + metadata = json.load(f) + return metadata + + +@pytest.fixture +def sentinel_c1_odc(): + metadata = yield from documents.load_documents( + TEST_DATA_FOLDER.joinpath(SENTINEL_ODC_C1) + ) + return metadata + + +@pytest.fixture +def sentinel_c1_rel_stac(): + """Return an example of new collection Sentinel 2 L2A C1 STAC metadata.""" + with TEST_DATA_FOLDER.joinpath(SENTINEL_STAC_C1_REL).open( + "r", encoding="utf8" + ) as f: + metadata = json.load(f) + return metadata + + +@pytest.fixture +def sentinel_c1_rel_odc(): + metadata = yield from documents.load_documents( + TEST_DATA_FOLDER.joinpath(SENTINEL_ODC_C1_REL) + ) + return metadata + + @pytest.fixture def nrt_dsid(): return "2e9f4623-c51c-5233-869a-bb690f8c2cac" diff --git a/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1-odc-metadata.yaml b/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1-odc-metadata.yaml new file mode 100644 index 000000000..1b6432499 --- /dev/null +++ b/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1-odc-metadata.yaml @@ -0,0 +1,165 @@ +--- +# Dataset +# url: https://explorer.dev.digitalearth.africa/dataset/83d02080-0394-5bb1-bd1d-cfc23e0db47d.odc-metadata.yaml +$schema: https://schemas.opendatacube.org/dataset +id: 83d02080-0394-5bb1-bd1d-cfc23e0db47d + +label: S2B_T37MGP_20240710T073012_L2A +product: + name: s2_l2a_c1 + +crs: epsg:32737 +geometry: + type: Polygon + coordinates: + [ + [ + [6.99961e+05, 9.399999e+06], + [6.99961e+05, 9.290201e+06], + [8.09759e+05, 9.290201e+06], + [8.09759e+05, 9.399999e+06], + [6.99961e+05, 9.399999e+06], + ], + ] +grids: + g20m: + shape: [5490, 5490] + transform: [2.e+01, 0.e+00, 6.9996e+05, 0.e+00, -2.e+01, 9.4e+06] + g60m: + shape: [1830, 1830] + transform: [6.e+01, 0.e+00, 6.9996e+05, 0.e+00, -6.e+01, 9.4e+06] + g320m: + shape: [343, 343] + transform: + [ + 3.2e+02, + 0.e+00, + 6.9996e+05, + 0.e+00, + -3.2e+02, + 9.4e+06, + 0.e+00, + 0.e+00, + 1.e+00, + ] + default: + shape: [10980, 10980] + transform: [1.e+01, 0.e+00, 6.9996e+05, 0.e+00, -1.e+01, 9.4e+06] + +properties: + created: "2024-07-10T13:38:13.736092+00:00" + datetime: "2024-07-10T07:41:33.657000Z" + earthsearch:payload_id: roda-sentinel-2-c1-l2a/workflow-sentinel-2-c1-l2a-to-stac/c76ef495ac22038115a5d0865aafac19 + eo:azimuth: 1.85518023779404e+02 + eo:cloud_cover: 8.3505428e+01 + eo:constellation: sentinel-2 + eo:instrument: MSI + eo:platform: sentinel-2b + eo:sun_azimuth: 3.99264277845747e+01 + eo:sun_elevation: 5.29617721681249e+01 + grid:code: MGRS-37MGP + mgrs:grid_square: GP + mgrs:latitude_band: M + mgrs:utm_zone: 37 + odc:file_format: GeoTIFF + odc:processing_datetime: "2024-07-10T13:38:13.736092+00:00" + odc:region_code: 37MGP + processing:software: + sentinel-2-c1-l2a-to-stac: v2024.02.01 + proj:centroid: + lat: -5.92002e+00 + lon: 4.130189e+01 + proj:epsg: 32737 + s2:cloud_shadow_percentage: 0.e+00 + s2:dark_features_percentage: 0.e+00 + s2:datastrip_id: S2B_OPER_MSI_L2A_DS_2BPS_20240710T103747_S20240710T073012_N05.10 + s2:datatake_id: GS2B_20240710T071619_038356_N05.10 + s2:datatake_type: INS-NOBS + s2:degraded_msi_data_percentage: 7.9e-03 + s2:generation_time: "2024-07-10T10:37:47.000000Z" + s2:high_proba_clouds_percentage: 3.4168363e+01 + s2:medium_proba_clouds_percentage: 2.7591509e+01 + s2:nodata_pixel_percentage: 0.e+00 + s2:not_vegetated_percentage: 2.3e-05 + s2:processing_baseline: "05.10" + s2:product_type: S2MSI2A + s2:product_uri: S2B_MSIL2A_20240710T071619_N0510_R006_T37MGP_20240710T103747.SAFE + s2:reflectance_conversion_factor: 9.67377332839659e-01 + s2:saturated_defective_pixel_percentage: 0.e+00 + s2:snow_ice_percentage: 0.e+00 + s2:thin_cirrus_percentage: 2.1745561e+01 + s2:tile_id: S2B_OPER_MSI_L2A_TL_2BPS_20240710T103747_A038356_T37MGP_N05.10 + s2:unclassified_percentage: 0.e+00 + s2:vegetation_percentage: 1.e-05 + s2:water_percentage: 1.6494533e+01 + storage:platform: AWS + storage:region: us-west-2 + storage:requester_pays: false + updated: "2024-07-10T13:38:13.736092+00:00" + view:incidence_angle: 2.79725898254359e+00 + +measurements: + aot: + grid: g20m + path: AOT.tif + nir: + path: B08.tif + red: + path: B04.tif + scl: + grid: g20m + path: SCL.tif + wvp: + grid: g20m + path: WVP.tif + blue: + path: B02.tif + snow: + grid: g20m + path: SNW_20m.tif + cloud: + grid: g20m + path: CLD_20m.tif + green: + path: B03.tif + nir08: + grid: g20m + path: B8A.tif + nir09: + grid: g60m + path: B09.tif + swir16: + grid: g20m + path: B11.tif + swir22: + grid: g20m + path: B12.tif + visual: + path: TCI.tif + coastal: + grid: g60m + path: B01.tif + preview: + grid: g320m + path: L2A_PVI.tif + rededge1: + grid: g20m + path: B05.tif + rededge2: + grid: g20m + path: B06.tif + rededge3: + grid: g20m + path: B07.tif + +accessories: + thumbnail: + path: https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/L2A_PVI.jpg + granule_metadata: + path: https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/metadata.xml + product_metadata: + path: https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/product_metadata.xml + tileinfo_metadata: + path: https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/tileInfo.json + +lineage: {} diff --git a/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1.json b/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1.json new file mode 100644 index 000000000..fd1dfcf7b --- /dev/null +++ b/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1.json @@ -0,0 +1,927 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "id": "S2B_T37MGP_20240710T073012_L2A", + "properties": { + "created": "2024-07-10T13:38:13.736092+00:00", + "platform": "sentinel-2b", + "constellation": "sentinel-2", + "instruments": [ + "msi" + ], + "eo:cloud_cover": 83.505428, + "proj:epsg": 32737, + "proj:centroid": { + "lat": -5.92002, + "lon": 41.30189 + }, + "mgrs:utm_zone": 37, + "mgrs:latitude_band": "M", + "mgrs:grid_square": "GP", + "grid:code": "MGRS-37MGP", + "view:azimuth": 185.518023779404, + "view:incidence_angle": 2.79725898254359, + "view:sun_azimuth": 39.9264277845747, + "view:sun_elevation": 52.9617721681249, + "s2:tile_id": "S2B_OPER_MSI_L2A_TL_2BPS_20240710T103747_A038356_T37MGP_N05.10", + "s2:degraded_msi_data_percentage": 0.0079, + "s2:nodata_pixel_percentage": 0, + "s2:saturated_defective_pixel_percentage": 0, + "s2:dark_features_percentage": 0, + "s2:cloud_shadow_percentage": 0, + "s2:vegetation_percentage": 0.00001, + "s2:not_vegetated_percentage": 0.000023, + "s2:water_percentage": 16.494533, + "s2:unclassified_percentage": 0, + "s2:medium_proba_clouds_percentage": 27.591509, + "s2:high_proba_clouds_percentage": 34.168363, + "s2:thin_cirrus_percentage": 21.745561, + "s2:snow_ice_percentage": 0, + "s2:product_type": "S2MSI2A", + "s2:processing_baseline": "05.10", + "s2:product_uri": "S2B_MSIL2A_20240710T071619_N0510_R006_T37MGP_20240710T103747.SAFE", + "s2:generation_time": "2024-07-10T10:37:47.000000Z", + "s2:datatake_id": "GS2B_20240710T071619_038356_N05.10", + "s2:datatake_type": "INS-NOBS", + "s2:datastrip_id": "S2B_OPER_MSI_L2A_DS_2BPS_20240710T103747_S20240710T073012_N05.10", + "s2:reflectance_conversion_factor": 0.967377332839659, + "datetime": "2024-07-10T07:41:33.657000Z", + "earthsearch:payload_id": "roda-sentinel-2-c1-l2a/workflow-sentinel-2-c1-l2a-to-stac/c76ef495ac22038115a5d0865aafac19", + "storage:platform": "AWS", + "storage:region": "us-west-2", + "storage:requester_pays": false, + "processing:software": { + "sentinel-2-c1-l2a-to-stac": "v2024.02.01" + }, + "updated": "2024-07-10T13:38:13.736092+00:00", + "odc:region_code": "37MGP" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 40.8047368017658, + -5.42554003528178 + ], + [ + 40.8079630068981, + -6.41831374593132 + ], + [ + 41.8000474756306, + -6.41385152861236 + ], + [ + 41.7950555793138, + -5.4217723655915 + ], + [ + 40.8047368017658, + -5.42554003528178 + ] + ] + ] + }, + "links": [ + { + "rel": "self", + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/S2B_T37MGP_20240710T073012_L2A.json", + "type": "application/json" + }, + { + "rel": "derived_from", + "href": "s3://e84-earth-search-sentinel-data/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/S2B_T37MGP_20240710T073012_L2A.json", + "title": "Source STAC Item" + }, + { + "rel": "collection", + "href": "https://explorer.digitalearth.africa/stac/collections/s2_l2a_c1", + "type": "application/json" + } + ], + "assets": { + "red": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B04.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red - 10m", + "eo:bands": [ + { + "name": "B04", + "common_name": "red", + "center_wavelength": 0.665, + "full_width_half_max": 0.038 + } + ], + "gsd": 10, + "proj:shape": [ + 10980, + 10980 + ], + "proj:transform": [ + 10, + 0, + 699960, + 0, + -10, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 10, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220e5dcfaeab33dc3ec24325538556024f618caf595ea427a3877db60949c69d565", + "file:size": 195515104, + "roles": [ + "data", + "reflectance" + ] + }, + "green": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B03.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Green - 10m", + "eo:bands": [ + { + "name": "B03", + "common_name": "green", + "center_wavelength": 0.56, + "full_width_half_max": 0.045 + } + ], + "gsd": 10, + "proj:shape": [ + 10980, + 10980 + ], + "proj:transform": [ + 10, + 0, + 699960, + 0, + -10, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 10, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "12209adc387c1ead64ecff05ad557f3654fabc9f6a1ebe7896ae9e31dd77e9fb307d", + "file:size": 195609876, + "roles": [ + "data", + "reflectance" + ] + }, + "blue": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B02.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Blue - 10m", + "eo:bands": [ + { + "name": "B02", + "common_name": "blue", + "center_wavelength": 0.49, + "full_width_half_max": 0.098 + } + ], + "gsd": 10, + "proj:shape": [ + 10980, + 10980 + ], + "proj:transform": [ + 10, + 0, + 699960, + 0, + -10, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 10, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "122095dbeb1082f87f2386e52c6b96f38abe0989c607873347d34e01bb18a315b1d5", + "file:size": 195577613, + "roles": [ + "data", + "reflectance" + ] + }, + "visual": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/TCI.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "True color image", + "eo:bands": [ + { + "name": "B04", + "common_name": "red", + "center_wavelength": 0.665, + "full_width_half_max": 0.038 + }, + { + "name": "B03", + "common_name": "green", + "center_wavelength": 0.56, + "full_width_half_max": 0.045 + }, + { + "name": "B02", + "common_name": "blue", + "center_wavelength": 0.49, + "full_width_half_max": 0.098 + } + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 10 + }, + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 10 + }, + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 10 + } + ], + "gsd": 10, + "proj:shape": [ + 10980, + 10980 + ], + "proj:transform": [ + 10, + 0, + 699960, + 0, + -10, + 9400000 + ], + "file:checksum": "1220e8dcb20c01b9e377fe8c65812af6038cbaa6b761a92e537b9c2afbaf08346777", + "file:size": 188029757, + "roles": [ + "visual" + ] + }, + "nir": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B08.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "NIR 1 - 10m", + "eo:bands": [ + { + "name": "B08", + "common_name": "nir", + "center_wavelength": 0.842, + "full_width_half_max": 0.145 + } + ], + "gsd": 10, + "proj:shape": [ + 10980, + 10980 + ], + "proj:transform": [ + 10, + 0, + 699960, + 0, + -10, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 10, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220553a1bc3a2566bea077eea9d47b6248669fdd06afb18156bad8911df9cd405c8", + "file:size": 195417506, + "roles": [ + "data", + "reflectance" + ] + }, + "swir22": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B12.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "SWIR 2.2μm - 20m", + "eo:bands": [ + { + "name": "B12", + "common_name": "swir22", + "center_wavelength": 2.19, + "full_width_half_max": 0.242 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "12204b30709712ba6f69363c2db46def1aac6c58f31ce4574d8505301df57e7d11d6", + "file:size": 52498228, + "roles": [ + "data", + "reflectance" + ] + }, + "rededge2": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B06.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red Edge 2 - 20m", + "eo:bands": [ + { + "name": "B06", + "common_name": "rededge", + "center_wavelength": 0.74, + "full_width_half_max": 0.018 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "12200c6ba4689a5b62374e6c8ac62e6b5643e0332f4123f035ad55909ad28a5470c4", + "file:size": 54586170, + "roles": [ + "data", + "reflectance" + ] + }, + "rededge3": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B07.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red Edge 3 - 20m", + "eo:bands": [ + { + "name": "B07", + "common_name": "rededge", + "center_wavelength": 0.783, + "full_width_half_max": 0.028 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "12205d5c44c1b64bd7dcf90ef8bc5ea28a8e450d64aac29c38ffbb48fa068433d5ea", + "file:size": 54487352, + "roles": [ + "data", + "reflectance" + ] + }, + "rededge1": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B05.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red Edge 1 - 20m", + "eo:bands": [ + { + "name": "B05", + "common_name": "rededge", + "center_wavelength": 0.704, + "full_width_half_max": 0.019 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220e6d10efe8a30616d2af2c59b63ff7464a54e9f065862f5c78e789119ba420bb9", + "file:size": 54815811, + "roles": [ + "data", + "reflectance" + ] + }, + "swir16": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B11.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "SWIR 1.6μm - 20m", + "eo:bands": [ + { + "name": "B11", + "common_name": "swir16", + "center_wavelength": 1.61, + "full_width_half_max": 0.143 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220cc32a6a3d9f6a7f93797650e88f2adc550c682891f44a830ae5d201013bcf94f", + "file:size": 52499952, + "roles": [ + "data", + "reflectance" + ] + }, + "wvp": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/WVP.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Water Vapour (WVP)", + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "unit": "cm", + "scale": 0.001, + "offset": 0 + } + ], + "file:checksum": "1220eed28c460450356c7c8f8ec0ef47cf12b728844a8bd5d5268a18b9c13f9e013f", + "file:size": 204756, + "roles": [ + "data" + ] + }, + "nir08": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B8A.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "NIR 2 - 20m", + "eo:bands": [ + { + "name": "B8A", + "common_name": "nir08", + "center_wavelength": 0.865, + "full_width_half_max": 0.033 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220e9f6907eab1513d40252005e84a4cf14c82a787ae4570c9a3d8efcdd777359f2", + "file:size": 54483375, + "roles": [ + "data", + "reflectance" + ] + }, + "scl": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/SCL.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Scene classification map (SCL)", + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 20 + } + ], + "file:checksum": "12208a86fee66da3e482bc97a63f44e12336124dcef5f4b1009e8b54d34348ca4513", + "file:size": 1728613, + "roles": [ + "data" + ] + }, + "aot": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/AOT.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Aerosol optical thickness (AOT)", + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.001, + "offset": 0 + } + ], + "file:checksum": "12200bfe8067764eaafbe505e8661bdac7752b497c1890e25601be55de76b1cd71ab", + "file:size": 299796, + "roles": [ + "data" + ] + }, + "coastal": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B01.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Coastal - 60m", + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal", + "center_wavelength": 0.443, + "full_width_half_max": 0.027 + } + ], + "gsd": 60, + "proj:shape": [ + 1830, + 1830 + ], + "proj:transform": [ + 60, + 0, + 699960, + 0, + -60, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 60, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220b0eccf115f188872d8b7ffb8a15efa993e7568e5168e418a5cca1b5e1c09556f", + "file:size": 6357846, + "roles": [ + "data", + "reflectance" + ] + }, + "nir09": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B09.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "NIR 3 - 60m", + "eo:bands": [ + { + "name": "B09", + "common_name": "nir09", + "center_wavelength": 0.945, + "full_width_half_max": 0.026 + } + ], + "gsd": 60, + "proj:shape": [ + 1830, + 1830 + ], + "proj:transform": [ + 60, + 0, + 699960, + 0, + -60, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 60, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "12207bc01c1645abe3fe12cceca1dea74f046389842c655de5f1e81c7151b1c547ef", + "file:size": 6581700, + "roles": [ + "data", + "reflectance" + ] + }, + "cloud": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/CLD_20m.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Cloud Probabilities", + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 20 + } + ], + "file:checksum": "1220372d986971f7639b2a45b971abae170fcce343b93319340adbe4cc3696ccb3ce", + "file:size": 6268207, + "roles": [ + "data", + "cloud" + ] + }, + "snow": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/SNW_20m.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Snow Probabilities", + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 20 + } + ], + "file:checksum": "1220449b9426ea2751ee5b7cb166311eea503ffca50943780674ee0a25f8f6310822", + "file:size": 53931, + "roles": [ + "data", + "snow-ice" + ] + }, + "preview": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/L2A_PVI.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "True color preview", + "eo:bands": [ + { + "name": "B04", + "common_name": "red", + "center_wavelength": 0.665, + "full_width_half_max": 0.038 + }, + { + "name": "B03", + "common_name": "green", + "center_wavelength": 0.56, + "full_width_half_max": 0.045 + }, + { + "name": "B02", + "common_name": "blue", + "center_wavelength": 0.49, + "full_width_half_max": 0.098 + } + ], + "file:checksum": "122081017743a1a9c7a5b635714d7678c586b60f54fceda60d7cbc169c0e708a838f", + "file:size": 182766, + "gsd": 320, + "proj:shape": [ + 343, + 343 + ], + "proj:transform": [ + 320, + 0, + 699960, + 0, + -320, + 9400000, + 0, + 0, + 1 + ], + "raster:bands": [ + [ + { + "nodata": null, + "data_type": "uint8", + "spatial_resolution": 320 + }, + { + "nodata": null, + "data_type": "uint8", + "spatial_resolution": 320 + }, + { + "nodata": null, + "data_type": "uint8", + "spatial_resolution": 320 + } + ] + ], + "roles": [ + "overview" + ] + }, + "granule_metadata": { + "href": "https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/metadata.xml", + "type": "application/xml", + "file:checksum": "1220daa594263ba606171ea0c914f92deeefc2a55c649520fc08e7fd7f34e4678370", + "file:size": 548196, + "roles": [ + "metadata" + ] + }, + "tileinfo_metadata": { + "href": "https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/tileInfo.json", + "type": "application/json", + "file:checksum": "122000b15e9029e429fe30e33194cb2703c57eb3ab6e4bc275acbbfc8310c4a0cb3a", + "file:size": 1491, + "roles": [ + "metadata" + ] + }, + "product_metadata": { + "href": "https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/product_metadata.xml", + "type": "application/xml", + "file:checksum": "1220a6c19a720f61a382e90ceabf2f4cd801b90066efb8e20c12faf7857e835aea14", + "file:size": 54777, + "roles": [ + "metadata" + ] + }, + "thumbnail": { + "href": "https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/L2A_PVI.jpg", + "type": "image/jpeg", + "title": "Thumbnail of preview image", + "file:checksum": "122004e7449551768295e4a30b2ddc88b93ea40bc58983cbf9bc8629555fbc292de5", + "file:size": 35902, + "roles": [ + "thumbnail" + ] + } + }, + "bbox": [ + 40.804737, + -6.418314, + 41.800047, + -5.421772 + ], + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json" + ], + "collection": "s2_l2a_c1" +} diff --git a/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1_rel_links-odc-metadata.yaml b/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1_rel_links-odc-metadata.yaml new file mode 100644 index 000000000..1fbab9c7b --- /dev/null +++ b/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1_rel_links-odc-metadata.yaml @@ -0,0 +1,165 @@ +--- +# Dataset +# url: https://explorer.dev.digitalearth.africa/dataset/83d02080-0394-5bb1-bd1d-cfc23e0db47d.odc-metadata.yaml +$schema: https://schemas.opendatacube.org/dataset +id: 83d02080-0394-5bb1-bd1d-cfc23e0db47d + +label: S2B_T37MGP_20240710T073012_L2A +product: + name: s2_l2a_c1 + +crs: epsg:32737 +geometry: + type: Polygon + coordinates: + [ + [ + [6.99961e+05, 9.399999e+06], + [6.99961e+05, 9.290201e+06], + [8.09759e+05, 9.290201e+06], + [8.09759e+05, 9.399999e+06], + [6.99961e+05, 9.399999e+06], + ], + ] +grids: + g20m: + shape: [5490, 5490] + transform: [2.e+01, 0.e+00, 6.9996e+05, 0.e+00, -2.e+01, 9.4e+06] + g60m: + shape: [1830, 1830] + transform: [6.e+01, 0.e+00, 6.9996e+05, 0.e+00, -6.e+01, 9.4e+06] + g320m: + shape: [343, 343] + transform: + [ + 3.2e+02, + 0.e+00, + 6.9996e+05, + 0.e+00, + -3.2e+02, + 9.4e+06, + 0.e+00, + 0.e+00, + 1.e+00, + ] + default: + shape: [10980, 10980] + transform: [1.e+01, 0.e+00, 6.9996e+05, 0.e+00, -1.e+01, 9.4e+06] + +properties: + created: "2024-07-10T13:38:13.736092+00:00" + datetime: "2024-07-10T07:41:33.657000Z" + earthsearch:payload_id: roda-sentinel-2-c1-l2a/workflow-sentinel-2-c1-l2a-to-stac/c76ef495ac22038115a5d0865aafac19 + eo:azimuth: 1.85518023779404e+02 + eo:cloud_cover: 8.3505428e+01 + eo:constellation: sentinel-2 + eo:instrument: MSI + eo:platform: sentinel-2b + eo:sun_azimuth: 3.99264277845747e+01 + eo:sun_elevation: 5.29617721681249e+01 + grid:code: MGRS-37MGP + mgrs:grid_square: GP + mgrs:latitude_band: M + mgrs:utm_zone: 37 + odc:file_format: GeoTIFF + odc:processing_datetime: "2024-07-10T13:38:13.736092+00:00" + odc:region_code: 37MGP + processing:software: + sentinel-2-c1-l2a-to-stac: v2024.02.01 + proj:centroid: + lat: -5.92002e+00 + lon: 4.130189e+01 + proj:epsg: 32737 + s2:cloud_shadow_percentage: 0.e+00 + s2:dark_features_percentage: 0.e+00 + s2:datastrip_id: S2B_OPER_MSI_L2A_DS_2BPS_20240710T103747_S20240710T073012_N05.10 + s2:datatake_id: GS2B_20240710T071619_038356_N05.10 + s2:datatake_type: INS-NOBS + s2:degraded_msi_data_percentage: 7.9e-03 + s2:generation_time: "2024-07-10T10:37:47.000000Z" + s2:high_proba_clouds_percentage: 3.4168363e+01 + s2:medium_proba_clouds_percentage: 2.7591509e+01 + s2:nodata_pixel_percentage: 0.e+00 + s2:not_vegetated_percentage: 2.3e-05 + s2:processing_baseline: "05.10" + s2:product_type: S2MSI2A + s2:product_uri: S2B_MSIL2A_20240710T071619_N0510_R006_T37MGP_20240710T103747.SAFE + s2:reflectance_conversion_factor: 9.67377332839659e-01 + s2:saturated_defective_pixel_percentage: 0.e+00 + s2:snow_ice_percentage: 0.e+00 + s2:thin_cirrus_percentage: 2.1745561e+01 + s2:tile_id: S2B_OPER_MSI_L2A_TL_2BPS_20240710T103747_A038356_T37MGP_N05.10 + s2:unclassified_percentage: 0.e+00 + s2:vegetation_percentage: 1.e-05 + s2:water_percentage: 1.6494533e+01 + storage:platform: AWS + storage:region: us-west-2 + storage:requester_pays: false + updated: "2024-07-10T13:38:13.736092+00:00" + view:incidence_angle: 2.79725898254359e+00 + +measurements: + aot: + grid: g20m + path: AOT.tif + nir: + path: B08.tif + red: + path: B04.tif + scl: + grid: g20m + path: SCL.tif + wvp: + grid: g20m + path: WVP.tif + blue: + path: B02.tif + snow: + grid: g20m + path: SNW_20m.tif + cloud: + grid: g20m + path: CLD_20m.tif + green: + path: B03.tif + nir08: + grid: g20m + path: B8A.tif + nir09: + grid: g60m + path: B09.tif + swir16: + grid: g20m + path: B11.tif + swir22: + grid: g20m + path: B12.tif + visual: + path: TCI.tif + coastal: + grid: g60m + path: B01.tif + preview: + grid: g320m + path: L2A_PVI.tif + rededge1: + grid: g20m + path: B05.tif + rededge2: + grid: g20m + path: B06.tif + rededge3: + grid: g20m + path: B07.tif + +accessories: + thumbnail: + path: L2A_PVI.jpg + granule_metadata: + path: metadata.xml + product_metadata: + path: product_metadata.xml + tileinfo_metadata: + path: tileInfo.json + +lineage: {} diff --git a/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1_rel_links.json b/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1_rel_links.json new file mode 100644 index 000000000..159dda475 --- /dev/null +++ b/apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1_rel_links.json @@ -0,0 +1,927 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "id": "S2B_T37MGP_20240710T073012_L2A", + "properties": { + "created": "2024-07-10T13:38:13.736092+00:00", + "platform": "sentinel-2b", + "constellation": "sentinel-2", + "instruments": [ + "msi" + ], + "eo:cloud_cover": 83.505428, + "proj:epsg": 32737, + "proj:centroid": { + "lat": -5.92002, + "lon": 41.30189 + }, + "mgrs:utm_zone": 37, + "mgrs:latitude_band": "M", + "mgrs:grid_square": "GP", + "grid:code": "MGRS-37MGP", + "view:azimuth": 185.518023779404, + "view:incidence_angle": 2.79725898254359, + "view:sun_azimuth": 39.9264277845747, + "view:sun_elevation": 52.9617721681249, + "s2:tile_id": "S2B_OPER_MSI_L2A_TL_2BPS_20240710T103747_A038356_T37MGP_N05.10", + "s2:degraded_msi_data_percentage": 0.0079, + "s2:nodata_pixel_percentage": 0, + "s2:saturated_defective_pixel_percentage": 0, + "s2:dark_features_percentage": 0, + "s2:cloud_shadow_percentage": 0, + "s2:vegetation_percentage": 0.00001, + "s2:not_vegetated_percentage": 0.000023, + "s2:water_percentage": 16.494533, + "s2:unclassified_percentage": 0, + "s2:medium_proba_clouds_percentage": 27.591509, + "s2:high_proba_clouds_percentage": 34.168363, + "s2:thin_cirrus_percentage": 21.745561, + "s2:snow_ice_percentage": 0, + "s2:product_type": "S2MSI2A", + "s2:processing_baseline": "05.10", + "s2:product_uri": "S2B_MSIL2A_20240710T071619_N0510_R006_T37MGP_20240710T103747.SAFE", + "s2:generation_time": "2024-07-10T10:37:47.000000Z", + "s2:datatake_id": "GS2B_20240710T071619_038356_N05.10", + "s2:datatake_type": "INS-NOBS", + "s2:datastrip_id": "S2B_OPER_MSI_L2A_DS_2BPS_20240710T103747_S20240710T073012_N05.10", + "s2:reflectance_conversion_factor": 0.967377332839659, + "datetime": "2024-07-10T07:41:33.657000Z", + "earthsearch:payload_id": "roda-sentinel-2-c1-l2a/workflow-sentinel-2-c1-l2a-to-stac/c76ef495ac22038115a5d0865aafac19", + "storage:platform": "AWS", + "storage:region": "us-west-2", + "storage:requester_pays": false, + "processing:software": { + "sentinel-2-c1-l2a-to-stac": "v2024.02.01" + }, + "updated": "2024-07-10T13:38:13.736092+00:00", + "odc:region_code": "37MGP" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 40.8047368017658, + -5.42554003528178 + ], + [ + 40.8079630068981, + -6.41831374593132 + ], + [ + 41.8000474756306, + -6.41385152861236 + ], + [ + 41.7950555793138, + -5.4217723655915 + ], + [ + 40.8047368017658, + -5.42554003528178 + ] + ] + ] + }, + "links": [ + { + "rel": "self", + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/S2B_T37MGP_20240710T073012_L2A.json", + "type": "application/json" + }, + { + "rel": "derived_from", + "href": "s3://e84-earth-search-sentinel-data/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/S2B_T37MGP_20240710T073012_L2A.json", + "title": "Source STAC Item" + }, + { + "rel": "collection", + "href": "https://explorer.digitalearth.africa/stac/collections/s2_l2a_c1", + "type": "application/json" + } + ], + "assets": { + "red": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B04.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red - 10m", + "eo:bands": [ + { + "name": "B04", + "common_name": "red", + "center_wavelength": 0.665, + "full_width_half_max": 0.038 + } + ], + "gsd": 10, + "proj:shape": [ + 10980, + 10980 + ], + "proj:transform": [ + 10, + 0, + 699960, + 0, + -10, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 10, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220e5dcfaeab33dc3ec24325538556024f618caf595ea427a3877db60949c69d565", + "file:size": 195515104, + "roles": [ + "data", + "reflectance" + ] + }, + "green": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B03.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Green - 10m", + "eo:bands": [ + { + "name": "B03", + "common_name": "green", + "center_wavelength": 0.56, + "full_width_half_max": 0.045 + } + ], + "gsd": 10, + "proj:shape": [ + 10980, + 10980 + ], + "proj:transform": [ + 10, + 0, + 699960, + 0, + -10, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 10, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "12209adc387c1ead64ecff05ad557f3654fabc9f6a1ebe7896ae9e31dd77e9fb307d", + "file:size": 195609876, + "roles": [ + "data", + "reflectance" + ] + }, + "blue": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B02.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Blue - 10m", + "eo:bands": [ + { + "name": "B02", + "common_name": "blue", + "center_wavelength": 0.49, + "full_width_half_max": 0.098 + } + ], + "gsd": 10, + "proj:shape": [ + 10980, + 10980 + ], + "proj:transform": [ + 10, + 0, + 699960, + 0, + -10, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 10, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "122095dbeb1082f87f2386e52c6b96f38abe0989c607873347d34e01bb18a315b1d5", + "file:size": 195577613, + "roles": [ + "data", + "reflectance" + ] + }, + "visual": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/TCI.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "True color image", + "eo:bands": [ + { + "name": "B04", + "common_name": "red", + "center_wavelength": 0.665, + "full_width_half_max": 0.038 + }, + { + "name": "B03", + "common_name": "green", + "center_wavelength": 0.56, + "full_width_half_max": 0.045 + }, + { + "name": "B02", + "common_name": "blue", + "center_wavelength": 0.49, + "full_width_half_max": 0.098 + } + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 10 + }, + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 10 + }, + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 10 + } + ], + "gsd": 10, + "proj:shape": [ + 10980, + 10980 + ], + "proj:transform": [ + 10, + 0, + 699960, + 0, + -10, + 9400000 + ], + "file:checksum": "1220e8dcb20c01b9e377fe8c65812af6038cbaa6b761a92e537b9c2afbaf08346777", + "file:size": 188029757, + "roles": [ + "visual" + ] + }, + "nir": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B08.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "NIR 1 - 10m", + "eo:bands": [ + { + "name": "B08", + "common_name": "nir", + "center_wavelength": 0.842, + "full_width_half_max": 0.145 + } + ], + "gsd": 10, + "proj:shape": [ + 10980, + 10980 + ], + "proj:transform": [ + 10, + 0, + 699960, + 0, + -10, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 10, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220553a1bc3a2566bea077eea9d47b6248669fdd06afb18156bad8911df9cd405c8", + "file:size": 195417506, + "roles": [ + "data", + "reflectance" + ] + }, + "swir22": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B12.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "SWIR 2.2μm - 20m", + "eo:bands": [ + { + "name": "B12", + "common_name": "swir22", + "center_wavelength": 2.19, + "full_width_half_max": 0.242 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "12204b30709712ba6f69363c2db46def1aac6c58f31ce4574d8505301df57e7d11d6", + "file:size": 52498228, + "roles": [ + "data", + "reflectance" + ] + }, + "rededge2": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B06.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red Edge 2 - 20m", + "eo:bands": [ + { + "name": "B06", + "common_name": "rededge", + "center_wavelength": 0.74, + "full_width_half_max": 0.018 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "12200c6ba4689a5b62374e6c8ac62e6b5643e0332f4123f035ad55909ad28a5470c4", + "file:size": 54586170, + "roles": [ + "data", + "reflectance" + ] + }, + "rededge3": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B07.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red Edge 3 - 20m", + "eo:bands": [ + { + "name": "B07", + "common_name": "rededge", + "center_wavelength": 0.783, + "full_width_half_max": 0.028 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "12205d5c44c1b64bd7dcf90ef8bc5ea28a8e450d64aac29c38ffbb48fa068433d5ea", + "file:size": 54487352, + "roles": [ + "data", + "reflectance" + ] + }, + "rededge1": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B05.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red Edge 1 - 20m", + "eo:bands": [ + { + "name": "B05", + "common_name": "rededge", + "center_wavelength": 0.704, + "full_width_half_max": 0.019 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220e6d10efe8a30616d2af2c59b63ff7464a54e9f065862f5c78e789119ba420bb9", + "file:size": 54815811, + "roles": [ + "data", + "reflectance" + ] + }, + "swir16": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B11.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "SWIR 1.6μm - 20m", + "eo:bands": [ + { + "name": "B11", + "common_name": "swir16", + "center_wavelength": 1.61, + "full_width_half_max": 0.143 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220cc32a6a3d9f6a7f93797650e88f2adc550c682891f44a830ae5d201013bcf94f", + "file:size": 52499952, + "roles": [ + "data", + "reflectance" + ] + }, + "wvp": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/WVP.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Water Vapour (WVP)", + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "unit": "cm", + "scale": 0.001, + "offset": 0 + } + ], + "file:checksum": "1220eed28c460450356c7c8f8ec0ef47cf12b728844a8bd5d5268a18b9c13f9e013f", + "file:size": 204756, + "roles": [ + "data" + ] + }, + "nir08": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B8A.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "NIR 2 - 20m", + "eo:bands": [ + { + "name": "B8A", + "common_name": "nir08", + "center_wavelength": 0.865, + "full_width_half_max": 0.033 + } + ], + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220e9f6907eab1513d40252005e84a4cf14c82a787ae4570c9a3d8efcdd777359f2", + "file:size": 54483375, + "roles": [ + "data", + "reflectance" + ] + }, + "scl": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/SCL.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Scene classification map (SCL)", + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 20 + } + ], + "file:checksum": "12208a86fee66da3e482bc97a63f44e12336124dcef5f4b1009e8b54d34348ca4513", + "file:size": 1728613, + "roles": [ + "data" + ] + }, + "aot": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/AOT.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Aerosol optical thickness (AOT)", + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 20, + "scale": 0.001, + "offset": 0 + } + ], + "file:checksum": "12200bfe8067764eaafbe505e8661bdac7752b497c1890e25601be55de76b1cd71ab", + "file:size": 299796, + "roles": [ + "data" + ] + }, + "coastal": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B01.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Coastal - 60m", + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal", + "center_wavelength": 0.443, + "full_width_half_max": 0.027 + } + ], + "gsd": 60, + "proj:shape": [ + 1830, + 1830 + ], + "proj:transform": [ + 60, + 0, + 699960, + 0, + -60, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 60, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "1220b0eccf115f188872d8b7ffb8a15efa993e7568e5168e418a5cca1b5e1c09556f", + "file:size": 6357846, + "roles": [ + "data", + "reflectance" + ] + }, + "nir09": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/B09.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "NIR 3 - 60m", + "eo:bands": [ + { + "name": "B09", + "common_name": "nir09", + "center_wavelength": 0.945, + "full_width_half_max": 0.026 + } + ], + "gsd": 60, + "proj:shape": [ + 1830, + 1830 + ], + "proj:transform": [ + 60, + 0, + 699960, + 0, + -60, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint16", + "spatial_resolution": 60, + "scale": 0.0001, + "offset": -0.1 + } + ], + "file:checksum": "12207bc01c1645abe3fe12cceca1dea74f046389842c655de5f1e81c7151b1c547ef", + "file:size": 6581700, + "roles": [ + "data", + "reflectance" + ] + }, + "cloud": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/CLD_20m.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Cloud Probabilities", + "gsd": 20, + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 20 + } + ], + "file:checksum": "1220372d986971f7639b2a45b971abae170fcce343b93319340adbe4cc3696ccb3ce", + "file:size": 6268207, + "roles": [ + "data", + "cloud" + ] + }, + "snow": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/SNW_20m.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Snow Probabilities", + "proj:shape": [ + 5490, + 5490 + ], + "proj:transform": [ + 20, + 0, + 699960, + 0, + -20, + 9400000 + ], + "raster:bands": [ + { + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 20 + } + ], + "file:checksum": "1220449b9426ea2751ee5b7cb166311eea503ffca50943780674ee0a25f8f6310822", + "file:size": 53931, + "roles": [ + "data", + "snow-ice" + ] + }, + "preview": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/L2A_PVI.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "True color preview", + "eo:bands": [ + { + "name": "B04", + "common_name": "red", + "center_wavelength": 0.665, + "full_width_half_max": 0.038 + }, + { + "name": "B03", + "common_name": "green", + "center_wavelength": 0.56, + "full_width_half_max": 0.045 + }, + { + "name": "B02", + "common_name": "blue", + "center_wavelength": 0.49, + "full_width_half_max": 0.098 + } + ], + "file:checksum": "122081017743a1a9c7a5b635714d7678c586b60f54fceda60d7cbc169c0e708a838f", + "file:size": 182766, + "gsd": 320, + "proj:shape": [ + 343, + 343 + ], + "proj:transform": [ + 320, + 0, + 699960, + 0, + -320, + 9400000, + 0, + 0, + 1 + ], + "raster:bands": [ + [ + { + "nodata": null, + "data_type": "uint8", + "spatial_resolution": 320 + }, + { + "nodata": null, + "data_type": "uint8", + "spatial_resolution": 320 + }, + { + "nodata": null, + "data_type": "uint8", + "spatial_resolution": 320 + } + ] + ], + "roles": [ + "overview" + ] + }, + "granule_metadata": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/metadata.xml", + "type": "application/xml", + "file:checksum": "1220daa594263ba606171ea0c914f92deeefc2a55c649520fc08e7fd7f34e4678370", + "file:size": 548196, + "roles": [ + "metadata" + ] + }, + "tileinfo_metadata": { + "href": "s3://deafrica-sentinel-2-l2a-c1-dev/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/tileInfo.json", + "type": "application/json", + "file:checksum": "122000b15e9029e429fe30e33194cb2703c57eb3ab6e4bc275acbbfc8310c4a0cb3a", + "file:size": 1491, + "roles": [ + "metadata" + ] + }, + "product_metadata": { + "href": "https://deafrica-sentinel-2-l2a-c1-dev.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/product_metadata.xml", + "type": "application/xml", + "file:checksum": "1220a6c19a720f61a382e90ceabf2f4cd801b90066efb8e20c12faf7857e835aea14", + "file:size": 54777, + "roles": [ + "metadata" + ] + }, + "thumbnail": { + "href": "https://deafrica-sentinel-2-l2a-c1-dev.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/L2A_PVI.jpg", + "type": "image/jpeg", + "title": "Thumbnail of preview image", + "file:checksum": "122004e7449551768295e4a30b2ddc88b93ea40bc58983cbf9bc8629555fbc292de5", + "file:size": 35902, + "roles": [ + "thumbnail" + ] + } + }, + "bbox": [ + 40.804737, + -6.418314, + 41.800047, + -5.421772 + ], + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json" + ], + "collection": "s2_l2a_c1" +} diff --git a/apps/dc_tools/tests/test_sqs_to_dc.py b/apps/dc_tools/tests/test_sqs_to_dc.py index 43a3741f3..13ca3a7f5 100644 --- a/apps/dc_tools/tests/test_sqs_to_dc.py +++ b/apps/dc_tools/tests/test_sqs_to_dc.py @@ -140,7 +140,7 @@ def test_extract_metadata_from_message(aws_credentials, odc_test_db_with_product def test_handle_json_message(ga_ls8c_ard_3_message, ga_ls8c_ard_3_yaml): actual_doc, uri = handle_json_message( - ga_ls8c_ard_3_message, None, "STAC-LINKS-REL:odc_yaml" + ga_ls8c_ard_3_message, "STAC-LINKS-REL:odc_yaml" ) assert type(actual_doc) is dict @@ -170,7 +170,7 @@ def test_handle_json_message(ga_ls8c_ard_3_message, ga_ls8c_ard_3_yaml): def test_odc_metadata_link(ga_ls8c_ard_3_message): actual_doc, uri = handle_json_message( - ga_ls8c_ard_3_message, None, "STAC-LINKS-REL:odc_yaml" + ga_ls8c_ard_3_message, "STAC-LINKS-REL:odc_yaml" ) assert ( uri == "https://dea-public-data.s3-ap-southeast-2.amazonaws.com/" @@ -180,7 +180,7 @@ def test_odc_metadata_link(ga_ls8c_ard_3_message): def test_stac_link(ga_ls8c_ard_3_message): - metadata, uri = handle_json_message(ga_ls8c_ard_3_message, stac_transform, None) + metadata, uri = handle_json_message(ga_ls8c_ard_3_message, None) assert ( uri != "https://dea-public-data.s3-ap-southeast-2.amazonaws.com/" "baseline/ga_ls8c_ard_3/088/080/2020/05/25/" @@ -194,7 +194,8 @@ def test_stac_link(ga_ls8c_ard_3_message): def test_transform(ga_ls8c_ard_3_message, ga_ls8c_ard_3_yaml): - actual_doc, uri = handle_json_message(ga_ls8c_ard_3_message, stac_transform, None) + actual_doc, uri = handle_json_message(ga_ls8c_ard_3_message, None) + actual_doc = stac_transform(actual_doc) assert ga_ls8c_ard_3_yaml["id"] == actual_doc["id"] assert ga_ls8c_ard_3_yaml["crs"] == actual_doc["crs"] diff --git a/apps/dc_tools/tests/test_stac_transform.py b/apps/dc_tools/tests/test_stac_transform.py index e5b89978a..882d2c2d4 100644 --- a/apps/dc_tools/tests/test_stac_transform.py +++ b/apps/dc_tools/tests/test_stac_transform.py @@ -62,7 +62,51 @@ def test_landsat_stac_transform(landsat_stac, landsat_odc): def test_sentinel_stac_transform(sentinel_stac_old, sentinel_odc): transformed_stac_doc = stac_transform(sentinel_stac_old) - assert len(get_doc_changes(transformed_stac_doc, sentinel_odc)) == 1 + doc_changes = get_doc_changes(transformed_stac_doc, sentinel_odc) + assert len(doc_changes) == 1 + + +def test_sentinel_c1_stac_transform(sentinel_c1_stac, sentinel_c1_odc): + transformed_stac_doc = stac_transform(sentinel_c1_stac) + doc_changes = get_doc_changes(transformed_stac_doc, sentinel_c1_odc) + assert len(doc_changes) == 0 + # test that absolute links are maintained for accessories + assert ( + transformed_stac_doc["accessories"]["thumbnail"]["path"] + == sentinel_c1_stac["assets"]["thumbnail"]["href"] + ) + assert ( + transformed_stac_doc["accessories"]["product_metadata"]["path"] + == sentinel_c1_stac["assets"]["product_metadata"]["href"] + ) + assert ( + transformed_stac_doc["accessories"]["tileinfo_metadata"]["path"] + == sentinel_c1_stac["assets"]["tileinfo_metadata"]["href"] + ) + assert ( + transformed_stac_doc["accessories"]["granule_metadata"]["path"] + == sentinel_c1_stac["assets"]["granule_metadata"]["href"] + ) + + +def test_sentinel_c1_rel_stac_transform(sentinel_c1_rel_stac, sentinel_c1_rel_odc): + transformed_stac_doc = stac_transform(sentinel_c1_rel_stac) + doc_changes = get_doc_changes(transformed_stac_doc, sentinel_c1_rel_odc) + assert len(doc_changes) == 0 + # test that absolute links are converted to relative links for accessories + assert transformed_stac_doc["accessories"]["thumbnail"]["path"] == "L2A_PVI.jpg" + assert ( + transformed_stac_doc["accessories"]["product_metadata"]["path"] + == "product_metadata.xml" + ) + assert ( + transformed_stac_doc["accessories"]["tileinfo_metadata"]["path"] + == "tileInfo.json" + ) + assert ( + transformed_stac_doc["accessories"]["granule_metadata"]["path"] + == "metadata.xml" + ) def test_usgs_landsat_stac_transform(usgs_landsat_stac): diff --git a/libs/cloud/odc/aio.py b/libs/cloud/odc/aio.py index b77bcf643..428e00eb4 100644 --- a/libs/cloud/odc/aio.py +++ b/libs/cloud/odc/aio.py @@ -44,7 +44,7 @@ def result(data=None, last_modified=None, error=None): ) bucket, key = s3_url_parse(url) - extra_args = dict(**kw) + extra_args = {**kw} if _range is not None: try: @@ -327,7 +327,7 @@ def __init__( s3_cfg = AioConfig( max_pool_connections=nconcurrent, **opts, - s3=dict(addressing_style=addressing_style), + s3={"addressing_style": addressing_style}, ) self._nconcurrent = nconcurrent @@ -582,4 +582,4 @@ def dirs_pred(f): else: stream = do_dir_query(qq, dirs_pred=dirs_pred) - return stream + return stream # pylint: disable=possibly-used-before-assignment diff --git a/libs/cloud/odc/aws/dns.py b/libs/cloud/odc/aws/dns.py index 2ea73cec0..a2f351b77 100644 --- a/libs/cloud/odc/aws/dns.py +++ b/libs/cloud/odc/aws/dns.py @@ -1,6 +1,8 @@ """ Tools for interacting with route53 """ +import sys + from . import _fetch_text, ec2_tags, mk_boto_session @@ -134,6 +136,4 @@ def display_help(): if __name__ == "__main__": - import sys - sys.exit(cli(sys.argv[1:])) diff --git a/libs/cloud/odc/aws/misc.py b/libs/cloud/odc/aws/misc.py index 3b103d730..eae8d9a0e 100644 --- a/libs/cloud/odc/aws/misc.py +++ b/libs/cloud/odc/aws/misc.py @@ -71,6 +71,6 @@ def build_request( auth.add_auth(req) - return Request(req.url, headers=dict(**req.headers), method="GET") + return Request(req.url, headers={**req.headers}, method="GET") return build_request diff --git a/libs/cloud/odc/aws/queue.py b/libs/cloud/odc/aws/queue.py index c8d0e1108..773d80021 100644 --- a/libs/cloud/odc/aws/queue.py +++ b/libs/cloud/odc/aws/queue.py @@ -136,8 +136,7 @@ def _sqs_message_stream(queue, **kw): if len(messages) == 0: return - for msg in messages: - yield msg + yield from messages def get_messages( diff --git a/libs/cloud/tests/test_azure.py b/libs/cloud/tests/test_azure.py index f0acbc810..02a60d8d7 100644 --- a/libs/cloud/tests/test_azure.py +++ b/libs/cloud/tests/test_azure.py @@ -1,9 +1,11 @@ """Test thredds downloader code """ +import pytest from odc.azure import download_yamls, find_blobs +@pytest.mark.xfail(reason="Libcloud azure tests are broken") def test_find_blobs(): """Find blobs in a sample Azure account, this will fail if the blob store changes or is removed""" @@ -21,6 +23,7 @@ def test_find_blobs(): assert len(blob_names) == 1 +@pytest.mark.xfail(reason="Libcloud azure tests are broken") def test_download_yamls(): """Test pass/fail arms of YAML download from Azure blobstore""" diff --git a/libs/io/odc/io/__init__.py b/libs/io/odc/io/__init__.py index 2067bcac7..d2fa9107e 100644 --- a/libs/io/odc/io/__init__.py +++ b/libs/io/odc/io/__init__.py @@ -1,11 +1,17 @@ """ Various file io helpers """ -from .tar import tar_doc_stream -from .text import parse_mtl, parse_yaml, read_stdin_lines, slurp, slurp_lines -from .timer import RateEstimator +from .tar import tar_doc_stream # pylint: disable=W0406 +from .text import ( + parse_mtl, # pylint: disable=W0406 + parse_yaml, # pylint: disable=W0406 + read_stdin_lines, # pylint: disable=W0406 + slurp, # pylint: disable=W0406 + slurp_lines, # pylint: disable=W0406 +) +from .timer import RateEstimator # pylint: disable=W0406 -from ._version import __version__ +from ._version import __version__ # pylint: disable=W0406 __all__ = ( "parse_yaml", diff --git a/libs/io/odc/io/text.py b/libs/io/odc/io/text.py index fde7d9788..c92d19231 100644 --- a/libs/io/odc/io/text.py +++ b/libs/io/odc/io/text.py @@ -73,7 +73,7 @@ def slurp(fname: PathLike, binary: bool = False) -> RawDoc: """ mode = "rb" if binary else "rt" - with open(fname, mode) as f: + with open(fname, mode) as f: # pylint: disable=unspecified-encoding return f.read() @@ -82,7 +82,7 @@ def slurp_lines(fname: str, *args, **kwargs) -> List[str]: if len(args) > 0 or len(kwargs) > 0: fname = fname.format(*args, **kwargs) - with open(fname, "rt") as f: + with open(fname, "rt") as f: # pylint: disable=unspecified-encoding return [s.rstrip() for s in f.readlines()] diff --git a/libs/ui/odc/ui/_dc_explore.py b/libs/ui/odc/ui/_dc_explore.py index f533b6fc9..9fdab60c4 100644 --- a/libs/ui/odc/ui/_dc_explore.py +++ b/libs/ui/odc/ui/_dc_explore.py @@ -59,7 +59,7 @@ def __init__( products = list(p.name for p, c in dc.index.datasets.count_by_product()) if style is None: - style = dict(fillOpacity=0.1, weight=1) + style = {"fillOpacity": 0.1, "weight": 1} state, gui = self._build_ui( products, time, zoom=zoom, center=center, height=height, width=width @@ -169,7 +169,7 @@ def bounds_handler(event): lat1 = max(lat1, -90) lat2 = min(lat2, +90) - state.bounds = dict(lat=(lat1, lat2), lon=(lon1, lon2)) + state.bounds = {"lat": (lat1, lat2), "lon": (lon1, lon2)} self.on_bounds(state.bounds) @@ -228,7 +228,7 @@ def _update_footprints(self): self._clear_footprints() self._dss_layer = new_layer - self._last_query_bounds = dict(**s.bounds) + self._last_query_bounds = {**s.bounds} self._last_query_polygon = _query_polygon(**s.bounds) else: self._clear_footprints() diff --git a/libs/ui/odc/ui/_images.py b/libs/ui/odc/ui/_images.py index 4c1ecf692..2dd2d00a3 100644 --- a/libs/ui/odc/ui/_images.py +++ b/libs/ui/odc/ui/_images.py @@ -71,7 +71,14 @@ def _compress_image(im: np.ndarray, driver="PNG", **opts) -> bytes: else: raise ValueError(f"Expect 2 or 3 dimensional array got: {im.ndim}") - rio_opts = dict(width=w, height=h, count=nc, driver=driver, dtype="uint8", **opts) + rio_opts = { + "width": w, + "height": h, + "count": nc, + "driver": driver, + "dtype": "uint8", + **opts, + } with warnings.catch_warnings(): warnings.simplefilter("ignore", rasterio.errors.NotGeoreferencedWarning) @@ -139,11 +146,11 @@ def mk_image_overlay( ipyleaflet.ImageOverlay or a list of them one per time slice """ - comp, mime = dict( - png=(to_png_data, "image/png"), - jpg=(to_jpeg_data, "image/jpeg"), - jpeg=(to_jpeg_data, "image/jpeg"), - ).get(fmt.lower(), (None, None)) + comp, mime = { + "png": (to_png_data, "image/png"), + "jpg": (to_jpeg_data, "image/jpeg"), + "jpeg": (to_jpeg_data, "image/jpeg"), + }.get(fmt.lower(), (None, None)) if comp is None or mime is None: raise ValueError("Only support png an jpeg formats") diff --git a/libs/ui/odc/ui/_map.py b/libs/ui/odc/ui/_map.py index 7da680163..954262929 100644 --- a/libs/ui/odc/ui/_map.py +++ b/libs/ui/odc/ui/_map.py @@ -33,15 +33,15 @@ def gridspec_to_geojson(gs, xx, yy, styles): def to_geojson(gs, tidx): bbox = gs.tile_geobox(tidx) - return dict( - geometry=bbox.geographic_extent.__geo_interface__, - type="Feature", - properties=dict( + return { + "geometry": bbox.geographic_extent.__geo_interface__, + "type": "Feature", + "properties": { # pylint:disable=consider-using-f-string - title="{:+d},{:+d}".format(*tidx), + "title": "{:+d},{:+d}".format(*tidx), **styles, - ), - ) + }, + } tiles = itertools.product(range(*xx), range(*yy)) @@ -179,9 +179,9 @@ def render_bounds(bounds): shape_opts = {"fillColor": "#fca45d", "color": "#000000", "fillOpacity": 0.1} draw.rectangle = {"shapeOptions": shape_opts, "metric": ["km", "m"]} - poly_opts = {"shapeOptions": dict(**shape_opts)} - poly_opts["shapeOptions"]["original"] = dict(**shape_opts) - poly_opts["shapeOptions"]["editing"] = dict(**shape_opts) + poly_opts = {"shapeOptions": {**shape_opts}} + poly_opts["shapeOptions"]["original"] = {**shape_opts} + poly_opts["shapeOptions"]["editing"] = {**shape_opts} draw.polygon = poly_opts draw.edit = True @@ -200,7 +200,7 @@ def bounds_handler(event): bounds = event["new"] render_bounds(bounds) (lat1, lon1), (lat2, lon2) = bounds - state.bounds = dict(lat=(lat1, lat2), lon=(lon1, lon2)) + state.bounds = {"lat": (lat1, lat2), "lon": (lon1, lon2)} def on_draw(event): v = event["new"] diff --git a/libs/ui/odc/ui/plt_tools.py b/libs/ui/odc/ui/plt_tools.py index 4cf8e9f30..d27235908 100644 --- a/libs/ui/odc/ui/plt_tools.py +++ b/libs/ui/odc/ui/plt_tools.py @@ -27,7 +27,7 @@ def compare_masks( - ``removed`` are pixels that turned False from A->B """ fig, axs = plt.subplots(2, 2, figsize=figsize) - opts = dict(interpolation=interpolation, cmap=cmap, **kw) + opts = {"interpolation": interpolation, "cmap": cmap, **kw} axs[0][0].set_title(names[0]) axs[0][0].imshow(a, **opts) diff --git a/scripts/mk-pip-tree.sh b/scripts/mk-pip-tree.sh index e95dfb91a..82e87a182 100755 --- a/scripts/mk-pip-tree.sh +++ b/scripts/mk-pip-tree.sh @@ -13,10 +13,10 @@ create_pip_tree() { local out="${dst}/${base}" mkdir -p "${out}" - echo "${src}/${w}"* "-> ${out}/" - echo "${src}/${base}"*tar.gz "-> ${out}/" + echo "${src}/${w}"*.whl "-> ${out}/" + echo "${src}/${w}"*tar.gz "-> ${out}/" cp "${src}/${w}"*.whl "${out}/" - cp "${src}/${base}"*.tar.gz "${out}/" + cp "${src}/${w}"*.tar.gz "${out}/" done } diff --git a/tests/test-env.yml b/tests/test-env.yml index 40da16153..0ec8b2224 100644 --- a/tests/test-env.yml +++ b/tests/test-env.yml @@ -26,6 +26,7 @@ dependencies: - lxml # needed for thredds-crawler - urlpath - datadog + - docker-py - eodatasets3 - importlib_resources>=6.0