diff --git a/.github/workflows/reusable-DeployStack-SearchAPI.yml b/.github/workflows/reusable-DeployStack-SearchAPI.yml index 9f2bd9a8..a5c914b5 100644 --- a/.github/workflows/reusable-DeployStack-SearchAPI.yml +++ b/.github/workflows/reusable-DeployStack-SearchAPI.yml @@ -96,6 +96,9 @@ jobs: # If it's writing to a branch, the pull request to that branch already # passed, OR it's a developing branch that doesn't need PR's. No need # to run suite either way. + - uses: actions/setup-python@v5 + with: + python-version: '3.9' - name: Install pytest requirements if: github.event_name == 'pull_request' run: | diff --git a/SearchAPI/CMR/Output/jsonlite.py b/SearchAPI/CMR/Output/jsonlite.py index 9183f8ac..6cab9f73 100644 --- a/SearchAPI/CMR/Output/jsonlite.py +++ b/SearchAPI/CMR/Output/jsonlite.py @@ -45,6 +45,11 @@ def req_fields_jsonlite(): 'additionalUrls', 's3Urls', 'ariaVersion', + 'framecoverage', + 'jointobservation', + 'mainbandpolarization', + 'sidebandpolarization', + 'rangebandwidth', ] return fields @@ -179,8 +184,7 @@ def getItem(self, p): burst['subswath'] = p['subswath'] result['burst'] = burst - - if p.get('operaBurstID') is not None or result['productID'].startswith('OPERA'): + if (p.get('operaBurstID') is not None or result['productID'].startswith('OPERA')) and not result['productID'].startswith('OPERA_L3_DISP'): result['opera'] = { 'operaBurstID': p.get('operaBurstID'), 'additionalUrls': p.get('additionalUrls'), @@ -191,7 +195,12 @@ def getItem(self, p): if p.get('platform') == 'NISAR': result['nisar'] = { 'additionalUrls': p.get('additionalUrls', []), - 's3Urls': p.get('s3Urls', []) + 's3Urls': p.get('s3Urls', []), + 'frameCoverage': p.get('framecoverage'), + 'jointObservation': p.get('jointobservation'), + 'mainBandPolarization': p.get('mainbandpolarization'), + 'sideBandPolarization': p.get('sidebandpolarization'), + 'rangeBandwidth': p.get('rangebandwidth') } diff --git a/SearchAPI/CMR/Query.py b/SearchAPI/CMR/Query.py index 77b3f146..8efc2e68 100644 --- a/SearchAPI/CMR/Query.py +++ b/SearchAPI/CMR/Query.py @@ -114,7 +114,7 @@ def chunk_list(source_list, n): if chunk_type in params: params[chunk_type] = chunk_list(list(set(params[chunk_type])), 500) # distinct and split - list_param_names = ['platform', 'collections', 'shortname'] # these parameters will dodge the subquery system + list_param_names = ['platform', 'collections', 'shortname', 'instrument'] # these parameters will dodge the subquery system for k, v in params.items(): if k in list_param_names: diff --git a/SearchAPI/CMR/SubQuery.py b/SearchAPI/CMR/SubQuery.py index 00087a00..5f77553f 100644 --- a/SearchAPI/CMR/SubQuery.py +++ b/SearchAPI/CMR/SubQuery.py @@ -61,7 +61,7 @@ def combine_params(self, params, extra_params): def should_use_asf_frame(self): asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'ALOS'] - asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2'] + asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2', 'ALOS-2'] asf_frame_collections = [] for dataset in asf_frame_datasets: diff --git a/SearchAPI/CMR/Translate/datasets.py b/SearchAPI/CMR/Translate/datasets.py index 7b9da157..874811e0 100644 --- a/SearchAPI/CMR/Translate/datasets.py +++ b/SearchAPI/CMR/Translate/datasets.py @@ -124,6 +124,8 @@ "C1259981910-ASF", "C1257995186-ASF", "C1259974840-ASF", + "C1271830354-ASF", # OPERA-DISP + "C3294057315-ASF", ], "OPERA-S1-CALVAL": [ "C1260721945-ASF", # CSLC @@ -132,6 +134,9 @@ "C2803501097-ASF", ], "SLC-BURST": ["C2709161906-ASF", "C1257024016-ASF"], + "ALOS-2": [ + "C3315903479-ASF" + ], "ALOS PALSAR": [ "C1206487504-ASF", "C1206485940-ASF", diff --git a/SearchAPI/CMR/Translate/fields.py b/SearchAPI/CMR/Translate/fields.py index 273c2eea..e3b70d65 100644 --- a/SearchAPI/CMR/Translate/fields.py +++ b/SearchAPI/CMR/Translate/fields.py @@ -62,7 +62,12 @@ def get_field_paths(): 'pgeVersion': "./PGEVersionClass/PGEVersion", 'additionalUrls': "./OnlineAccessURLs", 's3Urls': "./OnlineAccessURLs", + 'framecoverage': attr_path('FULL_FRAME'), + 'jointobservation': attr_path('JOINT_OBSERVATION'), + 'rangebandwidth': attr_path('RANGE_BANDWIDTH_CONCAT'), + 'mainbandpolarization': attr_path('FREQUENCY_A_POLARIZATION_CONCAT'), + 'sidebandpolarization': attr_path('FREQUENCY_B_POLARIZATION_CONCAT'), # BURST FIELDS 'absoluteBurstID': attr_path('BURST_ID_ABSOLUTE'), 'relativeBurstID': attr_path('BURST_ID_RELATIVE'), diff --git a/SearchAPI/CMR/Translate/input_fixer.py b/SearchAPI/CMR/Translate/input_fixer.py index 602c6b1e..fb581bba 100644 --- a/SearchAPI/CMR/Translate/input_fixer.py +++ b/SearchAPI/CMR/Translate/input_fixer.py @@ -29,6 +29,22 @@ def input_fixer(params, is_prod: bool = False, provider: str = "ASF"): 'A': 'ASCENDING', 'D': 'DESCENDING' }[v[0].upper()] + elif k == 'framecoverage': + if v[0].upper() not in ['F', 'P']: + raise ValueError(f'Invalid frame coverage selected: {v}') + fixed_params[k] = { + 'F': 'TRUE', + 'P': 'FALSE', + }[v[0].upper()] + elif k == 'jointobservation': + if isinstance(v, bool): # convert to string for CMR + v = str(v).upper() + elif v[0].upper() not in ['F', 'T']: + raise ValueError(f'Invalid joint observation specified (expected True or False). Got: {v}') + fixed_params[k] = { + 'T': 'TRUE', + 'F': 'FALSE', + }[v[0].upper()] elif k == 'season': # clamp range or abort if len(v) != 2: raise ValueError( diff --git a/SearchAPI/CMR/Translate/input_map.py b/SearchAPI/CMR/Translate/input_map.py index fc879799..390fec97 100644 --- a/SearchAPI/CMR/Translate/input_map.py +++ b/SearchAPI/CMR/Translate/input_map.py @@ -27,6 +27,9 @@ def input_map(): 'flightdirection': ['attribute[]', 'string,ASCENDING_DESCENDING,{0}', parse_string], 'flightline': ['attribute[]', 'string,FLIGHT_LINE,{0}', parse_string], 'frame': ['attribute[]', 'int,CENTER_ESA_FRAME,{0}', parse_int_or_range_list], + 'framecoverage': ['attribute[]', 'string,FULL_FRAME,{0}', parse_string], + 'jointobservation': ['attribute[]', 'string,JOINT_OBSERVATION,{0}', parse_string], + 'rangebandwidth': ['attribute[]', 'string,RANGE_BANDWIDTH_CONCAT,{0}',parse_string_list], 'granule_list': ['readable_granule_name[]', '{0}', parse_string_list], 'product_list': ['granule_ur[]', '{0}', parse_string_list], 'maxinsarstacksize': ['attribute[]', 'int,INSAR_STACK_SIZE,,{0}', parse_int], @@ -37,12 +40,15 @@ def input_map(): 'platform': ['platform[]', '{0}', parse_string_list], 'asfplatform': ['attribute[]', 'string,ASF_PLATFORM,{0}', parse_string_list], 'polarization': ['attribute[]', 'string,POLARIZATION,{0}', parse_string_list], + 'mainbandpolarization': ['attribute[]', 'string,FREQUENCY_A_POLARIZATION_CONCAT,{0}', parse_string_list], + 'sidebandpolarization': ['attribute[]', 'string,FREQUENCY_B_POLARIZATION_CONCAT,{0}', parse_string_list], 'polygon': ['polygon', '{0}', parse_coord_string], # intersectsWith ends up here 'linestring': ['line', '{0}', parse_coord_string], # or here 'point': ['point', '{0}', parse_point_string], # or here 'bbox': ['bounding_box', '{0}', parse_bbox_string], 'circle': ['circle[]', '{0}', parse_circle_string], 'processinglevel': ['attribute[]', 'string,PROCESSING_TYPE,{0}', parse_string_list], + 'processingtype': ['attribute[]', 'string,PROCESSING_LEVEL,{0}', parse_string_list], 'relativeorbit': ['attribute[]', 'int,PATH_NUMBER,{0}', parse_int_or_range_list], 'processingdate': ['updated_since', '{0}', parse_date], 'start': [None, '{0}', parse_date], @@ -51,7 +57,7 @@ def input_map(): 'temporal': ['temporal', '{0}', None], # start/end end up here 'groupid': ['attribute[]', 'string,GROUP_ID,{0}', parse_string_list], 'insarstackid': ['attribute[]', 'int,INSAR_STACK_ID,{0}', parse_string], - 'instrument': ['instrument[]', '{0}', parse_string], + 'instrument': ['instrument[]', '{0}', parse_string_list], 'collections': ['echo_collection_id[]', '{0}', parse_string_list], 'relativeburstid': ['attribute[]', 'int,BURST_ID_RELATIVE,{0}', parse_int_list], 'absoluteburstid': ['attribute[]', 'int,BURST_ID_ABSOLUTE,{0}', parse_int_list], diff --git a/SearchAPI/CMR/Translate/parse_cmr_response.py b/SearchAPI/CMR/Translate/parse_cmr_response.py index 194fbb59..bffc6605 100644 --- a/SearchAPI/CMR/Translate/parse_cmr_response.py +++ b/SearchAPI/CMR/Translate/parse_cmr_response.py @@ -66,11 +66,13 @@ def remove_field(f): platform = get_val('./Platforms/Platform/ShortName') result['platform'] = platform remove_field('platform') + if result['platform'] in ['ALOS-2']: + result['beamMode'] = get_val(attr_path('BEAM_MODE')) if 'frameNumber' in req_fields: asf_frame_platforms = [ 'Sentinel-1A', 'Sentinel-1B', 'ALOS', 'SENTINEL-1A', 'SENTINEL-1B', - 'ERS-1', 'ERS-2', 'JERS-1', 'RADARSAT-1' + 'ERS-1', 'ERS-2', 'JERS-1', 'RADARSAT-1', 'ALOS-2', 'NISAR' ] if result['platform'] in asf_frame_platforms: @@ -90,7 +92,7 @@ def remove_field(f): result['fileName'] = file_name.split('/')[-1] if file_name else None remove_field('fileName') - if 'stateVectors' in req_fields or ('canInsar' in req_fields and result['platform'] not in ['ALOS', 'RADARSAT-1', 'JERS-1', 'ERS-1', 'ERS-2']): + if 'stateVectors' in req_fields or ('canInsar' in req_fields and result['platform'] not in ['ALOS', 'ALOS-2', 'RADARSAT-1', 'JERS-1', 'ERS-1', 'ERS-2']): def parse_sv(sv): def float_or_none(a): try: @@ -131,7 +133,7 @@ def float_or_none(a): remove_field('stateVectors') if 'canInsar' in req_fields: - if result['platform'] in ['ALOS', 'RADARSAT-1', 'JERS-1', 'ERS-1', 'ERS-2']: + if result['platform'] in ['ALOS', 'ALOS-2', 'RADARSAT-1', 'JERS-1', 'ERS-1', 'ERS-2']: result['insarGrouping'] = get_val(field_paths['insarGrouping']) insarBaseline = get_val(field_paths['insarBaseline']) @@ -206,7 +208,6 @@ def float_or_none(a): result['fileName'] = result['granuleName'] + '.' + urls[0].split('.')[-1] - def get_all_urls(): accessPath = './OnlineAccessURLs/OnlineAccessURL/URL' resourcesPath = './OnlineResources/OnlineResource/URL' @@ -227,7 +228,7 @@ def get_http_urls(): def get_s3_urls(): return [url for url in get_all_urls() if not url.endswith('.md5') and (url.startswith('s3://') or 's3credentials' in url)] - if result.get('product_file_id', '').startswith('OPERA'): + if result.get('product_file_id', '').startswith('OPERA') and not result.get('product_file_id', '').startswith('OPERA_L3_DISP'): result['beamMode'] = get_val(attr_path('BEAM_MODE')) result['additionalUrls'] = get_http_urls() result['configurationName'] = "Interferometric Wide. 250 km swath, 5 m x 20 m spatial resolution and burst synchronization for interferometry. IW is considered to be the standard mode over land masses." @@ -240,11 +241,13 @@ def get_s3_urls(): elif result.get('product_file_id', '').startswith('S1-GUNW') and result.get('ariaVersion') is None: version_unformatted = result.get('granuleName').split('v')[-1] result['ariaVersion'] = re.sub(r'[^0-9\.]', '', version_unformatted.replace("_", '.')) + if result.get('product_file_id', '').startswith('OPERA_L3_DISP'): + if (providerbrowseUrls := get_all_vals('./AssociatedBrowseImageUrls/ProviderBrowseUrl/URL')): + result['browse'] = [url for url in providerbrowseUrls if not url.startswith('s3://')] if result.get('platform', '') == 'NISAR': result['additionalUrls'] = get_http_urls() result['s3Urls'] = get_s3_urls() - return result