From c1943742d8039b63072047107237eb174a33faa0 Mon Sep 17 00:00:00 2001 From: Raghuram Date: Mon, 3 Feb 2025 12:04:47 +0530 Subject: [PATCH 1/2] Script to upload the datasets for cli.081-085 --- .../.DS_Store | Bin 0 -> 6148 bytes .../README.md | 14 ++ ...081_india_andhra_tn_climate_projections.py | 133 ++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 cli_081_india_andhra_tn_climate_projections/.DS_Store create mode 100644 cli_081_india_andhra_tn_climate_projections/README.md create mode 100644 cli_081_india_andhra_tn_climate_projections/cli_081_india_andhra_tn_climate_projections.py diff --git a/cli_081_india_andhra_tn_climate_projections/.DS_Store b/cli_081_india_andhra_tn_climate_projections/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..0442cfde9d00e36db194a1b160d813857396f79b GIT binary patch literal 6148 zcmeHK!AiqG5Z!H~O(;SRiXH=A3%1mv#Y>3w2aM=Jr6#6mFlI}WnnNk%sz2nP_&v_- zZi-R`Pa<{(X5VCXW|MshI~m3p_oh*YF`F@Ff+E%|7`_mkM_rPd_S6A6t`Qd5I7~zc z*RM;_3O^6>Y32o!OT0Q0 zA_*%y2+!kDF|;?2MVbe3I-1CYI2u97)kT~}qVU8tjWU_*s|nLGt)aa=n|1n!E$5&& zZ#lDmw+H5acRshwt=+w&)8Rw%n2Kk`A&1XR%7(@YUcgvvC(3S<5FQVM7tSYtk|rX( z17EdBl}tzs5Cg=(sxY9>KeM?i8>W2{1H{0O8Nl z%yBfj8cT&R1Hx4*pi1Ss#o#I({Em)uHI@ohI^(+e;QG#7cPL!m9oBbbIODED8i@g7 z;4=f*`+?Y4|4+W}|78#j!~ikyKN;Z0z#n+9H(gtoc8Rst0lfi5!MIZ4A_WXtiXj$D d@eZgI@H=P#x*AJ`-~pi@0Z9W5#K5mI@D454PXGV_ literal 0 HcmV?d00001 diff --git a/cli_081_india_andhra_tn_climate_projections/README.md b/cli_081_india_andhra_tn_climate_projections/README.md new file mode 100644 index 0000000..3743e90 --- /dev/null +++ b/cli_081_india_andhra_tn_climate_projections/README.md @@ -0,0 +1,14 @@ +## Ted Wong created dataset for long term climate projections for two states in India + +This dataset is provided by the source as a series of tif files, which were uploaded to Google Earth Engine. + +You can view the processed dataset for display on Agriadapt at the links below. +https://agriadapt.org/explore/cli081crt0-Projected-minimum-temperature-mean?zoom=3&lat=7.2752923363727575&lng=65.69850221673482&pitch=0&bearing=0&basemap=dark&labels=light&tab=layers&emission_scenario=rcp4.5 +https://agriadapt.org/explore/cli082crt0-Projected-maximum-temperature-mean?zoom=3&lat=7.2752923363727575&lng=65.69850221673482&pitch=0&bearing=0&basemap=dark&labels=light&tab=layers&emission_scenario=rcp4.5 +https://agriadapt.org/explore/cli083crt0-Projected-mean-temperature?zoom=3&lat=7.2752923363727575&lng=65.69850221673482&pitch=0&bearing=0&basemap=dark&labels=light&tab=layers&emission_scenario=rcp4.5 +https://agriadapt.org/explore/cli084crt0-Projected-Extreme-rainy-days?zoom=3&lat=7.2752923363727575&lng=65.69850221673482&pitch=0&bearing=0&basemap=dark&labels=light&tab=layers&emission_scenario=rcp4.5 +https://agriadapt.org/explore/cli085crt0-Projected-total-rainfall?zoom=3&lat=7.2752923363727575&lng=65.69850221673482&pitch=0&bearing=0&basemap=dark&labels=light&tab=layers&emission_scenario=rcp4.5 + +You can also download the original datasets at https://wri-agriadapt-datasets.s3.us-east-1.amazonaws.com/cli_081/climate_data_proj_india_ap_tn_ted.zip + +###### Note: This dataset processing was done by [Raghuram BK] diff --git a/cli_081_india_andhra_tn_climate_projections/cli_081_india_andhra_tn_climate_projections.py b/cli_081_india_andhra_tn_climate_projections/cli_081_india_andhra_tn_climate_projections.py new file mode 100644 index 0000000..20af4d7 --- /dev/null +++ b/cli_081_india_andhra_tn_climate_projections/cli_081_india_andhra_tn_climate_projections.py @@ -0,0 +1,133 @@ +import ee +import os +import sys +from google.cloud import storage +from google_drive_downloader import GoogleDriveDownloader as gdd +import logging +from zipfile import ZipFile +from pathlib import Path + +utils_path = os.path.join(os.path.abspath(os.getenv("PROCESSING_DIR")), "utils") +if utils_path not in sys.path: + sys.path.append(utils_path) +import util_files +import util_cloud + +# Get the top-level logger object +logger = logging.getLogger() +for handler in logger.handlers: + logger.removeHandler(handler) +logger.setLevel(logging.DEBUG) +# make it print to the console. +console = logging.StreamHandler() +logger.addHandler(console) +logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") + +# name of asset on GEE where you want to upload data +# this should be an asset name that is not currently in use +dataset_name = "cli_081_india_andhra_tn_climate_projections" +logger.info("Executing script for dataset: " + dataset_name) + +data_dir = util_files.prep_dirs(dataset_name) +logger.debug("Data directory relative path: " + data_dir) +logger.debug("Data directory absolute path: " + os.path.abspath(data_dir)) + +dataset = "climate_data_proj_india_ap_tn_ted" + +# Extract the datafiles from the zip +with ZipFile(os.path.join(f"{os.path.abspath(data_dir)}", f"{dataset}.zip"), "r") as zf: + zf.extractall(os.path.abspath(data_dir)) + +""" +Process data and upload processed data to Google Earth Engine +""" + +# initialize ee and eeUtil modules for uploading to Google Earth Engine + +auth = ee.ServiceAccountCredentials( + os.getenv("GEE_SERVICE_ACCOUT"), os.getenv("GOOGLE_APPLICATION_CREDENTIALS") +) +ee.Initialize(auth) + +# set up Google Cloud Storage project and bucket objects +gcs_client = storage.Client(os.environ.get("CLOUDSDK_CORE_PROJECT")) +gcs_bucket = gcs_client.bucket(os.environ.get("GEE_STAGING_BUCKET")) + +indicators = { + "total_rainfall": "TotalRainfall/Annual", + "extreme_rainy_days": "ExtremeRainyDays", + "mean_temp": "MeanTemp", + "max_temp": "MaxTemp/Annual", + "min_temp": "MinTemp/Annual", +} +print("creating folder") + ee.data.createAsset( + {"type": "Folder"}, "projects/resource-watch-gee/climate_data_proj_india_ap_tn_ted" + ) +print("created folder") +measures = ["mean"] +for indicator, indicator_dir in indicators.items(): + for measure in measures: + # print(f"projects/resource-watch-gee/{dataset}/{dataset}_{indicator}_{measure}") + # print(f"{os.path.abspath(data_dir)}/{dataset}/{indicator_dir}") + + image_collection = ( + f"projects/resource-watch-gee/{dataset}/{dataset}_{indicator}_{measure}" + ) + print(image_collection) + # create IC + # Already created by Chris so skip creating the image collection + ee.data.createAsset({"type": "ImageCollection"}, image_collection) + # set dataset privacy to public + acl = {"all_users_can_read": True} + ee.data.setAssetAcl(image_collection, acl) + logger.info("Privacy set to public.") + + for path in Path( + f"{os.path.abspath(data_dir)}/{dataset}/{indicator_dir}" + ).rglob(f"*_{measure}*"): + logger.debug(f"Processing {path}") + + image_name = os.path.splitext(os.path.basename(path))[0] + + """ + Upload processed data to Google Earth Engine + """ + logger.info("Uploading processed data to Google Cloud Storage.") + + # upload local files to Google Cloud Storage + gs_uris = util_cloud.gcs_upload( + str(path), os.path.join(dataset, f"{indicator}_{measure}") + ) + + logger.info("Uploading processed data to Google Earth Engine.") + + # set pyramiding policy for GEE upload + pyramiding_policy = "MEAN" # check + + # create manifests for upload to GEE + band_manifest = [ + { + "id": "Band1", + "tileset_band_index": 0, + "tileset_id": os.path.basename(gs_uris[0]).split(".")[0], + "pyramidingPolicy": pyramiding_policy, + } + ] + + asset_name = f"{image_collection}/{image_name}" + manifest = util_cloud.gee_manifest_complete( + asset_name, gs_uris[0], band_manifest + ) + + properties = { + "SCP": "ssp245.ssp585", + "change_vs_absolute": "abs", + } + manifest["properties"] = properties + # upload from GCS to GEE + util_cloud.gee_ingest(manifest, public=True) + # delete files from GCS + util_cloud.gcs_remove(gs_uris, gcs_bucket=gcs_bucket) + + logger.info("Files deleted from Google Cloud Storage.") From 1a5cc1a97b2a65a65cf8af536fd381a94fb5e82a Mon Sep 17 00:00:00 2001 From: bkrram Date: Mon, 3 Feb 2025 12:09:18 +0530 Subject: [PATCH 2/2] Delete cli_081_india_andhra_tn_climate_projections/.DS_Store Mistakenly uploaded Mac DS store file.. --- .../.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 cli_081_india_andhra_tn_climate_projections/.DS_Store diff --git a/cli_081_india_andhra_tn_climate_projections/.DS_Store b/cli_081_india_andhra_tn_climate_projections/.DS_Store deleted file mode 100644 index 0442cfde9d00e36db194a1b160d813857396f79b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK!AiqG5Z!H~O(;SRiXH=A3%1mv#Y>3w2aM=Jr6#6mFlI}WnnNk%sz2nP_&v_- zZi-R`Pa<{(X5VCXW|MshI~m3p_oh*YF`F@Ff+E%|7`_mkM_rPd_S6A6t`Qd5I7~zc z*RM;_3O^6>Y32o!OT0Q0 zA_*%y2+!kDF|;?2MVbe3I-1CYI2u97)kT~}qVU8tjWU_*s|nLGt)aa=n|1n!E$5&& zZ#lDmw+H5acRshwt=+w&)8Rw%n2Kk`A&1XR%7(@YUcgvvC(3S<5FQVM7tSYtk|rX( z17EdBl}tzs5Cg=(sxY9>KeM?i8>W2{1H{0O8Nl z%yBfj8cT&R1Hx4*pi1Ss#o#I({Em)uHI@ohI^(+e;QG#7cPL!m9oBbbIODED8i@g7 z;4=f*`+?Y4|4+W}|78#j!~ikyKN;Z0z#n+9H(gtoc8Rst0lfi5!MIZ4A_WXtiXj$D d@eZgI@H=P#x*AJ`-~pi@0Z9W5#K5mI@D454PXGV_