diff --git a/lib/adf_derive.py b/lib/adf_derive.py new file mode 100644 index 000000000..39bf83230 --- /dev/null +++ b/lib/adf_derive.py @@ -0,0 +1,274 @@ +import glob +import os +from pathlib import Path +import xarray as xr + + +def check_derive(self, res, var, case_name, diag_var_list, constit_dict, hist_file_ds, hist0): + """ + For incoming variable, look for list of constituents if available + - as a list in variable defaults file + + If the variable does not have the argument `derivable_from` or `derivable_from_cam_chem`, + then it will be assumed not to be a derivable variable, just missing from history file + + If the variable does have the argument `derivable_from` or `derivable_from_cam_chem`, + first check cam-chem, then regular cam. + + Arguments + --------- + self: AdfDiag + - ADF object + res: dict + - variable defaults dictionary from yaml file + var: str + - derived variable name + case_name: str + - model case + diag_var_list: list + - list of variables for diagnostics + NOTE: this is user supplied, but gets modified here for constituents + constit_dict: dict + - dictionary of derived variables as keys and list of constituents as values + hist_file_ds: xarray.DataSet + - history file dataset for checking if constituents are available + hist0: str + - history number for case + + Returns + ------- + constit_list: list + - list of declared consituents from the variable defaults yaml file + - empty list: + * if missing `derived_from` argument(s) + * if `derived_from` argument(s) exist but not declared + + diag_var_list: list + - updated list (if applicable) of ADF variables for time series creation + """ + + # Aerosol Calcs + #-------------- + + # Always make sure PMID is made if aerosols are desired in config file + # Since there's no requirement for `aerosol_zonal_list`, allow it to be absent: + azl = res.get("aerosol_zonal_list", []) + if azl: + if "PMID" not in diag_var_list: + if any(item in azl for item in diag_var_list): + diag_var_list += ["PMID"] + if "T" not in diag_var_list: + if any(item in azl for item in diag_var_list): + diag_var_list += ["T"] + # End aerosol calcs + + # Set error messages for printing/debugging + # Derived variable, but missing constituent list + constit_errmsg = f"create time series for {case_name}:" + constit_errmsg += f"\n Can't create time series for {var}. \n\tThis variable" + constit_errmsg += " is flagged for derivation, but is missing list of constiuents." + constit_errmsg += "\n\tPlease add list of constituents to 'derivable_from' " + constit_errmsg += f"for {var} in variable defaults yaml file." + + # No time series creation + exit_msg = f"WARNING: {var} is not in the file {hist0} and can't be derived." + exit_msg += "\t ** No time series will be generated. **" + + # Initialiaze list for constituents + # NOTE: This is if the variable is NOT derivable but needs + # an empty list as a check later + constit_list = [] + + try_cam_constits = True + # Try finding info from variable defaults yaml file + try: + vres = res[var] + except KeyError: + print(exit_msg) + self.debug_log(exit_msg) + return diag_var_list, constit_dict + + # Check first if variable is potentially part of a CAM-CHEM run + if "derivable_from_cam_chem" in vres: + constit_list = vres["derivable_from_cam_chem"] + + if constit_list: + if all(item in hist_file_ds.data_vars for item in constit_list): + # Set check to look for regular CAM constituents in variable defaults + try_cam_constits = False + msg = f"derive time series for {case_name}:" + msg += "\n\tLooks like this a CAM-CHEM run, " + msg += f"checking constituents for '{var}'" + self.debug_log(msg) + else: + self.debug_log(constit_errmsg) + # End if + # End if + + # If not CAM-CHEM, check regular CAM runs + if try_cam_constits: + if "derivable_from" in vres: + constit_list = vres["derivable_from"] + else: + # Missing variable or missing derivable_from argument + der_from_msg = f"derive time series for {case_name}:" + der_from_msg += f"\n Can't create time series for {var}.\n\tEither " + der_from_msg += "the variable is missing from CAM output or it is a " + der_from_msg += "derived quantity and is missing the 'derivable_from' " + der_from_msg += "config argument.\n\tPlease add variable to CAM run " + der_from_msg += "or set appropriate argument in variable " + der_from_msg += "defaults yaml file." + self.debug_log(der_from_msg) + # End if + # End if + + # Log if this variable can be derived but is missing list of constituents + if isinstance(constit_list, list) and not constit_list: + self.debug_log(constit_errmsg) + + # Check if any constituents were found + if constit_list: + # Add variable and constituent list to dictionary + constit_dict[var] = constit_list + + # Add constituents to ADF diag variable list for time series generation + for constit in constit_list: + if constit not in diag_var_list: + diag_var_list.append(constit) + else: + print(exit_msg) + self.debug_log(exit_msg) + # End if + + return diag_var_list, constit_dict + +######## + +def derive_variable(self, case_name, var, res=None, ts_dir=None, + constit_list=None, overwrite=None): + """ + Derive variables acccording to steps given here. Since derivations will depend on the + variable, each variable to derive will need its own set of steps below. + + Caution: this method assumes that there will be one time series file per variable + + If the file for the derived variable exists, the kwarg `overwrite` determines + whether to overwrite the file (true) or exit with a warning message. + + """ + + # Loop through derived variables + print(f"\t - deriving time series for {var}") + + # Grab all required time series files for derived variable + constit_files = [] + for constit in constit_list: + # Check if the constituent file is present, if so add it to list + if glob.glob(os.path.join(ts_dir, f"*.{constit}.*.nc")): + constit_files.append(glob.glob(os.path.join(ts_dir, f"*.{constit}.*"))[0]) + # End for + + # Check if all the necessary constituent files were found + if len(constit_files) != len(constit_list): + ermsg = f"\t ** Not all constituent files present; {var} cannot be calculated. **\n" + ermsg += f"\t Please remove {var} from 'diag_var_list' or find the " + ermsg += "relevant CAM files.\n" + print(ermsg) + if constit_files: + # Add what's missing to debug log + dmsg = f"derived time series for {case_name}:" + dmsg += f"\n\tneeded constituents for derivation of " + dmsg += f"{var}:\n\t\t- {constit_list}\n\tfound constituent file(s) in " + dmsg += f"{Path(constit_files[0]).parent}:\n\t\t" + dmsg += f"- {[Path(f).parts[-1] for f in constit_files if Path(f).is_file()]}" + self.debug_log(dmsg) + else: + dmsg = f"derived time series for {case_name}:" + dmsg += f"\n\tneeded constituents for derivation of " + dmsg += f"{var}:\n\t\t- {constit_list}\n" + dmsg += f"\tNo constituent(s) found in history files" + self.debug_log(dmsg) + # End if + else: + # Open a new dataset with all the constituent files/variables + ds = self.data.load_dataset(constit_files) + if not ds: + dmsg = f"derived time series for {case_name}:" + dmsg += f"\n\tNo files to open." + self.debug_log(dmsg) + return + + # Grab attributes from first constituent file to be used in derived variable + attrs = ds[constit_list[0]].attrs + + # create new file name for derived variable + derived_file = constit_files[0].replace(constit_list[0], var) + + # Check if clobber is true for file + if Path(derived_file).is_file(): + if overwrite: + Path(derived_file).unlink() + else: + msg = f"[{__name__}] Warning: '{var}' file was found " + msg += "and overwrite is False. Will use existing file." + print(msg) + + #NOTE: this will need to be changed when derived equations are more complex! - JR + if var == "RESTOM": + der_val = ds["FSNT"]-ds["FLNT"] + else: + # Loop through all constituents and sum + der_val = 0 + for v in constit_list: + der_val += ds[v] + + # Set derived variable name and add to dataset + der_val.name = var + ds[var] = der_val + + # Aerosol Calculations + #---------------------------------------------------------------------------------- + # These will be multiplied by rho (density of dry air) + + # User-defined defaults might not include aerosol zonal list + azl = res.get("aerosol_zonal_list", []) + if var in azl: + # Check if PMID is in file: + ds_pmid = self.data.load_dataset(glob.glob(os.path.join(ts_dir, "*.PMID.*"))[0]) + if not ds_pmid: + errmsg = "Missing necessary files for dry air density (rho) " + errmsg += "calculation.\nPlease make sure 'PMID' is in the CAM " + errmsg += "run for aerosol calculations" + print(errmsg) + dmsg = "derived time series:" + dmsg += f"\n\t missing 'PMID' in {ts_dir}, can't make time series for {var} " + self.debug_log(dmsg) + + # Check if T is in file: + ds_t = self.data.load_dataset(glob.glob(os.path.join(ts_dir, "*.T.*"))[0]) + if not ds_t: + errmsg = "Missing necessary files for dry air density (rho) " + errmsg += "calculation.\nPlease make sure 'T' is in the CAM " + errmsg += "run for aerosol calculations" + print(errmsg) + + dmsg = "derived time series:" + dmsg += f"\n\t missing 'T' in {ts_dir}, can't make time series for {var} " + self.debug_log(dmsg) + + # Multiply aerosol by dry air density (rho): (P/Rd*T) + ds[var] = ds[var]*(ds_pmid["PMID"]/(res["Rgas"]*ds_t["T"])) + + # Sulfate conversion factor + if var == "SO4": + ds[var] = ds[var]*(96./115.) + #---------------------------------------------------------------------------------- + + # Drop all constituents from final saved dataset + # These are not necessary because they have their own time series files + ds_final = ds.drop_vars(constit_list) + # Copy attributes from constituent file to derived variable + ds_final[var].attrs = attrs + ds_final.to_netcdf(derived_file, unlimited_dims='time', mode='w') + # End if (all the necessary constituent files exist) +######## \ No newline at end of file diff --git a/lib/adf_diag.py b/lib/adf_diag.py index ddf452ecc..988296c44 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -97,6 +97,7 @@ # Finally, import needed ADF modules: from adf_web import AdfWeb from adf_dataset import AdfData +from adf_derive import check_derive, derive_variable ################# # Helper functions @@ -340,14 +341,11 @@ def call_ncrcat(cmd): It is declared as global to avoid AttributeError. """ return subprocess.run(cmd, shell=False) - # End def - # Check if baseline time-series files are being created: if baseline: - # Use baseline settings, while converting them all - # to lists: + # Use baseline settings, while converting them all to lists: case_names = [self.get_baseline_info("cam_case_name", required=True)] cam_ts_done = [self.get_baseline_info("cam_ts_done")] cam_hist_locs = [self.get_baseline_info("cam_hist_loc")] @@ -380,10 +378,14 @@ def call_ncrcat(cmd): # Loop over cases: for case_idx, case_name in enumerate(case_names): + # Notify user that script has started: + print(f"\n Generating CAM time series files for '{case_name}'...") + print(f"\n Writing time series files to {ts_dir[case_idx]}") + # Check if particular case should be processed: if cam_ts_done[case_idx]: - emsg = "\tNOTE: Configuration file indicates time series files have been pre-computed" - emsg += f" for case '{case_name}'. Will rely on those files directly." + emsg = "\tNOTE: Configuration file indicates time series files have been " + emsg += f"pre-computed for case '{case_name}'. Will rely on those files directly." print(emsg) continue # End if @@ -405,7 +407,7 @@ def call_ncrcat(cmd): # Extract time series file location ts_dir = ts_dirs[case_idx] - # Check if history files actually exqist. If not then kill script: + # Check if history files actually exist. If not then kill script: hist_str_case = hist_str_list[case_idx] for hist_str in hist_str_case: @@ -518,32 +520,18 @@ def call_ncrcat(cmd): time_string_finish = last_file_split[-1].replace("-", "") time_string = "-".join([time_string_start, time_string_finish]) - # Loop over CAM history variables: + # Intitialize list for NCO commands list_of_commands = [] list_of_ncattend_commands = [] list_of_hist_commands = [] - vars_to_derive = [] - # create copy of var list that can be modified for derivable variables + + # Create copy of var list that can be modified for derivable variables diag_var_list = self.diag_var_list - # Aerosol Calcs - # -------------- - # Always make sure PMID is made if aerosols are desired in config file - # Since there's no requirement for `aerosol_zonal_list` to be included, - # allow it to be absent: - - azl = res.get("aerosol_zonal_list", []) - if "PMID" not in diag_var_list: - if any(item in azl for item in diag_var_list): - diag_var_list += ["PMID"] - if "T" not in diag_var_list: - if any(item in azl for item in diag_var_list): - diag_var_list += ["T"] - # End aerosol calcs - - # Initialize dictionary for derived variable with needed list of constituents + # Intitialize dictionary for derived variables, if appplicable constit_dict = {} + # Loop over CAM history variables: for var in diag_var_list: # Notify user of new time series file: print(f"\t - time series for {var}") @@ -567,89 +555,18 @@ def call_ncrcat(cmd): print(msg) continue - # Set error messages for printing/debugging - # Derived variable, but missing constituent list - constit_errmsg = f"create time series for {case_name}:" - constit_errmsg += f"\n Can't create time series for {var}. \n\tThis variable" - constit_errmsg += " is flagged for derivation, but is missing list of constiuents." - constit_errmsg += "\n\tPlease add list of constituents to 'derivable_from' " - constit_errmsg += f"for {var} in variable defaults yaml file." - - # Check if current variable is a derived quantity + # Check if current variable is not in history file(s) if var not in hist_file_var_list: - vres = res.get(var, {}) - - # Initialiaze list for constituents - # NOTE: This is if the variable is NOT derivable but needs - # an empty list as a check later - constit_list = [] - - # intialize boolean to check if variable is derivable - derive = False # assume it can't be derived and update if it can - - # intialize boolean for regular CAM variable constituents - try_cam_constits = True - - # Check first if variable is potentially part of a CAM-CHEM run - if "derivable_from_cam_chem" in vres: - constit_list = vres["derivable_from_cam_chem"] - if constit_list: - if all(item in hist_file_ds.data_vars for item in constit_list): - # Set check to look for regular CAM constituents - try_cam_constits = False - derive = True - msg = f"create time series for {case_name}:" - msg += "\n\tLooks like this a CAM-CHEM run, " - msg += f"checking constituents for '{var}'" - self.debug_log(msg) - else: - self.debug_log(constit_errmsg) - # End if - # End if - - # If not CAM-CHEM, check regular CAM runs - if try_cam_constits: - if "derivable_from" in vres: - derive = True - constit_list = vres["derivable_from"] - else: - # Missing variable or missing derivable_from argument - der_from_msg = f"create time series for {case_name}:" - der_from_msg += f"\n Can't create time series for {var}.\n\tEither " - der_from_msg += "the variable is missing from CAM output or it is a " - der_from_msg += "derived quantity and is missing the 'derivable_from' " - der_from_msg += "config argument.\n\tPlease add variable to CAM run " - der_from_msg += "or set appropriate argument in variable " - der_from_msg += "defaults yaml file." - self.debug_log(der_from_msg) - # End if - - # Check if this variable can be derived - if (derive) and (constit_list): - for constit in constit_list: - if constit not in diag_var_list: - diag_var_list.append(constit) - # Add variable to list to derive - vars_to_derive.append(var) - # Add constituent list to variable key in dictionary - constit_dict[var] = constit_list - continue - # Log if variable can be derived but is missing list of constituents - elif (derive) and (not constit_list): - self.debug_log(constit_errmsg) - continue - # Lastly, raise error if the variable is not a derived quanitity - # but is also not in the history file(s) - else: - msg = f"\t WARNING: {var} is not in the history file for case '{case_name}' " - msg += "nor can it be derived. Script will continue to next variable." - print(msg) - logmsg = f"create time series for {case_name}:" - logmsg += f"\n {var} is not in the file {hist_files[0]} " - self.debug_log(logmsg) - continue - # End if - # End if (var in var_diag_list) + # Let user know variable is not + print(f"\t {var} not in history file, will try to derive if possible") + + # Check if variable can be derived + diag_var_list, constit_dict = check_derive(self, res, var, case_name, + diag_var_list, constit_dict, + hist_file_ds, hist_files[0]) + # Move to the next variable + continue + # End if # Check if variable has a "lev" dimension according to first file: has_lev = bool("lev" in hist_file_ds[var].dims or "ilev" in hist_file_ds[var].dims) @@ -696,7 +613,7 @@ def call_ncrcat(cmd): # PMID file to each one of those targets separately. -JN if "PMID" in hist_file_var_list: ncrcat_var_list = ncrcat_var_list + ",PMID" - print("Adding PMID to file") + print("\t Adding PMID to file") else: wmsg = "WARNING: PMID not found in history file." wmsg += " It might be needed at some point." @@ -712,13 +629,12 @@ def call_ncrcat(cmd): + ["-o", ts_outfil_str] ) - # Example ncatted command (you can modify it with the specific attribute changes you need) - #cmd_ncatted = ["ncatted", "-O", "-a", f"adf_user,global,a,c,{self.user}", ts_outfil_str] - # Step 1: Convert Path objects to strings and concatenate the list of historical files into a single string + # Convert Path objects to strings and concatenate the list of + # historical files into a single string hist_files_str = ', '.join(str(f.name) for f in hist_files) hist_locs_str = ', '.join(str(loc) for loc in cam_hist_locs) - # Step 2: Create the ncatted command to add both global attributes + # Create the ncatted command to add both global attributes cmd_ncatted = [ "ncatted", "-O", "-a", "adf_user,global,a,c," + f"{self.user}", @@ -727,7 +643,7 @@ def call_ncrcat(cmd): ts_outfil_str ] - # Step 3: Create the ncatted command to remove the history attribute + # Create the ncatted command to remove the history attribute cmd_remove_history = [ "ncatted", "-O", "-h", "-a", "history,global,d,,", @@ -744,28 +660,26 @@ def call_ncrcat(cmd): # NOTE: this may not be best practice, but it the history attr repeats # the files attrs so the global attrs become obtrusive... list_of_hist_commands.append(cmd_remove_history) - # End variable loop # Now run the "ncrcat" subprocesses in parallel: with mp.Pool(processes=self.num_procs) as mpool: _ = mpool.map(call_ncrcat, list_of_commands) - # End with # Run ncatted commands after ncrcat is done with mp.Pool(processes=self.num_procs) as mpool: _ = mpool.map(call_ncrcat, list_of_ncattend_commands) - # Run ncatted command to remove history attribute after the global attributes are set + # Run ncatted command to remove history attribute + # after the global attributes are set with mp.Pool(processes=self.num_procs) as mpool: _ = mpool.map(call_ncrcat, list_of_hist_commands) - if vars_to_derive: - self.derive_variables( - res=res, hist_str=hist_str, vars_to_derive=vars_to_derive, - constit_dict=constit_dict, ts_dir=ts_dir - ) - # End with + # Finally, run through the derived variables if applicable + if constit_dict: + for der_var, constit_list in constit_dict.items(): + derive_variable(self, case_name, der_var, res, + ts_dir[case_idx], constit_list) # End for hist_str # End cases loop @@ -1147,137 +1061,6 @@ def setup_run_cvdp(self): ######### - def derive_variables(self, res=None, hist_str=None, vars_to_derive=None, ts_dir=None, - constit_dict=None, overwrite=None): - """ - Derive variables acccording to steps given here. Since derivations will depend on the - variable, each variable to derive will need its own set of steps below. - - Caution: this method assumes that there will be one time series file per variable - - If the file for the derived variable exists, the kwarg `overwrite` determines - whether to overwrite the file (true) or exit with a warning message. - - """ - - # Loop through derived variables - for var in vars_to_derive: - print(f"\t - deriving time series for {var}") - - # Grab list of constituents for this variable - constit_list = constit_dict[var] - - # Grab all required time series files for derived variable - constit_files = [] - for constit in constit_list: - # Check if the constituent file is present, if so add it to list - if hist_str: - const_glob_str = f"*{hist_str}*.{constit}.*.nc" - else: - const_glob_str = f"*.{constit}.*.nc" - # end if - if glob.glob(os.path.join(ts_dir, const_glob_str)): - constit_files.append(glob.glob(os.path.join(ts_dir, const_glob_str ))[0]) - - # Check if all the necessary constituent files were found - if len(constit_files) != len(constit_list): - ermsg = f"\t WARNING: Not all constituent files present; {var} cannot be calculated." - ermsg += f" Please remove {var} from 'diag_var_list' or find the " - ermsg += "relevant CAM files.\n" - print(ermsg) - if constit_files: - # Add what's missing to debug log - dmsg = "create time series:" - dmsg += "\n\tneeded constituents for derivation of " - dmsg += f"{var}:\n\t\t- {constit_list}\n\tfound constituent file(s) in " - dmsg += f"{Path(constit_files[0]).parent}:\n\t\t" - dmsg += f"- {[Path(f).parts[-1] for f in constit_files if Path(f).is_file()]}" - self.debug_log(dmsg) - else: - dmsg = "create time series:" - dmsg += "\n\tneeded constituents for derivation of " - dmsg += f"{var}:\n\t\t- {constit_list}\n" - dmsg += "\tNo constituent(s) found in history files" - self.debug_log(dmsg) - - else: - # Open a new dataset with all the constituent files/variables - ds = xr.open_mfdataset(constit_files).compute() - - # Grab attributes from first constituent file to be used in derived variable - attrs = ds[constit_list[0]].attrs - - # create new file name for derived variable - derived_file = constit_files[0].replace(constit_list[0], var) - - # Check if clobber is true for file - if Path(derived_file).is_file(): - if overwrite: - Path(derived_file).unlink() - else: - msg = f"\t INFO: '{var}' file was found " - msg += "and overwrite is False. Will use existing file." - print(msg) - continue - - # NOTE: this will need to be changed when derived equations are more complex! - JR - if var == "RESTOM": - der_val = ds["FSNT"]-ds["FLNT"] - else: - # Loop through all constituents and sum - der_val = 0 - for v in constit_list: - der_val += ds[v] - - # Set derived variable name and add to dataset - der_val.name = var - ds[var] = der_val - - # Aerosol Calculations - # ---------------------------------------------------------------------------------- - # These will be multiplied by rho (density of dry air) - ds_pmid_done = False - ds_t_done = False - - # User-defined defaults might not include aerosol zonal list - azl = res.get("aerosol_zonal_list", []) - if var in azl: - # Only calculate once for all aerosol vars - if not ds_pmid_done: - ds_pmid = _load_dataset(glob.glob(os.path.join(ts_dir, "*.PMID.*"))[0]) - ds_pmid_done = True - if not ds_pmid: - errmsg = "\t WARNING: Missing necessary files for dry air density" - errmsg += " (rho) calculation.\n" - errmsg += "\t Please make sure 'PMID' is in the CAM run" - errmsg += " for aerosol calculations" - print(errmsg) - continue - if not ds_t_done: - ds_t = _load_dataset(glob.glob(os.path.join(ts_dir, "*.T.*"))[0]) - ds_t_done = True - if not ds_t: - errmsg = "\t WARNING: Missing necessary files for dry air density" - errmsg += " (rho) calculation.\n" - errmsg += "\t Please make sure 'T' is in the CAM run" - errmsg += " for aerosol calculations" - print(errmsg) - continue - - # Multiply aerosol by dry air density (rho): (P/Rd*T) - ds[var] = ds[var]*(ds_pmid["PMID"]/(res["Rgas"]*ds_t["T"])) - - # Sulfate conversion factor - if var == "SO4": - ds[var] = ds[var]*(96./115.) - # ---------------------------------------------------------------------------------- - - # Drop all constituents from final saved dataset - # These are not necessary because they have their own time series files - ds_final = ds.drop_vars(constit_list) - # Copy attributes from constituent file to derived variable - ds_final[var].attrs = attrs - ds_final.to_netcdf(derived_file, unlimited_dims='time', mode='w') ######### MDTF functions ######### def setup_run_mdtf(self): @@ -1299,8 +1082,8 @@ def setup_run_mdtf(self): # # Create a dict with all the case info needed for MDTF case_list - # Note that model and convention are hard-coded to CESM because that's all we expect here - # This could be changed by inputing them into ADF with other MDTF-specific variables + # Note that model and convention are hard-coded to CESM because that's all we expect here + # - This could be changed by inputing them into ADF with other MDTF-specific variables # case_list_keys = ["CASENAME", "FIRSTYR", "LASTYR", "model", "convention"] @@ -1356,7 +1139,9 @@ def setup_run_mdtf(self): # # Submit the MDTF script in background mode, send output to mdtf.out file # - mdtf_log = "mdtf.out" # maybe set this to cam_diag_plot_loc: /glade/scratch/${user}/ADF/plots + mdtf_log = "mdtf.out" + # maybe set this to cam_diag_plot_loc: /glade/scratch/${user}/ADF/plots + mdtf_exe = mdtf_codebase + os.sep + "mdtf -f " + mdtf_input_settings_filename if copy_files_only: print("\t ...Copy files only. NOT Running MDTF") @@ -1425,17 +1210,21 @@ def move_tsfiles_for_mdtf(self, verbose): elif len(adf_file_list) > 1: if verbose > 0: print( - f"WARNING: found multiple timeseries files {adf_file_list}. Continuing with best guess; suggest cleaning up multiple dates in ts dir" + f"""WARNING: found multiple timeseries files {adf_file_list}. + Continuing with best guess; suggest cleaning up multiple + dates in ts dir""" ) else: if verbose > 1: print( - f"WARNING: No files matching {case_name}.{hist_str}.{var} found in {adf_file_str}. Skipping" + f"""WARNING: No files matching {case_name}.{hist_str}.{var} + found in {adf_file_str}. Skipping""" ) continue # skip this case/hist_str/var file adf_file = adf_file_list[0] - # If freq is not set, it means we just started this hist_str. So check the first ADF file to find it + # If freq is not set, it means we just started this hist_str. + # So check the first ADF file to find it hist_file_ds = xr.open_dataset( adf_file, decode_cf=False, decode_times=False ) @@ -1446,7 +1235,8 @@ def move_tsfiles_for_mdtf(self, verbose): else: if verbose > 0: print( - f"WARNING: Necessary 'time_period_freq' attribute missing from {adf_file}. Skipping file." + f"""WARNING: Necessary 'time_period_freq' attribute missing + from {adf_file}. Skipping file.""" ) continue @@ -1461,12 +1251,14 @@ def move_tsfiles_for_mdtf(self, verbose): elif len(found_strings) > 1: if verbose > 0: print( - f"WARNING: Found dataset_freq {dataset_freq} matches multiple string possibilities:{', '.join(found_strings)}" + f"""WARNING: Found dataset_freq {dataset_freq} matches multiple + string possibilities:{', '.join(found_strings)}""" ) else: if verbose > 0: print( - f"WARNING: None of the frequency options {freq_string_cesm} are present in the time_period_freq attribute {dataset_freq}" + f"""WARNING: None of the frequency options {freq_string_cesm} are + present in the time_period_freq attribute {dataset_freq}""" ) print(f"Skipping {adf_file}") freq = "frequency_missing" @@ -1502,43 +1294,3 @@ def move_tsfiles_for_mdtf(self, verbose): # end for hist_str # end for var # end for case - - -######## - -# Helper Function(s) - - -def _load_dataset(fils): - """ - This method exists to get an xarray Dataset from input file information that - can be passed into the plotting methods. - - Parameters - ---------- - fils : list - strings or paths to input file(s) - - Returns - ------- - xr.Dataset - - Notes - ----- - When just one entry is provided, use `open_dataset`, otherwise `open_mfdatset` - """ - - import adf_utils as utils - import warnings # use to warn user about missing files - warnings.formatwarning = utils.my_formatwarning - - if len(fils) == 0: - warnings.warn("\t WARNING: Input file list is empty.") - return None - if len(fils) > 1: - return xr.open_mfdataset(fils, combine='by_coords') - else: - return xr.open_dataset(fils[0]) - #End if -# End def -######## diff --git a/lib/adf_web.py b/lib/adf_web.py index b5d8e77ed..6704b3517 100644 --- a/lib/adf_web.py +++ b/lib/adf_web.py @@ -866,7 +866,6 @@ def jinja_enumerate(arg): with open(index_html_file, 'w', encoding='utf-8') as ofil: ofil.write(index_rndr) #End with - #End for (web data loop) #If this is a multi-case instance, then copy website to "main" directory: