From 01486192b49167e9d5611482319d97c1b037e4a3 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Tue, 7 Feb 2023 12:00:03 -0700 Subject: [PATCH 01/12] Enhance multiple case diagnostics Add multi-case diagnostics and webpages: - Add multi-case plot section to config yaml - Update all website templates (add new for multi-case) - Update web generation - Add multi-case subplots and all case comp tables - Designate hub for all test cases individual vs baseline ADF analysis --- config_cam_baseline_example.yaml | 23 + lib/adf_diag.py | 13 + lib/adf_web.py | 495 +++++++++++++++--- lib/plotting_functions.py | 128 +++++ lib/website_templates/template.html | 15 +- lib/website_templates/template_index.html | 5 +- lib/website_templates/template_mean_diag.html | 10 +- .../template_mean_tables.html | 37 +- .../template_multi_case.html | 84 +++ .../template_multi_case_index.html | 63 ++- .../template_multi_case_mean_diag.html | 69 +++ .../template_multi_case_var.html | 74 +++ lib/website_templates/template_table.html | 39 +- lib/website_templates/template_var.html | 13 +- scripts/analysis/amwg_table.py | 105 +++- scripts/plotting/global_latlon_map.py | 75 ++- 16 files changed, 1098 insertions(+), 150 deletions(-) create mode 100644 lib/website_templates/template_multi_case.html create mode 100644 lib/website_templates/template_multi_case_mean_diag.html create mode 100644 lib/website_templates/template_multi_case_var.html diff --git a/config_cam_baseline_example.yaml b/config_cam_baseline_example.yaml index b42435794..75c042eac 100644 --- a/config_cam_baseline_example.yaml +++ b/config_cam_baseline_example.yaml @@ -151,6 +151,11 @@ diag_cam_climo: #Name of CAM case (or CAM run name): cam_case_name: b.e20.BHIST.f09_g17.20thC.297_05 + #Case nickname + #If left blank, it will default to cam_case_name + # ** NOTE: if nickname starts with a zero, the string must be in quotes! ** + case_nickname: ${diag_cam_climo.cam_case_name} + #Location of CAM history (h0) files: #Example test files cam_hist_loc: /glade/p/cesm/ADF/${diag_cam_climo.cam_case_name} @@ -201,6 +206,10 @@ diag_cam_climo: # - b.e20.BHIST.f09_g17.20thC.297_05 # - b1850.f19_g17.validation_mct.004 + #case_nickname: + # - nickname 1 + # - "026c" + #calc_cam_climo: # - true # - true @@ -259,6 +268,11 @@ diag_cam_baseline_climo: #Name of CAM baseline case: cam_case_name: b.e20.BHIST.f09_g16.20thC.125.02 + #Case nickname + #If left blank, it will default to baseline cam_case_name + # ** NOTE: if nickname starts with a zero, the string must be in quotes! ** + case_nickname: ${diag_cam_baseline_climo.cam_case_name} + #Location of CAM baseline history (h0) files: #Example test files cam_hist_loc: /glade/p/cesm/ADF/${diag_cam_baseline_climo.cam_case_name} @@ -377,6 +391,15 @@ diag_var_list: # +# Make mulit-plot comparison image for specified plots types +# This will only run if there are more than one specified test cases declared above +# NOTE: These plot types also need to be present above in the plotting_scripts section (for now) and variables in the diag_var_list +# NOTE: Only global LatLon multi-case plots are available at this time - more are coming. JR +multi_case_plots: + global_latlon_map: + - SST + - PSL + # Options for multi-case regional contour plots (./plotting/regional_map_multicase.py) # region_multicase: # region_spec: [slat, nlat, wlon, elon] diff --git a/lib/adf_diag.py b/lib/adf_diag.py index a539219f4..da300835b 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -169,6 +169,9 @@ def __init__(self, config_file, debug=False): self.expand_references(self.__cvdp_info) #End if + #Add multi plots info to object: + self.__multi_case_plots = self.read_config_var('multi_case_plots') + #Add averaging script names: self.__time_averaging_scripts = self.read_config_var('time_averaging_scripts') @@ -197,6 +200,16 @@ def get_cvdp_info(self, var_str, required=False): conf_dict=self.__cvdp_info, required=required) + def get_multi_case_info(self, var_str, required=False): + """ + Return the config variable from 'multi_case_plots' as requested by + the user. + """ + + return self.read_config_var(var_str, + conf_dict=self.__multi_case_plots, + required=required) + ######### #Script-running functions ######### diff --git a/lib/adf_web.py b/lib/adf_web.py index d15c2a8b3..5f1acb784 100644 --- a/lib/adf_web.py +++ b/lib/adf_web.py @@ -24,6 +24,7 @@ import os.path from pathlib import Path +from collections import OrderedDict #+++++++++++++++++++++++++++++++++++++++++++++++++ #import non-standard python modules, including ADF @@ -56,6 +57,7 @@ class _WebData: """ def __init__(self, web_data, web_name, case_name, + plot_ext = None, category = None, season = None, plot_type = "Special", @@ -71,6 +73,7 @@ def __init__(self, web_data, web_name, case_name, self.category = category self.season = season self.plot_type = plot_type + self.plot_ext = plot_ext self.data_frame = data_frame self.html_file = html_file self.asset_path = asset_path @@ -179,7 +182,8 @@ def create_html(self): ######### - def add_website_data(self, web_data, web_name, case_name, + def add_website_data(self, web_data, web_name, case_name, + plot_ext = None, category = None, season = None, plot_type = "Special", @@ -217,6 +221,7 @@ def add_website_data(self, web_data, web_name, case_name, #Initialize Pandas data frame logical: data_frame = False + html_file = [] #Check that the web_data is either a path #or a pandas dataframe: try: @@ -271,7 +276,11 @@ def add_website_data(self, web_data, web_name, case_name, #If multi-case, then save under the "multi-case" directory: if self.num_cases > 1: - html_file = self.__case_web_paths['multi-case']["table_pages_dir"] / html_name + #Avoid collecting individual case comparison tables + #Hacky - could probably use an update eventually - JR + if web_name != "case_comparison": + html_file.append(self.__case_web_paths['multi-case']["table_pages_dir"] / html_name) + html_file.append(self.__case_web_paths[case_name]["table_pages_dir"] / html_name) else: html_file = self.__case_web_paths[case_name]["table_pages_dir"] / html_name #End if @@ -283,7 +292,7 @@ def add_website_data(self, web_data, web_name, case_name, #End if #Initialize web data object: - web_data = _WebData(web_data, web_name, case_name, + web_data = _WebData(web_data, web_name, case_name, plot_ext, category = category, season = season, plot_type = plot_type, @@ -300,6 +309,9 @@ def add_website_data(self, web_data, web_name, case_name, if plot_type not in self.__plot_type_multi: self.__plot_type_multi.append(plot_type) #End if + if plot_type not in self.__plot_type_order: + self.__plot_type_order.append(plot_type) + #End if else: #single case plot/ADF run if plot_type not in self.__plot_type_order: self.__plot_type_order.append(plot_type) @@ -332,12 +344,18 @@ def create_website(self): #If there is more than one non-baseline case, then create new website directory: if self.num_cases > 1: - main_site_path = Path(self.get_basic_info('cam_diag_plot_loc', required=True)) - main_site_path = main_site_path / "main_website" + multi_path = Path(self.get_basic_info('cam_diag_plot_loc', required=True)) + main_site_path = multi_path / "main_website" main_site_path.mkdir(exist_ok=True) + main_site_img_path = main_site_path / "html_img" + main_site_img_path.mkdir(exist_ok=True) + main_site_assets_path = main_site_path / "assets" + main_site_assets_path.mkdir(exist_ok=True) case_sites = OrderedDict() + multi_layout = True else: main_site_path = "" #Set main_site_path to blank value + multi_layout = False #End if #Extract needed variables from yaml file: @@ -393,6 +411,9 @@ def create_website(self): #Extract variable defaults dictionary (for categories): var_defaults_dict = self.variable_defaults + # Dict for multi case if specified + multi_case_plots = self.read_config_var('multi_case_plots') + #Set plot type html dictionary (for Jinja templating): plot_type_html = OrderedDict() for plot_type in self.__plot_type_order: @@ -443,9 +464,30 @@ def create_website(self): #so that we only had to do the web_data loop once, #but for now this will do. -JN mean_html_info = OrderedDict() + multi_mean_html_info = OrderedDict() #Create another dictionary needed for HTML pages that render tables: table_html_info = OrderedDict() + multi_table_html_info = OrderedDict() + + #If this is a multi-case instance, then copy website to "main" directory: + if main_site_path: + #Create CSS templates file path: + main_templates_path = main_site_path / "templates" + + #Also add path to case_sites dictionary: + #loop over cases: + for idx, case_name in enumerate(case_names): + #Check if case name is present in plot + if case_name in self.__case_web_paths: + #Add path to case_sites dictionary: + case_sites[case_name] = [os.path.join(os.curdir, + f"{case_name}_{syear_cases[idx]}_{eyear_cases[idx]}_vs_{data_name}_{syear_baseline}_{eyear_baseline}", + "index.html"),syear_cases[idx],eyear_cases[idx]] + + else: + #make empty list for non multi-case web generation + case_sites = [] #Loop over all web data objects: for web_data in self.__website_data: @@ -461,26 +503,64 @@ def create_website(self): for css_file in jinja_template_dir.glob('*.css'): shutil.copyfile(css_file, css_files_dir / css_file.name) #End for - + #Copy GIF files over to output directory as well: for gif_file in jinja_template_dir.glob('*.gif'): shutil.copyfile(gif_file, css_files_dir / gif_file.name) #End for + #Check first for AMWG tables data frame if web_data.data_frame: #Create a directory that will hold table html files, if a table is present: if self.num_cases > 1: self.__case_web_paths['multi-case']['table_pages_dir'].mkdir(exist_ok=True) - else: - self.__case_web_paths[web_data.case]['table_pages_dir'].mkdir(exist_ok=True) #End if + self.__case_web_paths[web_data.case]['table_pages_dir'].mkdir(exist_ok=True) + #Add table HTML file to dictionary: #Note: Need to use data name instead of case name for tables. - table_html_info[web_data.name] = web_data.html_file.name + if len(case_names) > 1: + if web_data.name != "case_comparison": + table_html_info[web_data.name] = web_data.html_file[0].name + multi_table_html_info[web_data.name] = str(web_data.html_file[0].name).replace("main_website",f"{web_data.name}/website") + else: + table_html_info[web_data.name] = web_data.html_file.name + + + #Now check all plot types if not web_data.data_frame: + #Check to see if there are multiple-cases + if main_site_path: + #check to see if the user has multi-plots enabled + if multi_case_plots: + if web_data.name in [item for sublist in [multi_case_plots[x] for x in multi_case_plots] for item in sublist]: + season = web_data.season + category = web_data.category + ptype = web_data.plot_type + + if web_data.plot_ext in multi_case_plots.keys(): + for var in multi_case_plots[web_data.plot_ext]: + #Initialize Ordered Dictionary for multi case plot type: + if ptype not in multi_mean_html_info: + multi_mean_html_info[ptype] = OrderedDict() + #End if + + if category not in multi_mean_html_info[ptype]: + multi_mean_html_info[ptype][category] = OrderedDict() + #End if + + #Initialize Ordered Dictionary for variable: + if var not in multi_mean_html_info[ptype][category]: + multi_mean_html_info[ptype][category][var] = OrderedDict() + #End if + + if season not in multi_mean_html_info[ptype][category][var]: + multi_mean_html_info[ptype][category][var][season] = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" + #End if + #Create a directory that will hold just the html files for individual images: self.__case_web_paths[web_data.case]['img_pages_dir'].mkdir(exist_ok=True) @@ -541,17 +621,17 @@ def create_website(self): #End for (web_data list loop) #Loop over all web data objects again: - for web_data in self.__website_data: - + for idx,web_data in enumerate(self.__website_data): if web_data.data_frame: - #Create output HTML file path: if self.num_cases > 1: table_pages_dir = self.__case_web_paths['multi-case']['table_pages_dir'] - plot_types = multi_plot_type_html + table_pages_dir_indv = self.__case_web_paths[web_data.case]['table_pages_dir'] + else: table_pages_dir = self.__case_web_paths[web_data.case]['table_pages_dir'] - plot_types = plot_type_html + + plot_types = plot_type_html #End if #Check if plot image already handles multiple cases, @@ -570,23 +650,52 @@ def create_website(self): float_format='{:6g}'.format) #Construct amwg_table.html - table_tmpl = jinenv.get_template('template_table.html') - table_rndr = table_tmpl.render(title=main_title, - case1=case1, - case2=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - amwg_tables=table_html_info, - plot_types=plot_types, - table_name=web_data.name, - table_html=table_html - ) - - #Write mean diagnostic tables HTML file: - with open(web_data.html_file, 'w', encoding='utf-8') as ofil: - ofil.write(table_rndr) - #End with + if main_site_path: + if web_data.name != "case_comparison": + table_tmpl = jinenv.get_template('template_table.html') + table_rndr = table_tmpl.render(title=main_title, + case1=case1, + case2=data_name, + case_yrs=case_yrs, + base_name=data_name, + baseline_yrs=baseline_yrs, + amwg_tables=table_html_info, + plot_types=plot_types, + table_name=web_data.name, + table_html=table_html, + multi_head=False, + multi=multi_layout, + case_sites=case_sites, + ) + + #Write mean diagnostic tables HTML file: + html_file = web_data.html_file[0] + with open(html_file, 'w', encoding='utf-8') as ofil: + ofil.write(table_rndr) + + else: + table_tmpl = jinenv.get_template('template_table.html') + table_rndr = table_tmpl.render(title=main_title, + case1=case1, + case2=data_name, + case_yrs=case_yrs, + base_name=data_name, + baseline_yrs=baseline_yrs, + amwg_tables=table_html_info, + plot_types=plot_types, + table_name=web_data.name, + table_html=table_html, + multi_head=False, + multi=multi_layout, + case_sites=case_sites, + ) + + #Write mean diagnostic tables HTML file: + html_file = web_data.html_file + with open(html_file, 'w', encoding='utf-8') as ofil: + ofil.write(table_rndr) + #Check if the mean plot type page exists for this case (or for multi-case): mean_table_file = table_pages_dir / "mean_tables.html" if not mean_table_file.exists(): @@ -597,9 +706,13 @@ def create_website(self): case1=case1, case2=data_name, case_yrs=case_yrs, + base_name=data_name, baseline_yrs=baseline_yrs, amwg_tables=table_html_info, plot_types=plot_types, + multi_head=False, + multi=multi_layout, + case_sites=case_sites, ) #Write mean diagnostic tables HTML file: @@ -607,7 +720,7 @@ def create_website(self): ofil.write(mean_table_rndr) #End with #End if - + #End if (web_data.data_frame) else: #Plot image #Create output HTML file path: @@ -635,7 +748,8 @@ def create_website(self): case_yrs=case_yrs, baseline_yrs=baseline_yrs, mydata=mean_html_info[web_data.plot_type], - plot_types=plot_types) #The template rendered + plot_types=plot_types, + multi=multi_layout,) #The template rendered #Write HTML file: with open(web_data.html_file, 'w', encoding='utf-8') as ofil: @@ -656,7 +770,8 @@ def create_website(self): baseline_yrs=baseline_yrs, mydata=mean_html_info[web_data.plot_type], curr_type=web_data.plot_type, - plot_types=plot_types) + plot_types=plot_types, + multi=multi_layout,) #Write mean diagnostic plots HTML file: with open(mean_ptype_file,'w', encoding='utf-8') as ofil: @@ -681,7 +796,8 @@ def create_website(self): baseline_yrs=baseline_yrs, mydata=mean_html_info[web_data.plot_type], curr_type=web_data.plot_type, - plot_types=plot_types) + plot_types=plot_types, + multi=multi_layout,) #Write mean diagnostic plots HTML file: with open(mean_ptype_plot_page,'w', encoding='utf-8') as ofil: @@ -698,6 +814,7 @@ def create_website(self): plot_types = multi_plot_type_html else: plot_types = plot_type_html + plot_types = plot_type_html #End if #Construct index.html @@ -708,7 +825,8 @@ def create_website(self): case2=data_name, case_yrs=case_yrs, baseline_yrs=baseline_yrs, - plot_types=plot_types) + plot_types=plot_types, #plot_type_html + multi=multi_layout,) #Write Mean diagnostics index HTML file: with open(index_html_file, 'w', encoding='utf-8') as ofil: @@ -717,52 +835,303 @@ def create_website(self): #End for (web data loop) - #If this is a multi-case instance, then copy website to "main" directory: + # --- Starting multi-case plots if activated --- + # - - - - - - - - - - - - - - - - - - - - - - - - if main_site_path: - #Add "multi-case" to start of case_names: - case_names.insert(0, "multi-case") - - #Create CSS templates file path: - main_templates_path = main_site_path / "templates" - - #loop over cases: - for case_name in case_names: - - #Check if case name is present in plot - if case_name in self.__case_web_paths: - #Extract website directory: - website_dir = self.__case_web_paths[case_name]['website_dir'] - - #Copy website directory to "main site" directory: - shutil.copytree(website_dir, main_site_path / case_name) + #Loop over all web data objects AGAIN: + for web_data in self.__website_data: + + #Create CSS templates file path: + main_templates_path = main_site_path / "templates" + + #loop over cases: + for idx, case_name in enumerate(case_names): + #Check if case name is present in plot + if case_name in self.__case_web_paths: + #Extract website directory: + website_dir = self.__case_web_paths[case_name]['website_dir'] + if not website_dir.is_dir(): + #Copy website directory to "main site" directory: + shutil.copytree(website_dir, main_site_path / case_name) + + + #Multi Case Plots (LatLon for now) + ################################## + if multi_case_plots: + var = web_data.name + if var in [item for sublist in [multi_case_plots[x] for x in multi_case_plots] for item in sublist]: + #Check if the web data obj is table or not (plots) + if not web_data.data_frame: + + season = web_data.season + #Extract plot_type: + ptype = web_data.plot_type + + #Create a directory that will hold just the html files for individual images: + self.__case_web_paths[web_data.case]['img_pages_dir'].mkdir(exist_ok=True) + + #Create a directory that will hold copies of the actual images: + self.__case_web_paths[web_data.case]['assets_dir'].mkdir(exist_ok=True) + + #Move file to assets directory: + shutil.copy(web_data.data, web_data.asset_path) + + #Check if category has been provided for this web data: + if web_data.category: + #If so, then just use directly: + category = web_data.category + else: + + #Check if variable in defaults dictionary: + if web_data.name in var_defaults_dict: + #If so, then extract category from dictionary: + category = var_defaults_dict[web_data.name].get("category", + "No category yet") + else: + category = 'No category yet' + #End if + #End if + + #Create output HTML file path: + img_pages_dir = self.__case_web_paths["multi-case"]['img_pages_dir'] + + img_data = [os.path.relpath(main_site_assets_path / f"{var}_{season}_{ptype}_multi_plot.png", start=main_site_img_path), + f"{var}_{season}_{ptype}_multi_plot.png"] + + if not (img_pages_dir / Path(f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html")).exists(): + tmpl = jinenv.get_template('template_multi_case.html') #Set template + rndr = tmpl.render(title=main_title, + var_title=var, + season_title=season, + plottype_title=web_data.plot_type, + imgs=img_data, + base_name=data_name, + case_yrs=case_yrs, + baseline_yrs=baseline_yrs, + mydata=multi_mean_html_info[ptype], + plot_types=multi_plot_type_html, + multi=multi_layout, + case_sites=case_sites,) #The template rendered + + #Write HTML file: + with open(img_pages_dir / f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html", 'w', encoding='utf-8') as ofil: + ofil.write(rndr) + + + mean_ptype_file = main_site_img_path / f"multi_case_mean_diag_{web_data.plot_type}.html" + if not mean_ptype_file.exists(): + + #Construct individual plot type mean_diag html files, if they don't + #already exist: + mean_tmpl = jinenv.get_template('template_multi_case_mean_diag.html') + mean_rndr = mean_tmpl.render(title=main_title, + base_name=data_name, + case_yrs=case_yrs, + baseline_yrs=baseline_yrs, + mydata=multi_mean_html_info[ptype], + curr_type=web_data.plot_type, + plot_types=multi_plot_type_html, + multi=multi_layout, + case_sites=case_sites,) + + #Write mean diagnostic plots HTML file: + with open(mean_ptype_file,'w', encoding='utf-8') as ofil: + ofil.write(mean_rndr) + #End with + #End if (mean_ptype exists) + + #Check if the mean plot type and var page exists for this case: + self.__case_web_paths["multi-case"]['img_pages_dir'].mkdir(exist_ok=True) + img_pages_dir = self.__case_web_paths["multi-case"]['img_pages_dir'] + mean_ptype_plot_page = img_pages_dir / f"plot_page_multi_case_{var}_{web_data.plot_type}.html" + + if not mean_ptype_plot_page.exists(): + + #Construct individual plot type mean_diag html files, if they don't + #already exist: + plot_page_tmpl = jinenv.get_template('template_multi_case_var.html') + plot_page_rndr = plot_page_tmpl.render(title=main_title, + var_title=var, + season_title=season, + plottype_title=web_data.plot_type, + base_name=data_name, + case_yrs=case_yrs, + baseline_yrs=baseline_yrs, + mydata=multi_mean_html_info[ptype], + curr_type=web_data.plot_type, + plot_types=multi_plot_type_html, + multi=multi_layout, + case_sites=case_sites, + ) + + #Write mean diagnostic plots HTML file: + with open(mean_ptype_plot_page,'w', encoding='utf-8') as ofil: + ofil.write(plot_page_rndr) + #End with + #End if (mean_ptype_plot_page exists) + + #End if (not web_data.data_frame) + #End if + + #Create all individual tables + # for the individual websites + ############################# + if web_data.data_frame: + table_pages_dir_indv = self.__case_web_paths[web_data.case]['table_pages_dir'] + + #Check if the mean plot type page exists for this case (or for multi-case): + mean_table_file = table_pages_dir_indv / "mean_tables.html" + table_keys = [web_data.case,data_name,"case_comparison"] + table_dict = {key: multi_table_html_info[key] for key in table_keys} + + if not mean_table_file.exists(): + #Construct mean_table.html + mean_table_tmpl = jinenv.get_template('template_mean_tables.html') + mean_table_rndr = mean_table_tmpl.render(title=main_title, + case1=web_data.case, + case2=data_name, + case_yrs=case_yrs, + base_name=data_name, + baseline_yrs=baseline_yrs, + amwg_tables=table_dict, + plot_types=plot_type_html, + multi_head=True, + multi=False, + case_sites=case_sites, + ) + + #Write mean diagnostic tables HTML file: + with open(mean_table_file, 'w', encoding='utf-8') as ofil: + ofil.write(mean_table_rndr) + #End with + + #if web_data.case in case_names: + if web_data.case != data_name: + table_html = web_data.data.to_html(index=False, border=1, justify='center', + float_format='{:6g}'.format) + + #Construct amwg_table.html + case_table_keys = [web_data.case,data_name,"case_comparison"] + case_table_dict = {key: multi_table_html_info[key] for key in case_table_keys} + indv_html = table_pages_dir_indv / f"amwg_table_{web_data.name}.html" + + if not indv_html.exists(): + table_tmpl = jinenv.get_template('template_table.html') + table_rndr = table_tmpl.render(title=main_title, + case1=web_data.case, + case2=data_name, + case_yrs=case_yrs, + base_name=data_name, + baseline_yrs=baseline_yrs, + amwg_tables=case_table_dict, + plot_types=plot_type_html, + table_name=web_data.name, + table_html=table_html, + multi_head=True, + multi=False, + case_sites=case_sites, + ) - #Also add path to case_sites dictionary: - case_sites[case_name] = os.path.join(os.curdir, case_name, "index.html") + #Write mean diagnostic tables HTML file: + with open(indv_html, 'w', encoding='utf-8') as ofil: + ofil.write(table_rndr) - #Also make sure CSS template files have been copied over: - if not main_templates_path.is_dir(): - css_files_dir = self.__case_web_paths[case_name]['css_files_dir'] - shutil.copytree(css_files_dir, main_site_path / "templates") + #Baseline case added to all test case directories + # - this block should only run once when web_data is the baseline case + else: + table_html = web_data.data.to_html(index=False, border=1, justify='center', + float_format='{:6g}'.format) + + for case_name in case_names: + table_pages_dir_sp = self.__case_web_paths[case_name]['table_pages_dir'] + base_table_keys = [case_name,data_name,"case_comparison"] + base_table_dict = {key: multi_table_html_info[key] for key in base_table_keys} + + sp_html = table_pages_dir_sp / f"amwg_table_{data_name}.html" + if not sp_html.exists(): + table_tmpl = jinenv.get_template('template_table.html') + table_rndr = table_tmpl.render(title=main_title, + case1=case_name, + case2=data_name, + case_yrs=case_yrs, + base_name=data_name, + baseline_yrs=baseline_yrs, + amwg_tables=base_table_dict, + plot_types=plot_type_html, + table_name=web_data.name, + table_html=table_html, + multi_head=True, + multi=False, + case_sites=case_sites, + ) + + with open(sp_html, 'w', encoding='utf-8') as ofil: + ofil.write(table_rndr) + + #Check if the mean plot type page exists for this case: + mean_table_file = table_pages_dir_indv / "mean_tables.html" + if not mean_table_file.exists(): + #Construct mean_table.html + mean_table_tmpl = jinenv.get_template('template_mean_tables.html') + mean_table_rndr = mean_table_tmpl.render(title=main_title, + case1=web_data.case, + case2=data_name, + case_yrs=case_yrs, + base_name=data_name, + baseline_yrs=baseline_yrs, + amwg_tables=table_dict, + plot_types=plot_type_html, + multi_head=True, + multi=False, + case_sites=case_sites, + ) + + #Write mean diagnostic tables HTML file: + with open(mean_table_file, 'w', encoding='utf-8') as ofil: + ofil.write(mean_table_rndr) + #End with #End if - #End if + #End if (web_data.data_frame) #End for (model case loop) + #Also make sure CSS template files have been copied over: + if not main_templates_path.is_dir(): + css_files_dir = self.__case_web_paths[case_names[-1]]['css_files_dir'] + shutil.copytree(css_files_dir, main_templates_path) + #End if + #Create multi-case site: + multi_case_dict = {"global_latlon_map":"LatLon", + "zonal_mean":"Zonal", + "meridional":"Meridional", + "global_latlon_vect_map":"LatLon_Vector", + #"polar_maps":"", + } + + multi_plots = {"Tables": "html_table/mean_tables.html",} + for key,_ in multi_case_plots.items(): + #Update the dictionary to add any plot types specified in the yaml file + multi_plots[multi_case_dict[key]] = f"html_img/multi_case_mean_diag_{multi_case_dict[key]}.html" + main_title = "ADF Diagnostics" main_tmpl = jinenv.get_template('template_multi_case_index.html') main_rndr = main_tmpl.render(title=main_title, - case_sites=case_sites, - ) + case_sites=case_sites, + base_name=data_name, + baseline_yrs=baseline_yrs, + multi_plots=multi_plots, + ) #Write multi-case main HTML file: outputfile = main_site_path / "index.html" with open(outputfile, 'w', encoding='utf-8') as ofil: ofil.write(main_rndr) #End with - #End if + #End if (multi case) #Notify user that script has finishedd: print(" ...Webpages have been generated successfully.") + #++++++++++++++++++++ #End Class definition -#++++++++++++++++++++ +#++++++++++++++++++++ \ No newline at end of file diff --git a/lib/plotting_functions.py b/lib/plotting_functions.py index 42ae53347..26a2ce6f1 100644 --- a/lib/plotting_functions.py +++ b/lib/plotting_functions.py @@ -1635,5 +1635,133 @@ def square_contour_difference(fld1, fld2, **kwargs): cb2 = fig.colorbar(img3, cax=cbax_bot, orientation='horizontal') return fig +##### + +############################### +# Multi-Case Multi-Plot Section +############################### + +def multi_latlon_plots(wks, ptype, case_names, nicknames, multi_dict, adfobj): + """ This is a multi-case comparison of test minus baseline for each test case: + wks: path for saved image. + Should be assets directory inside of the main_website directory + + ptype: ADF shortname for plot type. + For this plot it will be either LatLon or LatLon_Vector + + case_names: list of test case names only + + nicknames: list of test case nicknames + First entry of list will be list of test case nicknames + Second entry will be string object of baseline nickname + + multi_dict: ordered dictionary of difference data for each var, test case, and season + multi_dict[var][case_name][s] + + adfobj: ADF object + Needed to test if redo_plot is called in config yaml file + """ + + ncols = 3 + nplots = len(nicknames[0]) + + #Try and format spacing based on number of cases + # NOTE: ** this will have to change if figsize or dpi change ** + if nplots < 4: + hspace = -1.0 + else: + hspace = -0.85 + + nrows = int(np.ceil(nplots/ncols)) + if nrows < 2: + nrows = 2 + + # specify the central longitude for the plot + central_longitude = 180 + proj = ccrs.PlateCarree(central_longitude=central_longitude) + # formatting for tick labels + lon_formatter = LongitudeFormatter(number_format='0.0f', + degree_symbol='', + dateline_direction_label=False) + lat_formatter = LatitudeFormatter(number_format='0.0f', + degree_symbol='') + + for var in multi_dict.keys(): + for j in multi_dict[var].keys(): + for season in multi_dict[var][j].keys(): + from pathlib import Path + # Check redo_plot. If set to True: remove old plot, if it already exists: + redo_plot = adfobj.get_basic_info('redo_plot') + if (not redo_plot) and Path(wks / f"{var}_{season}_{ptype}_multi_plot.png").is_file(): + #Continue to next iteration: + continue + elif (redo_plot) or (not Path(wks / f"{var}_{season}_{ptype}_multi_plot.png").is_file()): + fig_width = 15 + fig_height = 15+(3*nrows) #try and dynamically create size of fig based off number of cases (therefore rows) + fig, axs = plt.subplots(nrows=nrows,ncols=ncols,figsize=(fig_width,fig_height), facecolor='w', edgecolor='k', + sharex=True, + sharey=True, + subplot_kw={"projection": proj}) + + #Set figure title + plt.suptitle(f'All Case Comparison (Test - Baseline) {var}: {season}\n', fontsize=16,y=0.325) + axs[0,1].set_title("$\mathbf{Baseline}:$"+f'{nicknames[1]}\n', fontsize=12) + + count = 0 + img = [] + titles = [] + for r in range(0,nrows): + for c in range(0,ncols): + if count < nplots: + mdlfld = multi_dict[var][case_names[count]][season]["diff_data"] + lat = mdlfld['lat'] + mwrap, lon = add_cyclic_point(mdlfld, coord=mdlfld['lon']) + + # mesh for plots: + lons, lats = np.meshgrid(lon, lat) + + levelsdiff = multi_dict[var][case_names[count]][season]["vres"]["diff_contour_range"] + levelsdiff = np.arange(levelsdiff[0],levelsdiff[1]+levelsdiff[-1],levelsdiff[-1]) + + normfunc, mplv = use_this_norm() + + # color normalization for difference + if ((np.min(levelsdiff) < 0) and (0 < np.max(levelsdiff))) and mplv > 2: + normdiff = normfunc(vmin=np.min(levelsdiff), vmax=np.max(levelsdiff), vcenter=0.0) + else: + normdiff = mpl.colors.Normalize(vmin=np.min(levelsdiff), vmax=np.max(levelsdiff)) + + cmap = multi_dict[var][case_names[count]][season]["vres"]['diff_colormap'] + + img.append(axs[r,c].contourf(lons, lats, mwrap, levels=levelsdiff, + cmap=cmap, norm=normdiff, + transform=ccrs.PlateCarree())) + + #Set individual plot titles (case name/nickname) + titles.append(axs[r,c].set_title("$\mathbf{Test}:$"+f" {nicknames[0][count]}",loc='left',fontsize=8)) + + axs[r,c].spines['geo'].set_linewidth(1.5) #cartopy's recommended method + axs[r,c].coastlines() + axs[r,c].set_xticks(np.linspace(-180, 120, 6), crs=ccrs.PlateCarree()) + axs[r,c].set_yticks(np.linspace(-90, 90, 7), crs=ccrs.PlateCarree()) + axs[r,c].tick_params('both', length=5, width=1.5, which='major') + axs[r,c].tick_params('both', length=5, width=1.5, which='minor') + axs[r,c].xaxis.set_major_formatter(lon_formatter) + axs[r,c].yaxis.set_major_formatter(lat_formatter) + + else: + #Clear left over subplots if they don't fill the row x column matrix + axs[r,c].set_visible(False) + count = count + 1 + # __COLORBARS__ + fig.colorbar(img[-1], ax=axs.ravel().tolist(), orientation='horizontal',aspect=20,shrink=.5,location="bottom",anchor=(0.5,-0.3),extend='both') + + plt.subplots_adjust(wspace=0.3, hspace=hspace) + fig.savefig(wks / f"{var}_{season}_{ptype}_multi_plot.png", bbox_inches='tight', dpi=300) + + #Close plots: + plt.close() + + ##################### #END HELPER FUNCTIONS diff --git a/lib/website_templates/template.html b/lib/website_templates/template.html index cfa488ab2..151be93c2 100644 --- a/lib/website_templates/template.html +++ b/lib/website_templates/template.html @@ -1,9 +1,7 @@ - - - ADF {{var_title}} + ADF {{ var_title }} @@ -13,7 +11,10 @@
@@ -75,4 +76,4 @@

Baseline Case:

-

Click on case name for that ADF vs baseline page

+

Click on case name for that ADF case vs baseline page

{% for case_name, case_dtls in case_sites.items() %} -

Test Case {{ loop.index }}:

- {{ case_name }}

+

Test Case {{ loop.index }}:

+

{{ case_name }}

- years: {{ case_dtls[1] }} - {{ case_dtls[2] }}


{% endfor %} -

Baseline Case:

- {{ base_name }}

+

Baseline Case:

+

{{ base_name }}

- years: {{ baseline_yrs }}

diff --git a/lib/website_templates/template_multi_case_var.html b/lib/website_templates/template_multi_case_var.html index 4df72ac9b..b1cf18b40 100644 --- a/lib/website_templates/template_multi_case_var.html +++ b/lib/website_templates/template_multi_case_var.html @@ -31,14 +31,14 @@ -

Click on case name for that ADF vs baseline page

+

Click on case name for that ADF case vs baseline page

{% for case_name, case_dtls in case_sites.items() %} -

Test Case {{loop.index}}:

- {{ case_name }}

+

Test Case {{loop.index}}:

+

{{ case_name }}

- years: {{ case_dtls[1] }} - {{ case_dtls[2] }}


{% endfor %} -

Baseline Case:

- {{ base_name }}

+

Baseline Case:

+

{{ base_name }}

- years: {{ baseline_yrs }}

diff --git a/lib/website_templates/template_table.html b/lib/website_templates/template_table.html index c3f019f22..d3161dc07 100644 --- a/lib/website_templates/template_table.html +++ b/lib/website_templates/template_table.html @@ -12,8 +12,10 @@ {% if multi==True %} -

Click on case name for that ADF vs baseline page

+

Click on case name for that ADF case vs baseline page

{% for case_name, case_dtls in case_sites.items() %} -

Test Case {{ loop.index }}:

- {{ case_name }}

+

Test Case {{ loop.index }}:

+

{{ case_name }}

- years: {{ case_dtls[1] }} - {{ case_dtls[2] }}


{% endfor %} -

Baseline Case:

- {{ base_name }}

+

Baseline Case:

+

{{ base_name }}

- years: {{ baseline_yrs }}

{% else %}
diff --git a/scripts/analysis/amwg_table.py b/scripts/analysis/amwg_table.py index 7e2cc913e..99a8a7088 100644 --- a/scripts/analysis/amwg_table.py +++ b/scripts/analysis/amwg_table.py @@ -114,11 +114,8 @@ def amwg_table(adf): #CAM simulation variables (these quantities are always lists): case_names = adf.get_cam_info("cam_case_name", required=True) - #Grab test case nickname(s) - test_nicknames = adf.get_cam_info('case_nickname') - for idx,nick_name in enumerate(test_nicknames): - if nick_name == None: - test_nicknames[idx] = case_names[idx] + #Grab all case nickname(s) + test_nicknames = adf.case_nicknames["test_nicknames"] input_ts_locs = adf.get_cam_info("cam_ts_loc", required=True) @@ -129,10 +126,10 @@ def amwg_table(adf): input_ts_baseline = adf.get_baseline_info("cam_ts_loc", required=True) #Grab baseline case nickname - base_nickname = adf.get_baseline_info('case_nickname') - if base_nickname == None: - base_nickname = baseline_name + base_nickname = adf.case_nicknames["base_nickname"] + test_nicknames += [base_nickname] + if "CMIP" in baseline_name: print("CMIP files detected, skipping AMWG table (for now)...") @@ -293,6 +290,18 @@ def amwg_table(adf): df.to_csv(output_csv_file, header=cols, index=False) #End if + #last step is to add table dataframe to website (if enabled): + table_df = pd.read_csv(output_csv_file) + + #Reorder RESTOM to top of tables + idx = table_df.index[table_df['variable'] == 'RESTOM'].tolist()[0] + table_df = pd.concat([table_df[table_df['variable'] == 'RESTOM'], table_df]).reset_index(drop = True) + table_df = table_df.drop([idx+1]).reset_index(drop=True) + table_df = table_df.drop_duplicates() + + #Re-save the csv file + table_df.to_csv(output_csv_file, header=cols, index=False) + else: #Print message to debug log: adf.debug_log("RESTOM not calculated because FSNT and/or FLNT variables not in dataset") @@ -304,8 +313,6 @@ def amwg_table(adf): #End of model case loop #---------------------- - - #Check if observations are being comapred to, if so skip table comparison... test_case_names = adf.get_cam_info("cam_case_name", required=True) #Notify user that script has ended: @@ -396,7 +403,13 @@ def _get_row_vals(data): ##### -def _df_comp_table(adf, output_location, base_output_location ,case_names): +def _df_comp_table(adf, output_location, base_output_location, case_names): + """ + Function to build case vs baseline AMWG table + ----- + - Read in table data and create side by side comaprison table + - Write output to csv file and add to website + """ output_csv_file_comp = output_location / "amwg_table_comp.csv" @@ -429,27 +442,42 @@ def _df_comp_table(adf, output_location, base_output_location ,case_names): ##### def _df_multi_comp_table(adf, csv_locs, case_names, test_nicknames): + """ + Function to build all case comparison AMWG table + ------ + - Read in each previously made table from file + and compile full comparison. + """ + #Create path to main website in mutli-case directory main_site_path = Path(adf.get_basic_info('cam_diag_plot_loc', required=True)) output_csv_file_comp = main_site_path / "amwg_table_comp_all.csv" + + #Create the "comparison" dataframe: df_comp = pd.DataFrame(dtype=object) + #Create new colummns cols_comp = ['variable', 'unit'] + #Read baseline case baseline = str(csv_locs[-1])+f"/amwg_table_{case_names[-1]}.csv" df_base = pd.read_csv(baseline) + #Read all test cases and add to table for i,val in enumerate(csv_locs[:-1]): case = str(val)+f"/amwg_table_{case_names[i]}.csv" df_case = pd.read_csv(case) + #If no custom nicknames, shorten column name to case number if test_nicknames[i] == case_names[i]: df_comp[['variable','unit',f"case {i+1}"]] = df_case[['variable','unit','mean']] cols_comp.append(f"case {i+1}") + #Else, name columns after nicknames else: df_comp[['variable','unit',f"{test_nicknames[i]}"]] = df_case[['variable','unit','mean']] cols_comp.append(test_nicknames[i]) - + + #Add baseline cases to end of the table if test_nicknames[-1] == case_names[-1]: df_comp["baseline"] = df_base[['mean']] cols_comp.append("baseline") @@ -457,17 +485,24 @@ def _df_multi_comp_table(adf, csv_locs, case_names, test_nicknames): df_comp[f"{test_nicknames[-1]} ( baseline )"] = df_base[['mean']] cols_comp.append(f"{test_nicknames[-1]} ( baseline )") + #Format the floats: for col in df_comp.columns: + #Ignore columns that don't contain floats if (col != 'variable') and (col != "unit"): if "baseline" not in col: + + #Iterate over rows and check magnitude of value for idx,row in enumerate(df_comp[col]): + #Check if value is less than one, keep 3 non-zero decimal values + #Else, keep 3 main digits, including decimal values if np.abs(df_comp[col][idx]) < 1: formatter = ".3g" else: formatter = ".3f" - + #Replace value in dataframe df_comp.at[idx,col]= f'{df_comp[col][idx]:{formatter}} ({(df_comp[col][idx]-df_base["mean"][idx]):{formatter}})' + #Finally, write data to csv df_comp.to_csv(output_csv_file_comp, header=cols_comp, index=False) #Add comparison table dataframe to website (if enabled): diff --git a/scripts/plotting/cam_taylor_diagram.py b/scripts/plotting/cam_taylor_diagram.py index 59b0d6c8b..a1c4c730c 100644 --- a/scripts/plotting/cam_taylor_diagram.py +++ b/scripts/plotting/cam_taylor_diagram.py @@ -11,6 +11,7 @@ # # --- imports and configuration --- # +from collections import OrderedDict from pathlib import Path import numpy as np import xarray as xr @@ -49,13 +50,23 @@ def cam_taylor_diagram(adfobj): # test case(s) == case(s) to be diagnosed will be called `case` (assumes a list) case_names = adfobj.get_cam_info('cam_case_name', required=True) # Loop over these + #Grab case climo years syear_cases = adfobj.climo_yrs["syears"] eyear_cases = adfobj.climo_yrs["eyears"] - #Grab test case nickname(s) - test_nicknames = adfobj.get_cam_info('case_nickname') - if test_nicknames == None: - test_nicknames = case_names + #Grab all case nickname(s) + test_nicknames = adfobj.case_nicknames["test_nicknames"] + base_nickname = adfobj.case_nicknames["base_nickname"] + + if len(case_names) > 1: + multi_case = True + + main_site_path = adfobj.main_site_paths["main_site_path"] + main_site_assets_path = adfobj.main_site_paths["main_site_assets_path"] + + else: + multi_case = False + #End if (check for multiple cases) case_climo_loc = adfobj.get_cam_info('cam_climo_loc', required=True) @@ -74,7 +85,10 @@ def cam_taylor_diagram(adfobj): plot_loc = Path(plot_location[0]) else: print(f"Ambiguous plotting location since all cases go on same plot. Will put them in first location: {plot_location[0]}") - plot_loc = Path(plot_location[0]) + if multi_case: + plot_loc = main_site_path + else: + plot_loc = Path(plot_location[0]) else: plot_loc = Path(plot_location) @@ -90,14 +104,9 @@ def cam_taylor_diagram(adfobj): data_name = adfobj.get_baseline_info('cam_case_name', required=True) data_list = data_name # should not be needed (?) data_loc = adfobj.get_baseline_info("cam_climo_loc", required=True) - - #Grab baseline case nickname - base_nickname = adfobj.get_baseline_info('case_nickname') - if base_nickname == None: - base_nickname = data_name #End if - #Extract baseline years (which may be empty strings if using Obs): + #Grab baseline years (which may be empty strings if using Obs): syear_baseline = adfobj.climo_yrs["syear_baseline"] eyear_baseline = adfobj.climo_yrs["eyear_baseline"] @@ -144,20 +153,6 @@ def cam_taylor_diagram(adfobj): # LOOP OVER SEASON # for s in seasons: - - plot_name = plot_loc / f"TaylorDiag_{s}_Special_Mean.{plot_type}" - print(f"\t - Plotting Taylor Diagram, {s}") - - # Check redo_plot. If set to True: remove old plot, if it already exists: - if (not redo_plot) and plot_name.is_file(): - #Add already-existing plot to website (if enabled): - adfobj.add_website_data(plot_name, "TaylorDiag", None, season=s, multi_case=True) - - #Continue to next iteration: - continue - elif (redo_plot) and plot_name.is_file(): - plot_name.unlink() - # hold the data in a DataFrame for each case # variable | correlation | stddev ratio | bias df_template = pd.DataFrame(index=var_list, columns=['corr', 'ratio', 'bias']) @@ -175,22 +170,85 @@ def cam_taylor_diagram(adfobj): # # -- PLOTTING (one per season) -- # + fig, ax = taylor_plot_setup(title=f"Taylor Diagram - {s}", - baseline=f"Baseline: {data_name} yrs: {syear_baseline}-{eyear_baseline}") + baseline=f"Baseline: {base_nickname} yrs: {syear_baseline}-{eyear_baseline}") for i, case in enumerate(case_names): ax = plot_taylor_data(ax, result_by_case[case], case_color=case_colors[i], use_bias=True) - ax = taylor_plot_finalize(ax, case_names, case_colors, syear_cases, eyear_cases, needs_bias_labels=True) + #If multi-case, make all individual case vs baseline taylor diagram + if multi_case: + fig_m, ax_m = taylor_plot_setup(title=f"Taylor Diagram - {s}", + baseline=f"Baseline: {base_nickname} yrs: {syear_baseline}-{eyear_baseline}") + ax_m = plot_taylor_data(ax_m, result_by_case[case], case_color=case_colors[i], use_bias=True) + ax_m = taylor_plot_finalize(ax_m, test_nicknames[i], case_colors[i], + syear_cases[i], eyear_cases[i], + needs_bias_labels=True,multi=True) + + # add text with variable names: + txtstrs = [f"{i+1} - {v}" for i, v in enumerate(var_list)] + fig_m.text(0.9, 0.9, "\n".join(txtstrs), va='top') + + plot_name = Path(plot_location[i]) / f"TaylorDiag_{s}_Special_Mean.{plot_type}" + fig_m.savefig(plot_name, bbox_inches='tight') + print(f"\t Taylor Diagram: completed {s}. \n\t File: {plot_name}") + plt.close() + + #Add plot to website (if enabled): + adfobj.add_website_data(plot_name, "TaylorDiag", case, category=None, season=s, plot_type = "Special") + #End if (multi-case) + #End for (cases) + + ax = taylor_plot_finalize(ax, test_nicknames, case_colors, syear_cases, eyear_cases, needs_bias_labels=True,multi=False) + #ax = taylor_plot_finalize(ax, case_names, case_colors, syear_cases, eyear_cases, needs_bias_labels=True) # add text with variable names: txtstrs = [f"{i+1} - {v}" for i, v in enumerate(var_list)] fig.text(0.9, 0.9, "\n".join(txtstrs), va='top') - fig.savefig(plot_name, bbox_inches='tight') - print(f"\t Taylor Diagram: completed {s}. \n\t File: {plot_name}") - #Add plot to website (if enabled): - adfobj.add_website_data(plot_name, "TaylorDiag", None, season=s, multi_case=True) + plot_name = plot_loc / f"TaylorDiag_{s}_Special_Mean.{plot_type}" + print(f"\t - Plotting Taylor Diagram, {s}") + + if multi_case: + plot_name = main_site_assets_path / f"TaylorDiag_{s}_Special_multi_plot.{plot_type}" + + # Check redo_plot. If set to True: remove old plot, if it already exists: + if (not redo_plot) and plot_name.is_file(): + #Add already-existing plot to website (if enabled): + adfobj.add_website_data(plot_name, "TaylorDiag", None, category=None, season=s, multi_case=True,plot_type = "Special") + + #Continue to next iteration: + continue + elif (redo_plot) and plot_name.is_file(): + plot_name.unlink() + + fig.savefig(plot_name, bbox_inches='tight') + plt.close() + + #Add plot to website (if enabled): + adfobj.add_website_data(plot_name, "TaylorDiag", None, category=None, season=s, multi_case=True,plot_type = "Special") + + print(" ...Taylor Diagram multi-case plots have been generated successfully.") + else: + plot_name = plot_loc / f"TaylorDiag_{s}_Special_Mean.{plot_type}" + + # Check redo_plot. If set to True: remove old plot, if it already exists: + if (not redo_plot) and plot_name.is_file(): + #Add already-existing plot to website (if enabled): + adfobj.add_website_data(plot_name, "TaylorDiag", case_names[0], category=None, season=s, plot_type = "Special") + #Continue to next iteration: + continue + elif (redo_plot) and plot_name.is_file(): + plot_name.unlink() + + fig.savefig(plot_name, bbox_inches='tight') + print(f"\t Taylor Diagram: completed {s}. \n\t File: {plot_name}") + plt.close() + + #Add plot to website (if enabled): + adfobj.add_website_data(plot_name, "TaylorDiag", case_names[0], category=None, season=s, plot_type = "Special") + #End if (multi-case check) #Notify user that script has ended: print(" ...Taylor Diagrams have been generated successfully.") @@ -561,7 +619,7 @@ def plot_taylor_data(wks, df, **kwargs): return wks -def taylor_plot_finalize(wks, casenames, casecolors, syear_cases, eyear_cases, needs_bias_labels=True): +def taylor_plot_finalize(wks, casenames, casecolors, syear_cases, eyear_cases, needs_bias_labels=True, multi=False): """Apply final formatting to a Taylor diagram. wks -> Axes object that has passed through taylor_plot_setup and plot_taylor_data casenames -> list of case names for the legend @@ -570,16 +628,23 @@ def taylor_plot_finalize(wks, casenames, casecolors, syear_cases, eyear_cases, n """ # CASE LEGEND -- Color-coded bottom_of_text = 0.05 - number_of_lines = len(casenames) height_of_lines = 0.03 - text = wks.text(0.052, 0.08, "Cases:", - color='k', ha='left', va='bottom', transform=wks.transAxes, fontsize=11) - n = 0 - for case_idx, (s, c) in enumerate(zip(casenames, casecolors)): - text = wks.text(0.052, bottom_of_text + n*height_of_lines, f"{s} yrs: {syear_cases[case_idx]}-{eyear_cases[case_idx]}", - color=c, ha='left', va='bottom', transform=wks.transAxes, fontsize=10) + case_pos = 0.75 + wks.text(0.99, case_pos, "Cases:", va='top', transform=wks.transAxes, fontsize=10) + + n = 0 + if multi: + for case_idx, (s, c) in enumerate(zip([casenames], [casecolors])): + text = wks.text(0.99, case_pos-((case_idx+1)*height_of_lines), f"{s} yrs: {syear_cases}-{eyear_cases}", + color=c, va='top', transform=wks.transAxes, fontsize=10) n += 1 + else: + for case_idx, (s, c) in enumerate(zip(casenames, casecolors)): + text = wks.text(0.99, case_pos-((case_idx+1)*height_of_lines), f"{s} yrs: {syear_cases[case_idx]}-{eyear_cases[case_idx]}", + color=c, va='top', transform=wks.transAxes, fontsize=10) + n += 1 + # BIAS LEGEND if needs_bias_labels: # produce an info-box showing the markers/sizes based on bias diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index 2767c6558..3d1104904 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -65,24 +65,23 @@ def global_latlon_map(adfobj): #CAM simulation variables (this is always assumed to be a list): case_names = adfobj.get_cam_info("cam_case_name", required=True) - + #read_config_var('multi_case_plots') if len(case_names) > 1: #Check if multi-plots are desired from yaml file - if adfobj.read_config_var('multi_case_plots'): - multi_plots = True - if multi_plots: + if adfobj.get_multi_case_info("global_latlon_map"): + multi_plots = True multi_dict = OrderedDict() + else: + multi_plots = False + #End if (check for multi-case plots for LatLon) else: multi_plots = False + #End if (check for multiple cases) + #Grab case climo years syear_cases = adfobj.climo_yrs["syears"] eyear_cases = adfobj.climo_yrs["eyears"] - #Grab test case nickname(s) - test_nicknames = adfobj.get_cam_info('case_nickname') - if test_nicknames == None: - test_nicknames = case_names - # CAUTION: # "data" here refers to either obs or a baseline simulation, # Until those are both treated the same (via intake-esm or similar) @@ -91,29 +90,26 @@ def global_latlon_map(adfobj): #Extract variable-obs dictionary: var_obs_dict = adfobj.var_obs_dict - base_nickname = "Obs" #If dictionary is empty, then there are no observations to regrid to, #so quit here: if not var_obs_dict: print("No observations found to plot against, so no lat/lon maps will be generated.") return - else: data_name = adfobj.get_baseline_info("cam_case_name", required=True) # does not get used, is just here as a placemarker data_list = [data_name] # gets used as just the name to search for climo files HAS TO BE LIST data_loc = model_rgrid_loc #Just use the re-gridded model data path - - #Grab baseline case nickname - base_nickname = adfobj.get_baseline_info('case_nickname') - if base_nickname == None: - base_nickname = data_name #End if - #Extract baseline years (which may be empty strings if using Obs): + #Grab baseline years (which may be empty strings if using Obs): syear_baseline = adfobj.climo_yrs["syear_baseline"] eyear_baseline = adfobj.climo_yrs["eyear_baseline"] + #Grab all case nickname(s) + test_nicknames = adfobj.case_nicknames["test_nicknames"] + base_nickname = adfobj.case_nicknames["base_nickname"] + res = adfobj.variable_defaults # will be dict of variable-specific plot preferences # or an empty dictionary if use_defaults was not specified in YAML. @@ -151,15 +147,16 @@ def global_latlon_map(adfobj): "MAM": [3, 4, 5], "SON": [9, 10, 11] } - + # probably want to do this one variable at a time: for var in var_list: + + #Check if multi-case scenario, if so grab details if multi_plots: - for multi_var in adfobj.read_config_var('multi_case_plots')["global_latlon_map"]: + for multi_var in adfobj.get_multi_case_info("global_latlon_map"): if multi_var not in multi_dict: multi_dict[multi_var] = OrderedDict() - if adfobj.compare_obs: #Check if obs exist for the variable: if var in var_obs_dict: @@ -213,6 +210,7 @@ def global_latlon_map(adfobj): oclim_fils = sorted(dclimo_loc.glob(f"{data_src}_{var}_baseline.nc")) oclim_ds = _load_dataset(oclim_fils) + if oclim_ds is None: print("WARNING: Did not find any oclim_fils. Will try to skip.") print(f"INFO: Data Location, dclimo_loc is {dclimo_loc}") @@ -238,7 +236,7 @@ def global_latlon_map(adfobj): print(" {} not found, making new directory".format(plot_loc)) plot_loc.mkdir(parents=True) - # load re-gridded model files: + #Load re-gridded model files: mclim_fils = sorted(mclimo_rg_loc.glob(f"{data_src}_{case_name}_{var}_*.nc")) mclim_ds = _load_dataset(mclim_fils) @@ -246,18 +244,18 @@ def global_latlon_map(adfobj): odata = oclim_ds[data_var].squeeze() # squeeze in case of degenerate dimensions mdata = mclim_ds[var].squeeze() - # APPLY UNITS TRANSFORMATION IF SPECIFIED: - # NOTE: looks like our climo files don't have all their metadata + #APPLY UNITS TRANSFORMATION IF SPECIFIED: + #NOTE: looks like our climo files don't have all their metadata mdata = mdata * vres.get("scale_factor",1) + vres.get("add_offset", 0) - # update units + #Update units mdata.attrs['units'] = vres.get("new_unit", mdata.attrs.get('units', 'none')) - # Do the same for the baseline case if need be: + #Do the same for the baseline case if need be: if not adfobj.compare_obs: odata = odata * vres.get("scale_factor",1) + vres.get("add_offset", 0) # update units odata.attrs['units'] = vres.get("new_unit", odata.attrs.get('units', 'none')) - # Or for observations: + #Or for observations: else: odata = odata * vres.get("obs_scale_factor",1) + vres.get("obs_add_offset", 0) # Note: we are going to assume that the specification ensures the conversion makes the units the same. Doesn't make sense to add a different unit. @@ -286,10 +284,6 @@ def global_latlon_map(adfobj): oseasons = {} dseasons = {} # hold the differences - #Initialize Ordered Dictionary for variable: - #if case_name not in restom_dict: - #restom_dict[case_name] = OrderedDict() - #Loop over season dictionary: for s in seasons: @@ -490,22 +484,18 @@ def global_latlon_map(adfobj): #End if (dimensions check and plotting pressure levels) #End for (case loop) #End for (obs/baseline loop) - #End for (variable loop) #This will be a list of variables for multi-case plotting based off LatLon plot type if multi_plots: #Notify user that script has started: - print("\n Generating lat/lon multi-case plots...") - multi_path = Path(adfobj.get_basic_info('cam_diag_plot_loc', required=True)) - main_site_path = multi_path / "main_website" - main_site_path.mkdir(exist_ok=True) - main_site_assets_path = main_site_path / "assets" - main_site_assets_path.mkdir(exist_ok=True) + main_site_assets_path = adfobj.main_site_paths["main_site_assets_path"] - pf.multi_latlon_plots(main_site_assets_path, "LatLon", case_names, [test_nicknames,base_nickname], multi_dict, adfobj) + pf.multi_latlon_plots(main_site_assets_path, "LatLon", case_names, + [test_nicknames,base_nickname], multi_dict, + web_category, adfobj) print(" ...lat/lon multi-case plots have been generated successfully.") #Notify user that script has ended: diff --git a/scripts/plotting/global_latlon_vect_map.py b/scripts/plotting/global_latlon_vect_map.py index 8ba845c8d..14978c583 100644 --- a/scripts/plotting/global_latlon_vect_map.py +++ b/scripts/plotting/global_latlon_vect_map.py @@ -61,13 +61,13 @@ def global_latlon_vect_map(adfobj): #CAM simulation variables: case_names = adfobj.get_cam_info("cam_case_name", required=True) + #Grab case climo years syear_cases = adfobj.climo_yrs["syears"] eyear_cases = adfobj.climo_yrs["eyears"] - #Grab test case nickname(s) - test_nicknames = adfobj.get_cam_info('case_nickname') - if test_nicknames == None: - test_nicknames = case_names + #Grab all case nickname(s) + test_nicknames = adfobj.case_nicknames["test_nicknames"] + base_nickname = adfobj.case_nicknames["base_nickname"] # CAUTION: # "data" here refers to either obs or a baseline simulation, @@ -77,7 +77,6 @@ def global_latlon_vect_map(adfobj): #Extract variable-obs dictionary: var_obs_dict = adfobj.var_obs_dict - base_nickname = "Obs" #If dictionary is empty, then there are no observations to regrid to, #so quit here: @@ -89,14 +88,9 @@ def global_latlon_vect_map(adfobj): data_name = adfobj.get_baseline_info("cam_case_name", required=True) # does not get used, is just here as a placemarker data_list = [data_name] # gets used as just the name to search for climo files HAS TO BE LIST data_loc = model_rgrid_loc #Just use the re-gridded model data path - - #Grab baseline case nickname - base_nickname = adfobj.get_baseline_info('case_nickname') - if base_nickname == None: - base_nickname = data_name #End if - #Extract baseline years (which may be empty strings if using Obs): + #Grab baseline years (which may be empty strings if using Obs): syear_baseline = adfobj.climo_yrs["syear_baseline"] eyear_baseline = adfobj.climo_yrs["eyear_baseline"] diff --git a/scripts/plotting/meridional_mean.py b/scripts/plotting/meridional_mean.py index 1fadc2669..30363ca2d 100644 --- a/scripts/plotting/meridional_mean.py +++ b/scripts/plotting/meridional_mean.py @@ -36,13 +36,13 @@ def meridional_mean(adfobj): #CAM simulation variables (this is always assumed to be a list): case_names = adfobj.get_cam_info("cam_case_name", required=True) + #Grab case climo years syear_cases = adfobj.climo_yrs["syears"] eyear_cases = adfobj.climo_yrs["eyears"] - #Grab test case nickname(s) - test_nicknames = adfobj.get_cam_info('case_nickname') - if test_nicknames == None: - test_nicknames = case_names + #Grab all case nickname(s) + test_nicknames = adfobj.case_nicknames["test_nicknames"] + base_nickname = adfobj.case_nicknames["base_nickname"] # CAUTION: # "data" here refers to either obs or a baseline simulation, @@ -52,7 +52,6 @@ def meridional_mean(adfobj): #Extract variable-obs dictionary: var_obs_dict = adfobj.var_obs_dict - base_nickname = "Obs" #If dictionary is empty, then there are no observations to regrid to, #so quit here: @@ -64,14 +63,9 @@ def meridional_mean(adfobj): data_name = adfobj.get_baseline_info("cam_case_name", required=True) # does not get used, is just here as a placemarker data_list = [data_name] # gets used as just the name to search for climo files HAS TO BE LIST data_loc = model_rgrid_loc #Just use the re-gridded model data path - - #Grab baseline case nickname - base_nickname = adfobj.get_baseline_info('case_nickname') - if base_nickname == None: - base_nickname = data_name #End if - #Extract baseline years (which may be empty strings if using Obs): + #Grab baseline years (which may be empty strings if using Obs): syear_baseline = adfobj.climo_yrs["syear_baseline"] eyear_baseline = adfobj.climo_yrs["eyear_baseline"] diff --git a/scripts/plotting/polar_map.py b/scripts/plotting/polar_map.py index e4b6018a3..71e135e27 100644 --- a/scripts/plotting/polar_map.py +++ b/scripts/plotting/polar_map.py @@ -41,13 +41,13 @@ def polar_map(adfobj): #CAM simulation variables (this is always assumed to be a list): case_names = adfobj.get_cam_info("cam_case_name", required=True) + #Grab case climo years syear_cases = adfobj.climo_yrs["syears"] eyear_cases = adfobj.climo_yrs["eyears"] - #Grab test case nickname(s) - test_nicknames = adfobj.get_cam_info('case_nickname') - if test_nicknames == None: - test_nicknames = case_names + #Grab all case nickname(s) + test_nicknames = adfobj.case_nicknames["test_nicknames"] + base_nickname = adfobj.case_nicknames["base_nickname"] # CAUTION: # "data" here refers to either obs or a baseline simulation, @@ -57,7 +57,6 @@ def polar_map(adfobj): #Extract variable-obs dictionary: var_obs_dict = adfobj.var_obs_dict - base_nickname = "Obs" #If dictionary is empty, then there are no observations to regrid to, #so quit here: @@ -69,14 +68,9 @@ def polar_map(adfobj): data_name = adfobj.get_baseline_info("cam_case_name", required=True) # does not get used, is just here as a placemarker data_list = [data_name] # gets used as just the name to search for climo files HAS TO BE LIST data_loc = model_rgrid_loc #Just use the re-gridded model data path - - #Grab baseline case nickname - base_nickname = adfobj.get_baseline_info('case_nickname') - if base_nickname == None: - base_nickname = data_name #End if - #Extract baseline years (which may be empty strings if using Obs): + #Grab baseline years (which may be empty strings if using Obs): syear_baseline = adfobj.climo_yrs["syear_baseline"] eyear_baseline = adfobj.climo_yrs["eyear_baseline"] diff --git a/scripts/plotting/qbo.py b/scripts/plotting/qbo.py index 33e65aa2e..3ef492afd 100644 --- a/scripts/plotting/qbo.py +++ b/scripts/plotting/qbo.py @@ -32,7 +32,7 @@ def qbo(adfobj): print("\n Generating qbo plots...") #Extract relevant info from the ADF: - case_name = adfobj.get_cam_info('cam_case_name', required=True) + case_names = adfobj.get_cam_info('cam_case_name', required=True) case_loc = adfobj.get_cam_info('cam_ts_loc', required=True) base_name = adfobj.get_baseline_info('cam_case_name') base_loc = adfobj.get_baseline_info('cam_ts_loc') @@ -40,6 +40,20 @@ def qbo(adfobj): plot_locations = adfobj.plot_location plot_type = adfobj.get_basic_info('plot_type') + #Grab all case nickname(s) + test_nicknames = adfobj.case_nicknames["test_nicknames"] + base_nickname = adfobj.case_nicknames["base_nickname"] + case_nicknames = test_nicknames + [base_nickname] + + if len(case_names) > 1: + multi_case = True + main_site_assets_path = adfobj.main_site_paths["main_site_assets_path"] + else: + multi_case = False + #End if (check for multiple cases) + + + # check if existing plots need to be redone redo_plot = adfobj.get_basic_info('redo_plot') print(f"\t NOTE: redo_plot is set to {redo_plot}") @@ -66,25 +80,10 @@ def qbo(adfobj): #that the QBO plot will be kept in the first case directory: print(f"\t QBO plots will be saved here: {plot_locations[0]}") - # Check redo_plot. If set to True: remove old plots, if they already exist: - if (not redo_plot) and plot_loc_ts.is_file() and plot_loc_amp.is_file(): - #Add already-existing plot to website (if enabled): - adfobj.add_website_data(plot_loc_ts, "QBO", None, season="QBOts", multi_case=True) - adfobj.add_website_data(plot_loc_amp, "QBO", None, season="QBOamp", multi_case=True) - - #Continue to next iteration: - return - elif (redo_plot): - if plot_loc_ts.is_file(): - plot_loc_ts.unlink() - if plot_loc_amp.is_file(): - plot_loc_amp.unlink() - #End if - #Check if model vs model run, and if so, append baseline to case lists: if not adfobj.compare_obs: case_loc.append(base_loc) - case_name.append(base_name) + case_names.append(base_name) #End if #----Read in the OBS (ERA5, 5S-5N average already @@ -92,13 +91,14 @@ def qbo(adfobj): #----Read in the case data and baseline ncases = len(case_loc) - casedat = [ _load_dataset(case_loc[i], case_name[i],'U') for i in range(0,ncases,1) ] + + casedat = [ _load_dataset(case_loc[i], case_names[i],'U') for i in range(0,ncases,1) ] #Find indices for all case datasets that don't contain a zonal wind field (U): bad_idxs = [] for idx, dat in enumerate(casedat): if 'U' not in dat.variables: - warnings.warn(f"QBO: case {case_name[idx]} contains no 'U' field, skipping...") + warnings.warn(f"QBO: case {case_names[idx]} contains no 'U' field, skipping...") bad_idxs.append(idx) #End if #End for @@ -124,57 +124,198 @@ def qbo(adfobj): #----QBO timeseries plots fig = plt.figure(figsize=(16,16)) + fig.suptitle('QBO Time Series', fontsize=14) x1, x2, y1, y2 = plotpos() - ax = plotqbotimeseries(fig, obs, minny, x1[0], x2[0], y1[0], y2[0],'ERA5') + #ax = plotqbotimeseries(fig, obs, minny, x1[0], x2[0], y1[0], y2[0],'ERA5') + + if (adfobj.compare_obs) and (not multi_case): + xo1 = 0.18 + xo2 = 0.45 + xm1 = 0.55 + xm2 = 0.82 + ax = plotqbotimeseries(fig, obs, minny, xo1, xo2, y1[0], y2[0],'ERA5') + else: + ax = plotqbotimeseries(fig, obs, minny, x1[0], x2[0], y1[0], y2[0],'ERA5') - casecount=0 + #casecount=0 for icase in range(0,ncases,1): if (icase < 11 ): # only only going to work with 12 panels currently - ax = plotqbotimeseries(fig, casedat_5S_5N[icase],minny, - x1[icase+1],x2[icase+1],y1[icase+1],y2[icase+1], case_name[icase]) - casecount=casecount+1 + #Check if this is multi-case diagnostics + if multi_case: + if icase != ncases-1: + plot_loc_ts = Path(plot_locations[icase]) / f'QBOts.{plot_type}' + + #----QBO timeseries plots + fig_m = plt.figure(figsize=(16,16)) + fig_m.suptitle('QBO Time Series', fontsize=14) + + """#Plot ERA5 data + ax_m = plotqbotimeseries(fig_m, obs, minny, x1[0], x2[0], y1[0], y2[0],'ERA5')""" + + """#Plot individual case + ax_m = plotqbotimeseries(fig_m, casedat_5S_5N[icase],minny, + x1[1],x2[1],y1[1],y2[1], + case_nicknames[icase])""" + + #Check if compared vs baseline obs, alter x-positions + if adfobj.compare_obs: + """#Plot ERA5 data + ax_m = plotqbotimeseries(fig_m, obs, minny, 0.18, 0.45, y1[0], y2[0],'ERA5') + + #Plot individual case + ax_m = plotqbotimeseries(fig_m, casedat_5S_5N[icase],minny, + 0.55, 0.82, y1[2], y2[2], + case_nicknames[icase])""" + + xo1 = 0.18 + xo2 = 0.45 + xm1 = 0.55 + xm2 = 0.82 + #No observation baseline + else: + """#Plot ERA5 data + ax_m = plotqbotimeseries(fig_m, obs, minny, x1[0], x2[0], y1[0], y2[0],'ERA5') + + #Plot individual case + ax_m = plotqbotimeseries(fig_m, casedat_5S_5N[icase],minny, + x1[1],x2[1],y1[1],y2[1], + case_nicknames[icase])""" + + xo1 = x1[0] + xo2 = x2[0] + xm1 = x1[1] + xm2 = x2[1] + + ax_m = plotqbotimeseries(fig_m, casedat_5S_5N[-1],minny, + x1[2],x2[2],y1[2],y2[2], + base_nickname) + #End if (compare obs) + + #Plot ERA5 data + ax_m = plotqbotimeseries(fig_m, obs, minny, xo1, xo2, y1[0], y2[0],'ERA5') + + #Plot individual case + ax_m = plotqbotimeseries(fig_m, casedat_5S_5N[icase],minny, + xm1, xm2, y1[2], y2[2], + case_nicknames[icase]) + + #Plot colorbar + ax_m = plotcolorbar(fig_m, x1[0]+0.2, x2[2]-0.2,y1[2]-0.035,y1[2]-0.03) + + #Save figure to file: + fig_m.savefig(plot_loc_ts, bbox_inches='tight', facecolor='white') + + #Add plot to website (if enabled): + adfobj.add_website_data(plot_loc_ts, "QBO", case_names[icase], category=None, season="QBOts", + multi_case=True,plot_type="Special") + #End if (multi-case) + + #Check if compared vs baseline obs, alter x-positions + if (adfobj.compare_obs) and (not multi_case): + ax = plotqbotimeseries(fig, casedat_5S_5N[icase],minny, + xm1,xm2,y1[icase+1],y2[icase+1], case_nicknames[icase]) + else: + ax = plotqbotimeseries(fig, casedat_5S_5N[icase],minny, + x1[icase+1],x2[icase+1],y1[icase+1],y2[icase+1], case_nicknames[icase]) + #casecount=casecount+1 else: warnings.warn("The QBO diagnostics can only manage up to twelve cases!") break #End if #End for + #ax = plotcolorbar(fig, x1[0]+0.2, x2[2]-0.2,y1[casecount]-0.035,y1[casecount]-0.03) + ax = plotcolorbar(fig, x1[0]+0.2, x2[2]-0.2,y1[ncases]-0.035,y1[ncases]-0.03) + + if multi_case:#Notify user that script has started: + print("\n Generating qbo multi-case plots...") + + + plot_loc_ts_multi = main_site_assets_path / f'QBO_QBOts_Special_multi_plot.{plot_type}' + fig.savefig(plot_loc_ts_multi, bbox_inches='tight', facecolor='white') + adfobj.add_website_data(plot_loc_ts_multi, "QBO", None, category=None, season="QBOts", + multi_case=True,plot_type="Special") + + else: + #Save figure to file: + fig.savefig(plot_loc_ts, bbox_inches='tight', facecolor='white') - ax = plotcolorbar(fig, x1[0]+0.2, x2[2]-0.2,y1[casecount]-0.035,y1[casecount]-0.03) - - #Save figure to file: - fig.savefig(plot_loc_ts, bbox_inches='tight', facecolor='white') - - #Add plot to website (if enabled): - adfobj.add_website_data(plot_loc_ts, "QBO", None, season="QBOts", multi_case=True) - + #Add plot to website (if enabled): + #adfobj.add_website_data(plot_loc_ts, "QBO", None, season="QBOts", multi_case=True,plot_type = "Special") #multi_case=True + adfobj.add_website_data(plot_loc_ts, "QBO", case_names[0], category=None, season="QBOts", + multi_case=True,plot_type="Special") #----------------- #---Dunkerton and Delisi QBO amplitude obsamp = calcddamp(obs) modamp = [ calcddamp(casedat_5S_5N[i]) for i in range(0,ncases,1) ] + if multi_case: + for icase in range(0,ncases,1): + #Skip baseline case + if icase != ncases-1: + fig = plt.figure(figsize=(16,16)) + + ax = fig.add_axes([0.05,0.6,0.4,0.4]) + ax.plot(modamp[icase], -np.log10(modamp[icase].lev), linewidth=2, label=case_nicknames[icase]) + + #Check to plot baseline if not compared to obs + if not adfobj.compare_obs: + ax.plot(modamp[-1], -np.log10(modamp[-1].lev), linewidth=2, label=base_nickname) + + ax.plot(obsamp, -np.log10(obsamp.pre), color='black', linewidth=2, label='ERA5') + + ax.set_ylim(-np.log10(150),-np.log10(1)) + ax.set_yticks([-np.log10(100),-np.log10(30),-np.log10(10),-np.log10(3),-np.log10(1)]) + ax.set_yticklabels(['100','30','10','3','1'], fontsize=12) + ax.set_ylabel('Pressure (hPa)', fontsize=12) + ax.set_xlabel('Dunkerton and Delisi QBO amplitude (ms$^{-1}$)', fontsize=12) + ax.set_title('Dunkerton and Delisi QBO amplitude', fontsize=14) + + ax.legend(loc='upper left') + + plot_loc_amp = Path(plot_locations[icase]) / f'QBOamp.{plot_type}' + + fig.savefig(plot_loc_amp, bbox_inches='tight', facecolor='white') + plt.close() + #Add plot to website (if enabled): + adfobj.add_website_data(plot_loc_amp, "QBO", case_names[icase], category = None, season="QBOamp", multi_case=True,plot_type = "Special") + #End if (not baseline) + #End for (cases) + #End if (multi-case) fig = plt.figure(figsize=(16,16)) ax = fig.add_axes([0.05,0.6,0.4,0.4]) + + ax.set_ylim(-np.log10(150),-np.log10(1)) ax.set_yticks([-np.log10(100),-np.log10(30),-np.log10(10),-np.log10(3),-np.log10(1)]) ax.set_yticklabels(['100','30','10','3','1'], fontsize=12) ax.set_ylabel('Pressure (hPa)', fontsize=12) ax.set_xlabel('Dunkerton and Delisi QBO amplitude (ms$^{-1}$)', fontsize=12) ax.set_title('Dunkerton and Delisi QBO amplitude', fontsize=14) - ax.plot(obsamp, -np.log10(obsamp.pre), color='black', linewidth=2, label='ERA5') - for icase in range(0,ncases,1): - ax.plot(modamp[icase], -np.log10(modamp[icase].lev), linewidth=2, label=case_name[icase]) + ax.plot(modamp[icase], -np.log10(modamp[icase].lev), linewidth=2, label=case_nicknames[icase]) ax.legend(loc='upper left') - fig.savefig(plot_loc_amp, bbox_inches='tight', facecolor='white') - #Add plot to website (if enabled): - adfobj.add_website_data(plot_loc_amp, "QBO", None, season="QBOamp", multi_case=True) + # + if multi_case: + plot_loc_amp_multi = main_site_assets_path / f'QBO_QBOamp_Special_multi_plot.{plot_type}' + fig.savefig(plot_loc_amp_multi, bbox_inches='tight', facecolor='white') + #Add plot to website (if enabled): + adfobj.add_website_data(plot_loc_amp_multi, "QBO", None, category=None, season="QBOamp", + multi_case=True,plot_type = "Special") + else: + fig.savefig(plot_loc_amp, bbox_inches='tight', facecolor='white') + + #Add plot to website (if enabled): + adfobj.add_website_data(plot_loc_amp, "QBO", case_names[0], category = None, season="QBOamp", multi_case=True,plot_type = "Special") + + #Close main fig + plt.close() #------------------- #Notify user that script has ended: @@ -282,8 +423,9 @@ def plotqbotimeseries(fig, dat, ny, x1, x2, y1, y2, title): ax.set_yticks([-np.log10(1000),-np.log10(300),-np.log10(100),-np.log10(30),-np.log10(10), -np.log10(3),-np.log10(1)]) ax.set_yticklabels(['1000','300','100','30','10','3','1']) - ax.set_ylabel('Pressure (hPa)') + ax.set_ylabel('Pressure (hPa)', fontsize=12) ax.set_title(title, fontsize=14) + #ax.set_xlabel("Years",fontsize=12,labelpad=10) return ax @@ -332,5 +474,17 @@ def blue2red_cmap(n, nowhite = False): return mymap +def _set_ymargin(ax, top, bottom): + """ + Allow for custom padding of plot lines and axes borders + ----- + """ + ax.set_ymargin(0) + ax.autoscale_view() + lim = ax.get_ylim() + delta = np.diff(lim) + top = lim[1] + delta*top + bottom = lim[0] - delta*bottom + ax.set_ylim(bottom,top) diff --git a/scripts/plotting/zonal_mean.py b/scripts/plotting/zonal_mean.py index 950996230..3343630df 100644 --- a/scripts/plotting/zonal_mean.py +++ b/scripts/plotting/zonal_mean.py @@ -60,13 +60,13 @@ def zonal_mean(adfobj): #CAM simulation variables (this is always assumed to be a list): case_names = adfobj.get_cam_info("cam_case_name", required=True) + #Grab case climo years syear_cases = adfobj.climo_yrs["syears"] eyear_cases = adfobj.climo_yrs["eyears"] - #Grab test case nickname(s) - test_nicknames = adfobj.get_cam_info('case_nickname') - if test_nicknames == None: - test_nicknames = case_names + #Grab all case nickname(s) + test_nicknames = adfobj.case_nicknames["test_nicknames"] + base_nickname = adfobj.case_nicknames["base_nickname"] # CAUTION: # "data" here refers to either obs or a baseline simulation, @@ -76,7 +76,6 @@ def zonal_mean(adfobj): #Extract variable-obs dictionary: var_obs_dict = adfobj.var_obs_dict - base_nickname = "Obs" #If dictionary is empty, then there are no observations to regrid to, #so quit here: @@ -88,14 +87,9 @@ def zonal_mean(adfobj): data_name = adfobj.get_baseline_info("cam_case_name", required=True) # does not get used, is just here as a placemarker data_list = [data_name] # gets used as just the name to search for climo files HAS TO BE LIST data_loc = model_rgrid_loc #Just use the re-gridded model data path - - #Grab baseline case nickname - base_nickname = adfobj.get_baseline_info('case_nickname') - if base_nickname == None: - base_nickname = data_name #End if - #Extract baseline years (which may be empty strings if using Obs): + #Grab baseline years (which may be empty strings if using Obs): syear_baseline = adfobj.climo_yrs["syear_baseline"] eyear_baseline = adfobj.climo_yrs["eyear_baseline"] diff --git a/scripts/regridding/regrid_and_vert_interp.py b/scripts/regridding/regrid_and_vert_interp.py index 113171a4e..a87c827a1 100644 --- a/scripts/regridding/regrid_and_vert_interp.py +++ b/scripts/regridding/regrid_and_vert_interp.py @@ -202,6 +202,9 @@ def regrid_and_vert_interp(adf): if len(mclim_fils) > 1: #Combine all cam files together into a single data set: mclim_ds = xr.open_mfdataset(mclim_fils, combine='by_coords') + elif len(mclim_fils) == 0: + print(f"\t - regridding {var} failed, no file. Continuing to next variable.") + continue else: #Open single file as new xsarray dataset: mclim_ds = xr.open_dataset(mclim_fils[0]) From 2e55128bc04074e51cc68844a27dea0cf730e7f6 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Tue, 21 Mar 2023 09:43:23 -0600 Subject: [PATCH 06/12] Clean up GitHub source_code_tests errors --- lib/adf_web.py | 103 ++++++++++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/lib/adf_web.py b/lib/adf_web.py index 655f8201d..7b0b9093b 100644 --- a/lib/adf_web.py +++ b/lib/adf_web.py @@ -456,8 +456,7 @@ def create_website(self): "mean_tables.html") else: multi_plot_type_html[plot_type] = os.path.join("html_img", - f"multi_case_mean_diag_{plot_type}.html") - #multi_plot_type_html[plot_type] = os.path.join("html_img", f"mean_diag_{plot_type}.html") + f"multi_case_mean_diag_{plot_type}.html") #End if #End for @@ -590,7 +589,7 @@ def create_website(self): category = 'No category yet' #End if #End if - + #Check to see if there are multiple-cases if main_site_path: #Check to see if the user has multi-plots enabled @@ -602,28 +601,28 @@ def create_website(self): for var in multi_case_plots[web_data.plot_ext]: #Check if this is compared to observations - if self.compare_obs: - if var in self.var_obs_dict: - - #Initialize Ordered Dictionary for multi case plot type: - if ptype not in multi_plot_html_info: - multi_plot_html_info[ptype] = OrderedDict() - #End if - - #Initialize Ordered Dictionary for category: - if category not in multi_plot_html_info[ptype]: - multi_plot_html_info[ptype][category] = OrderedDict() - #End if - - if var not in multi_plot_html_info[ptype][category]: - multi_plot_html_info[ptype][category][var] = OrderedDict() - #End if - - p = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" - if season not in multi_plot_html_info[ptype][category][var]: - multi_plot_html_info[ptype][category][var][season] = p - #End if - #End if (obs w/ available var) + if (self.compare_obs) and (var in self.var_obs_dict): + #if var in self.var_obs_dict: + + #Initialize Ordered Dictionary for multi case plot type: + if ptype not in multi_plot_html_info: + multi_plot_html_info[ptype] = OrderedDict() + #End if + + #Initialize Ordered Dictionary for category: + if category not in multi_plot_html_info[ptype]: + multi_plot_html_info[ptype][category] = OrderedDict() + #End if + + if var not in multi_plot_html_info[ptype][category]: + multi_plot_html_info[ptype][category][var] = OrderedDict() + #End if + + p = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" + if season not in multi_plot_html_info[ptype][category][var]: + multi_plot_html_info[ptype][category][var][season] = p + #End if + else: #Initialize Ordered Dictionary for multi case plot type: if ptype not in multi_plot_html_info: @@ -649,7 +648,7 @@ def create_website(self): #End if (variable in multi-case plot variables) #End if multi-case multi-plots - #Need to isolate the multi-case regular plots from the multi-case multi-plots above + #Need to isolate multi-case regular plots from the multi-case multi-plots #QUESTION: Is there a better way? if "multi_plot" not in str(web_data.html_file.name): if ptype not in multi_mean_html_info: @@ -785,8 +784,8 @@ def create_website(self): mean_table_tmpl = jinenv.get_template('template_mean_tables.html') #Reuse the rend_kwarg_dict, but ignore certain keys #since all others are the same - new_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'table_name', 'table_html'}} - + new_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'table_name', 'table_html'}} + if main_site_path: plot_types = multi_plot_type_html new_dict["multi_head"] = "Table" @@ -801,7 +800,7 @@ def create_website(self): #End with #End if #End if (tables) - + else: #Plot image plot_types = plot_type_html @@ -831,7 +830,7 @@ def create_website(self): ofil.write(rndr) #End with - #Check if the mean plot type page exists for this case: + #Check if the mean plot type page exists for this case: mean_ptype_file = img_pages_dir / f"mean_diag_{web_data.plot_type}.html" if not mean_ptype_file.exists(): @@ -908,7 +907,7 @@ def create_website(self): with open(index_html_file, 'w', encoding='utf-8') as ofil: ofil.write(index_rndr) #End with - + #End if (plot images) #End for (web data loop) @@ -1063,7 +1062,7 @@ def create_website(self): #Starting multi-case plots if requested # - - - - - - - - - - - - - - - - - - - if not web_data.data_frame: - + #Extract plot details season = web_data.season ptype = web_data.plot_type @@ -1084,7 +1083,7 @@ def create_website(self): category = 'No category yet' #End if #End if - + #Check for multi-case multi-plots if multi_case_plots: #This currently runs web_data.case for every case, but in reality @@ -1228,17 +1227,17 @@ def create_website(self): #html files, if they don't already exist: page_tmpl = jinenv.get_template('template_multi_case_var.html') plot_page_rndr = page_tmpl.render(title=main_title, - var_title=var, - season_title=season, - plottype_title=ptype, - base_name=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=multi_mean_html_info[ptype], - curr_type=ptype, - plot_types=multi_plot_type_html, - multi=multi_layout, - case_sites=case_sites) + var_title=var, + season_title=season, + plottype_title=ptype, + base_name=data_name, + case_yrs=case_yrs, + baseline_yrs=baseline_yrs, + mydata=multi_mean_html_info[ptype], + curr_type=ptype, + plot_types=multi_plot_type_html, + multi=multi_layout, + case_sites=case_sites) #Write mean diagnostic plots HTML file: with open(mean_ptype_plot_page,'w', encoding='utf-8') as ofil: @@ -1254,14 +1253,14 @@ def create_website(self): #html files, if they don't already exist: tmp = jinenv.get_template('template_multi_case_mean_diag.html') mean_rndr = tmp.render(title=main_title, - base_name=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=multi_mean_html_info[ptype], - curr_type=ptype, - plot_types=multi_plot_type_html, - multi=multi_layout, - case_sites=case_sites) + base_name=data_name, + case_yrs=case_yrs, + baseline_yrs=baseline_yrs, + mydata=multi_mean_html_info[ptype], + curr_type=ptype, + plot_types=multi_plot_type_html, + multi=multi_layout, + case_sites=case_sites) #Write mean diagnostic plots HTML file: with open(mean_ptype_file,'w', encoding='utf-8') as ofil: From c77a9645800eaa2c58308c984a586d3ef06c3499 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 23 Mar 2023 14:27:06 -0600 Subject: [PATCH 07/12] Remove print statement --- lib/adf_web.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/adf_web.py b/lib/adf_web.py index 7b0b9093b..be86eba0e 100644 --- a/lib/adf_web.py +++ b/lib/adf_web.py @@ -502,7 +502,6 @@ def create_website(self): #Also add path to case_sites dictionary: #loop over cases: - print("os.curdir",os.curdir,"\n") for idx, case_name in enumerate(case_names): #Check if case name is present in plot if case_name in self.__case_web_paths: From 29f23b8f383a0f49a6cddfde679aa4d03e5bfaba Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 30 Mar 2023 10:59:14 -0600 Subject: [PATCH 08/12] Finish PR fixes Also clean up `adf_web` and html pages a bit more --- lib/adf_info.py | 30 +- lib/adf_web.py | 397 +++++++----------- lib/website_templates/template.html | 4 +- lib/website_templates/template_index.html | 4 +- lib/website_templates/template_mean_diag.html | 8 +- .../template_mean_tables.html | 4 +- .../template_multi_case_index.html | 10 +- .../template_multi_case_mean_diag.html | 4 +- lib/website_templates/template_table.html | 4 +- lib/website_templates/template_var.html | 4 +- 10 files changed, 181 insertions(+), 288 deletions(-) diff --git a/lib/adf_info.py b/lib/adf_info.py index 807b016d8..7b06daa95 100644 --- a/lib/adf_info.py +++ b/lib/adf_info.py @@ -123,9 +123,10 @@ def __init__(self, config_file, debug=False): case_names = self.get_cam_info('cam_case_name', required=True) #Grab test case nickname(s) - test_nicknames = self.get_cam_info('case_nickname') + test_nicknames = self.get_cam_info('case_nickname', required=True) if test_nicknames is None: - test_nicknames = [None]*len(case_names) + for idx,case_name in enumerate(case_names): + test_nicknames[idx] = case_name #Check if a CAM vs AMWG obs comparison is being performed: if self.__compare_obs: @@ -175,8 +176,12 @@ def __init__(self, config_file, debug=False): #End if #End if + #Grab baseline nickname + base_nickname = self.get_baseline_info('case_nickname') + if base_nickname == None: + base_nickname = data_name + data_name += f"_{syear_baseline}_{eyear_baseline}" - base_nickname = self.get_baseline_info('case_nickname') #End if #Initialize case nicknames: @@ -249,8 +254,7 @@ def __init__(self, config_file, debug=False): #Make directoriess for multi-case diagnostics if applicable if len(case_names) > 1: multi_path = Path(self.get_basic_info('cam_diag_plot_loc', required=True)) - print(multi_path) - multi_path.mkdir(parents=True, exist_ok=True) #exist_ok=True + multi_path.mkdir(parents=True, exist_ok=True) main_site_path = multi_path / "main_website" main_site_path.mkdir(exist_ok=True) main_site_assets_path = main_site_path / "assets" @@ -403,22 +407,12 @@ def climo_yrs(self): @property def case_nicknames(self): """Return the test case and baseline nicknames to the user if requested.""" - #Grab test case nickname(s) - test_nicknames = copy.copy(self.__test_nicknames) - #Case names: - case_names = self.get_cam_info('cam_case_name', required=True) - for idx,nick_name in enumerate(test_nicknames): - if nick_name == None: - test_nicknames[idx] = case_names[idx] - - #Grab test case nickname(s) + #Note that copies are needed in order to avoid having a script mistakenly + #modify these variables, as they are mutable and thus passed by reference: + test_nicknames = copy.copy(self.__test_nicknames) base_nickname = copy.copy(self.__base_nickname) - #Grab baseline nickname - if base_nickname == None: - base_nickname = self.get_baseline_info('cam_case_name', required=True) - return {"test_nicknames":test_nicknames,"base_nickname":base_nickname} # Create property needed to return the multi-case directories to scripts: diff --git a/lib/adf_web.py b/lib/adf_web.py index be86eba0e..c2e0cae2e 100644 --- a/lib/adf_web.py +++ b/lib/adf_web.py @@ -359,28 +359,13 @@ def create_website(self): #Extract needed variables from yaml file: case_names = self.get_cam_info('cam_case_name', required=True) - #Time series files for unspecified climo years - cam_ts_locs = self.get_cam_info('cam_ts_loc', required=True) + #Grab case climo years + syear_cases = self.climo_yrs["syears"] + eyear_cases = self.climo_yrs["eyears"] - #Attempt to grab case start_years (not currently required): - # This is for the header in the html files for climo yrs - syear_cases = self.get_cam_info('start_year') - eyear_cases = self.get_cam_info('end_year') - - if (syear_cases and eyear_cases) == None: - syear_cases = [None]*len(case_names) - eyear_cases = [None]*len(case_names) - - #Loop over model cases to catch all cases that have no climo years specified: - for case_idx, case_name in enumerate(case_names): - - if (syear_cases[case_idx] and eyear_cases[case_idx]) == None: - starting_location = Path(cam_ts_locs[case_idx]) - files_list = sorted(starting_location.glob('*nc')) - #This assumes CAM file names stay with this convention - #Better way to do this? - syear_cases[case_idx] = int(files_list[0].stem[-13:-9]) - eyear_cases[case_idx] = int(files_list[0].stem[-6:-2]) + #Grab baseline years (which may be empty strings if using Obs): + syear_baseline = self.climo_yrs["syear_baseline"] + eyear_baseline = self.climo_yrs["eyear_baseline"] #Set name of comparison data, which depends on "compare_obs": if self.compare_obs: @@ -391,14 +376,6 @@ def create_website(self): else: data_name = self.get_baseline_info('cam_case_name', required=True) - #Attempt to grab baseline start_years (not currently required): - syear_baseline = self.get_baseline_info('start_year') - eyear_baseline = self.get_baseline_info('end_year') - - if (syear_baseline and eyear_baseline) == None: - syear_baseline = self.climo_yrs["syear_baseline"] - eyear_baseline = self.climo_yrs["eyear_baseline"] - #End if baseline_yrs=f"{syear_baseline} - {eyear_baseline}" #End if @@ -408,8 +385,7 @@ def create_website(self): #Extract variable defaults dictionary (for categories): var_defaults_dict = self.variable_defaults - #Dict for multi case if specified - #Grab requested multi-case plots + #Extract requested multi-case multi-plots multi_case_plots = self.read_config_var('multi_case_plots') if multi_case_plots: @@ -418,9 +394,10 @@ def create_website(self): #ext are plot type extentions (keys for multi-case plots) #var_list should be a list of all vars for each plot map extentions #var is iterative for all plot map extensions - for var_list in [multi_case_plots[ext] for ext in multi_case_plots]: - for var in var_list: - mvars.append(var) + for multi_var_list in [multi_case_plots[ext] for ext in multi_case_plots]: + for multi_var in multi_var_list: + if ((self.compare_obs) and (multi_var in self.var_obs_dict)) or (not self.compare_obs): + mvars.append(multi_var) #Create multi-case site: #Make a dictionary for plot type extensions for given plot type @@ -431,11 +408,9 @@ def create_website(self): "meridional":"Meridional", "global_latlon_vect_map":"LatLon_Vector"} - #This one should be auto popoulated for sure - #TODO: Do that what was said prior to this line, but no further, Harry + #Dictionary for multi-case website plot types multi_plots = {"Tables": "html_table/mean_tables.html", - "Special":"html_img/multi_case_mean_diag_Special.html" - } + "Special":"html_img/multi_case_mean_diag_Special.html"} #Set plot type html dictionary (for Jinja templating): plot_type_html = OrderedDict() @@ -592,58 +567,26 @@ def create_website(self): #Check to see if there are multiple-cases if main_site_path: #Check to see if the user has multi-plots enabled - #if (multi_case_plots) and (var in mvars): if multi_case_plots: #Loop over each variable in multi-case plot variables #Check if plot ext is in requested multi-case plot types if (web_data.plot_ext in multi_case_plots.keys()) and (var in mvars): - for var in multi_case_plots[web_data.plot_ext]: - - #Check if this is compared to observations - if (self.compare_obs) and (var in self.var_obs_dict): - #if var in self.var_obs_dict: - - #Initialize Ordered Dictionary for multi case plot type: - if ptype not in multi_plot_html_info: - multi_plot_html_info[ptype] = OrderedDict() - #End if - - #Initialize Ordered Dictionary for category: - if category not in multi_plot_html_info[ptype]: - multi_plot_html_info[ptype][category] = OrderedDict() - #End if - - if var not in multi_plot_html_info[ptype][category]: - multi_plot_html_info[ptype][category][var] = OrderedDict() - #End if - - p = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" - if season not in multi_plot_html_info[ptype][category][var]: - multi_plot_html_info[ptype][category][var][season] = p - #End if - - else: - #Initialize Ordered Dictionary for multi case plot type: - if ptype not in multi_plot_html_info: - multi_plot_html_info[ptype] = OrderedDict() - #End if - - #Initialize Ordered Dictionary for category: - if category not in multi_plot_html_info[ptype]: - multi_plot_html_info[ptype][category] = OrderedDict() - #End if - - if var not in multi_plot_html_info[ptype][category]: - multi_plot_html_info[ptype][category][var] = OrderedDict() - #End if - - p = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" - if season not in multi_plot_html_info[ptype][category][var]: - multi_plot_html_info[ptype][category][var][season] = p - #End if - #End if (comapre obs) - - #End for (var) + + #Initialize Ordered Dictionary for multi case plot type: + if ptype not in multi_plot_html_info: + multi_plot_html_info[ptype] = OrderedDict() + #End if + #Initialize Ordered Dictionary for category: + if category not in multi_plot_html_info[ptype]: + multi_plot_html_info[ptype][category] = OrderedDict() + #End if + if var not in multi_plot_html_info[ptype][category]: + multi_plot_html_info[ptype][category][var] = OrderedDict() + #End if + p = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" + if season not in multi_plot_html_info[ptype][category][var]: + multi_plot_html_info[ptype][category][var][season] = p + #End if #End if (variable in multi-case plot variables) #End if multi-case multi-plots @@ -653,15 +596,12 @@ def create_website(self): if ptype not in multi_mean_html_info: multi_mean_html_info[ptype] = OrderedDict() #End if - if category not in multi_mean_html_info[ptype]: multi_mean_html_info[ptype][category] = OrderedDict() #End if - if var not in multi_mean_html_info[ptype][category]: multi_mean_html_info[ptype][category][var] = OrderedDict() #End if - p = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" if season not in multi_mean_html_info[ptype][category][var]: multi_mean_html_info[ptype][category][var][season] = p @@ -678,7 +618,6 @@ def create_website(self): self.__case_web_paths[web_data.case]['assets_dir'].mkdir(exist_ok=True) #Move file to assets directory: - #if not web_data.data.is_file(): try: shutil.copy(web_data.data, web_data.asset_path) except: @@ -733,8 +672,8 @@ def create_website(self): float_format='{:6g}'.format) #Construct amwg_table.html - rend_kwarg_dict = {"title": main_title, "case1": case1, - "case2": data_name, + rend_kwarg_dict = {"title": main_title, "case_name": case1, + "base_name": data_name, "case_yrs": case_yrs, "base_name": data_name, "baseline_yrs": baseline_yrs, @@ -764,8 +703,7 @@ def create_website(self): else: rend_kwarg_dict["plot_types"] = plot_type_html if web_data.case == data_name: - #case1 = case_name - rend_kwarg_dict["case1"] = case_name + rend_kwarg_dict["case_name"] = case_names[0] table_rndr = table_tmpl.render(rend_kwarg_dict) @@ -781,9 +719,10 @@ def create_website(self): #Construct mean_table.html mean_table_tmpl = jinenv.get_template('template_mean_tables.html') + #Reuse the rend_kwarg_dict, but ignore certain keys #since all others are the same - new_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'table_name', 'table_html'}} + new_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'table_name', 'table_html'}} if main_site_path: plot_types = multi_plot_type_html @@ -803,6 +742,7 @@ def create_website(self): else: #Plot image plot_types = plot_type_html + #Ensure that these will ignore any multi-case pages if they exist if "main_website" not in str(web_data.html_file): #Create output HTML file path: img_pages_dir = self.__case_web_paths[web_data.case]['img_pages_dir'] @@ -810,19 +750,22 @@ def create_website(self): img_data = [os.path.relpath(web_data.asset_path, start=img_pages_dir), web_data.asset_path.stem] + rend_kwarg_dict = {"title": main_title, + "var_title": web_data.name, + "season_title": web_data.season, + "case_name": web_data.case, + "case_yrs": case_yrs, + "base_name": data_name, + "baseline_yrs": baseline_yrs, + "plottype_title": web_data.plot_type, + "imgs": img_data, + "mydata": mean_html_info[web_data.plot_type], + "plot_types": plot_types, + "multi": multi_layout} + tmpl = jinenv.get_template('template.html') #Set template - rndr = tmpl.render(title=main_title, - var_title=web_data.name, - season_title=web_data.season, - plottype_title=web_data.plot_type, - imgs=img_data, - case1=web_data.case, - case2=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=mean_html_info[web_data.plot_type], - plot_types=plot_types, - multi=multi_layout) #The template rendered + + rndr = tmpl.render(rend_kwarg_dict) #The template rendered #Write HTML file: with open(web_data.html_file, 'w', encoding='utf-8') as ofil: @@ -836,15 +779,11 @@ def create_website(self): #Construct individual plot type mean_diag html files, if they don't #already exist: mean_tmpl = jinenv.get_template('template_mean_diag.html') - mean_rndr = mean_tmpl.render(title=main_title, - case1=web_data.case, - case2=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=mean_html_info[web_data.plot_type], - curr_type=web_data.plot_type, - plot_types=plot_types, - multi=multi_layout) + + #Remove keys from main dictionary for this html page + templ_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'imgs', 'var_title', 'season_title'}} + + mean_rndr = mean_tmpl.render(templ_rend_kwarg_dict) #Write mean diagnostic plots HTML file: with open(mean_ptype_file,'w', encoding='utf-8') as ofil: @@ -860,18 +799,11 @@ def create_website(self): #Construct individual plot type mean_diag html files, if they don't #already exist: plot_page_tmpl = jinenv.get_template('template_var.html') - plot_page_rndr = plot_page_tmpl.render(title=main_title, - var_title=web_data.name, - season_title=web_data.season, - plottype_title=web_data.plot_type, - case1=web_data.case, - case2=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=mean_html_info[web_data.plot_type], - curr_type=web_data.plot_type, - plot_types=plot_types, - multi=multi_layout) + + #Remove key from main dictionary for this html page + templ_var_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'imgs'}} + + plot_page_rndr = plot_page_tmpl.render(templ_var_rend_kwarg_dict) #Write mean diagnostic plots HTML file: with open(mean_ptype_plot_page,'w', encoding='utf-8') as ofil: @@ -894,12 +826,13 @@ def create_website(self): #Construct index.html index_title = "AMP Diagnostics Prototype" index_tmpl = jinenv.get_template('template_index.html') + index_rndr = index_tmpl.render(title=index_title, - case1=web_data.case, - case2=data_name, + case_name=web_data.case, + base_name=data_name, case_yrs=case_yrs, baseline_yrs=baseline_yrs, - plot_types=plot_types, #plot_type_html + plot_types=plot_types, multi=multi_layout) #Write Mean diagnostics index HTML file: @@ -961,20 +894,26 @@ def create_website(self): table_dict[key] = multi_table_html_info[key] #End for + #Construct amwg_table.html + rend_kwarg_dict = {"title": main_title, "case_name": web_data.case, + "case_yrs": case_yrs, + "base_name": data_name, + "baseline_yrs": baseline_yrs, + "amwg_tables": table_dict, + "table_name": web_data.name, + "plot_types": plot_type_html, + "multi_head": True, + "multi": False, + "case_sites": case_sites} + if not mean_table_file.exists(): #Construct mean_table.html mean_table_tmpl = jinenv.get_template('template_mean_tables.html') - mean_table_rndr = mean_table_tmpl.render(title=main_title, - case1=web_data.case, - case2=data_name, - case_yrs=case_yrs, - base_name=data_name, - baseline_yrs=baseline_yrs, - amwg_tables=table_dict, - plot_types=plot_type_html, - multi_head=True, - multi=False, - case_sites=case_sites) + + #Remove key from main dictionary for this html page + tmpl_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'table_name'}} + + mean_table_rndr = mean_table_tmpl.render(tmpl_rend_kwarg_dict) #Write mean diagnostic tables HTML file: with open(mean_table_file, 'w', encoding='utf-8') as ofil: @@ -987,35 +926,15 @@ def create_website(self): table_html = web_data.data.to_html(index=False, border=1, justify='center', float_format='{:6g}'.format) - #Construct amwg_table.html - table_keys = [web_data.case,data_name,"case_comparison"] - - case_table_dict = {} - for key in table_keys: - if self.compare_obs: - if (key != "Obs") and (key != "case_comparison"): - case_table_dict[key] = multi_table_html_info[key] - else: - case_table_dict[key] = multi_table_html_info[key] - #End for - indv_html = table_pages_dir_indv / f"amwg_table_{web_data.name}.html" if not indv_html.exists(): table_tmpl = jinenv.get_template('template_table.html') - table_rndr = table_tmpl.render(title=main_title, - case1=web_data.case, - case2=data_name, - case_yrs=case_yrs, - base_name=data_name, - baseline_yrs=baseline_yrs, - amwg_tables=case_table_dict, - plot_types=plot_type_html, - table_name=web_data.name, - table_html=table_html, - multi_head=True, - multi=False, - case_sites=case_sites) + + tmpl_rend_kwarg_dict = rend_kwarg_dict + rend_kwarg_dict["table_html"] = table_html + + table_rndr = table_tmpl.render(rend_kwarg_dict) #Write mean diagnostic tables HTML file: with open(indv_html, 'w', encoding='utf-8') as ofil: @@ -1028,28 +947,23 @@ def create_website(self): table_html = web_data.data.to_html(index=False, border=1, justify='center', float_format='{:6g}'.format) + rend_kwarg_dict["table_html"] = table_html + for case_name in case_names: table_pages_dir_sp = self.__case_web_paths[case_name]['table_pages_dir'] table_key = [case_name,data_name,"case_comparison"] - # [web_data.case,data_name,"case_comparison"] + base_table_dict = {key: multi_table_html_info[key] for key in table_key} + rend_kwarg_dict["case_name"] = case_name + rend_kwarg_dict["amwg_tables"] = base_table_dict + sp_html = table_pages_dir_sp / f"amwg_table_{data_name}.html" if not sp_html.exists(): + table_tmpl = jinenv.get_template('template_table.html') - table_rndr = table_tmpl.render(title=main_title, - case1=case_name, - case2=data_name, - case_yrs=case_yrs, - base_name=data_name, - baseline_yrs=baseline_yrs, - amwg_tables=base_table_dict, - plot_types=plot_type_html, - table_name=web_data.name, - table_html=table_html, - multi_head=True, - multi=False, - case_sites=case_sites) + + table_rndr = table_tmpl.render(rend_kwarg_dict) with open(sp_html, 'w', encoding='utf-8') as ofil: ofil.write(table_rndr) @@ -1094,7 +1008,6 @@ def create_website(self): #Check if variable is in desired multi-case plot #and if plot_type is in given multi-case plot set: if (var in mvars) and (ext in multi_case_plots): - #Move file to assets directory: if not web_data.data.is_file(): shutil.copy(web_data.data, web_data.asset_path) @@ -1106,27 +1019,31 @@ def create_website(self): start=main_site_img_path), multi_plot_page] + rend_kwarg_dict = {"title": main_title, + "var_title": var, + "season_title": season, + "case_yrs": case_yrs, + "base_name": data_name, + "baseline_yrs": baseline_yrs, + "plottype_title": ptype, + "imgs": img_data, + "mydata": multi_plot_html_info[ptype], + "plot_types": multi_plot_type_html, + "multi": multi_layout, + "case_sites": case_sites} + multimean = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" if not (img_pages_dir / multimean).exists(): + tmpl = jinenv.get_template('template_multi_case.html') - rndr = tmpl.render(title=main_title, - var_title=var, - season_title=season, - plottype_title=ptype, - imgs=img_data, - base_name=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=multi_plot_html_info[ptype], - plot_types=multi_plot_type_html, - multi=multi_layout, - case_sites=case_sites) + + rndr = tmpl.render(rend_kwarg_dict) #Write HTML file: with open(img_pages_dir / multimean, 'w', encoding='utf-8') as ofil: ofil.write(rndr) - #End if (multimean) + #End if (multimean) #Check if the mean plot type and var page exists for this case: img_pages_dir = self.__case_web_paths["multi-case"]['img_pages_dir'] @@ -1135,21 +1052,14 @@ def create_website(self): if not mean_ptype_plot_page.exists(): + #Remove key from main dictionary for this html page + templ_var_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'imgs'}} + #Construct individual plot type mean_diag #html files, if they don't already exist: page_tmpl = jinenv.get_template('template_multi_case_var.html') - plot_page_rndr = page_tmpl.render(title=main_title, - var_title=var, - season_title=season, - plottype_title=ptype, - base_name=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=multi_plot_html_info[ptype], - curr_type=ptype, - plot_types=multi_plot_type_html, - multi=multi_layout, - case_sites=case_sites) + + plot_page_rndr = page_tmpl.render(templ_var_rend_kwarg_dict) #Write mean diagnostic plots HTML file: with open(mean_ptype_plot_page,'w', encoding='utf-8') as ofil: @@ -1161,18 +1071,14 @@ def create_website(self): mean_ptype_file = main_site_img_path / multi_mean if not mean_ptype_file.exists(): + #Remove keys from main dictionary for this html page + templ_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'imgs', 'var_title', 'season_title'}} + #Construct individual plot type mean_diag #html files, if they don't already exist: tmp = jinenv.get_template('template_multi_case_mean_diag.html') - mean_rndr = tmp.render(title=main_title, - base_name=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=multi_plot_html_info[ptype], - curr_type=ptype, - plot_types=multi_plot_type_html, - multi=multi_layout, - case_sites=case_sites) + + mean_rndr = tmp.render(templ_rend_kwarg_dict) #Write mean diagnostic plots HTML file: with open(mean_ptype_file,'w', encoding='utf-8') as ofil: @@ -1181,6 +1087,7 @@ def create_website(self): #End if (mean_ptype exists) #Loop over any non multi-case multi-plot scenarios + #ie multi-case Taylor Diagrams and multi-case QBO if ext not in multi_case_dict: #Move file to assets directory: if not web_data.data.is_file(): @@ -1193,21 +1100,24 @@ def create_website(self): start=main_site_img_path), multi_plot_page] + rend_kwarg_dict = {"title": main_title, + "var_title": var, + "season_title": season, + "case_yrs": case_yrs, + "base_name": data_name, + "baseline_yrs": baseline_yrs, + "plottype_title": ptype, + "imgs": img_data, + "mydata": multi_mean_html_info[ptype], + "plot_types": multi_plot_type_html, + "multi": multi_layout, + "case_sites": case_sites} + multimean = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" if not (img_pages_dir / multimean).exists(): tmpl = jinenv.get_template('template_multi_case.html') - rndr = tmpl.render(title=main_title, - var_title=var, - season_title=season, - plottype_title=ptype, - imgs=img_data, - base_name=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=multi_mean_html_info[ptype], - plot_types=multi_plot_type_html, - multi=multi_layout, - case_sites=case_sites) + + rndr = tmpl.render(rend_kwarg_dict) #Write HTML file: with open(img_pages_dir / multimean, @@ -1222,21 +1132,14 @@ def create_website(self): if not mean_ptype_plot_page.exists(): + #Remove key from main dictionary for this html page + templ_var_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'imgs'}} + #Construct individual plot type mean_diag #html files, if they don't already exist: page_tmpl = jinenv.get_template('template_multi_case_var.html') - plot_page_rndr = page_tmpl.render(title=main_title, - var_title=var, - season_title=season, - plottype_title=ptype, - base_name=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=multi_mean_html_info[ptype], - curr_type=ptype, - plot_types=multi_plot_type_html, - multi=multi_layout, - case_sites=case_sites) + + plot_page_rndr = page_tmpl.render(templ_var_rend_kwarg_dict) #Write mean diagnostic plots HTML file: with open(mean_ptype_plot_page,'w', encoding='utf-8') as ofil: @@ -1248,18 +1151,14 @@ def create_website(self): mean_ptype_file = main_site_img_path / multi_mean if not mean_ptype_file.exists(): + #Remove keys from main dictionary for this html page + templ_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'imgs', 'var_title', 'season_title'}} + #Construct individual plot type mean_diag #html files, if they don't already exist: tmp = jinenv.get_template('template_multi_case_mean_diag.html') - mean_rndr = tmp.render(title=main_title, - base_name=data_name, - case_yrs=case_yrs, - baseline_yrs=baseline_yrs, - mydata=multi_mean_html_info[ptype], - curr_type=ptype, - plot_types=multi_plot_type_html, - multi=multi_layout, - case_sites=case_sites) + + mean_rndr = tmp.render(templ_rend_kwarg_dict) #Write mean diagnostic plots HTML file: with open(mean_ptype_file,'w', encoding='utf-8') as ofil: diff --git a/lib/website_templates/template.html b/lib/website_templates/template.html index 151be93c2..025810630 100644 --- a/lib/website_templates/template.html +++ b/lib/website_templates/template.html @@ -35,8 +35,8 @@
-

Test Case:

{{ case1 }}

- years: {{ case_yrs }}


-

Baseline Case:

{{ case2 }}

- years: {{ baseline_yrs }}

+

Test Case:

{{ case_name }}

- years: {{ case_yrs }}


+

Baseline Case:

{{ base_name }}

- years: {{ baseline_yrs }}

diff --git a/lib/website_templates/template_index.html b/lib/website_templates/template_index.html index 9f19054ad..076310ee7 100644 --- a/lib/website_templates/template_index.html +++ b/lib/website_templates/template_index.html @@ -28,8 +28,8 @@
-

Test Case:

{{ case1 }}

- years: {{ case_yrs }}


-

Baseline Case:

{{ case2 }}

- years: {{ baseline_yrs }}

+

Test Case:

{{ case_name }}

- years: {{ case_yrs }}


+

Baseline Case:

{{ base_name }}

- years: {{ baseline_yrs }}

diff --git a/lib/website_templates/template_mean_diag.html b/lib/website_templates/template_mean_diag.html index 9f9f55c2c..e688f3a2a 100644 --- a/lib/website_templates/template_mean_diag.html +++ b/lib/website_templates/template_mean_diag.html @@ -35,13 +35,13 @@
-

Test Case:

{{ case1 }}

- years: {{ case_yrs }}


-

Baseline Case:

{{ case2 }}

- years: {{ baseline_yrs }}

+

Test Case:

{{ case_name }}

- years: {{ case_yrs }}


+

Baseline Case:

{{ base_name }}

- years: {{ baseline_yrs }}

-

{{ curr_type }} Plots

+

{{ plottype_title }} Plots

@@ -52,7 +52,7 @@

{{ category }}

{% for var_name, ptype_seas in var_seas.items() %}
- {{var_name}}
+ {{var_name}}
{% endfor %}
diff --git a/lib/website_templates/template_mean_tables.html b/lib/website_templates/template_mean_tables.html index 5a47f5460..686a097c4 100644 --- a/lib/website_templates/template_mean_tables.html +++ b/lib/website_templates/template_mean_tables.html @@ -50,8 +50,8 @@

{{ base_name }}

- years: {{ baseline_yrs }}

{% else %}
-

Test Case:

{{ case1 }}

- years: {{ case_yrs }}


-

Baseline Case:

{{ case2 }}

- years: {{ baseline_yrs }}

+

Test Case:

{{ case_name }}

- years: {{ case_yrs }}


+

Baseline Case:

{{ base_name }}

- years: {{ baseline_yrs }}

{% endif %}
diff --git a/lib/website_templates/template_multi_case_index.html b/lib/website_templates/template_multi_case_index.html index 411eb5ed2..f4ec173f5 100644 --- a/lib/website_templates/template_multi_case_index.html +++ b/lib/website_templates/template_multi_case_index.html @@ -24,14 +24,14 @@ -

Click on case name for that ADF vs baseline page

+

Click on case name for that ADF case vs baseline page

{% for case_name, case_dtls in case_sites.items() %} -

Test Case {{ loop.index }}:

- {{ case_name }}

+

Test Case {{ loop.index }}:

+

{{ case_name }}

- years: {{ case_dtls[1] }} - {{ case_dtls[2] }}


{% endfor %} -

Baseline Case:

- {{ base_name }}

+

Baseline Case:

+

{{ base_name }}

- years: {{ baseline_yrs }}

diff --git a/lib/website_templates/template_multi_case_mean_diag.html b/lib/website_templates/template_multi_case_mean_diag.html index d0f1421ea..832348f9f 100644 --- a/lib/website_templates/template_multi_case_mean_diag.html +++ b/lib/website_templates/template_multi_case_mean_diag.html @@ -44,7 +44,7 @@

- years: {{ baseline_yrs }}

-

All Case Comparison - {{ curr_type }} Plots

+

All Case Comparison - {{ plottype_title }} Plots

@@ -55,7 +55,7 @@

{{ category }}

{% for var_name, ptype_seas in var_seas.items() %}
- {{ var_name }}
+ {{ var_name }}
{% endfor %}
diff --git a/lib/website_templates/template_table.html b/lib/website_templates/template_table.html index d3161dc07..250a899a5 100644 --- a/lib/website_templates/template_table.html +++ b/lib/website_templates/template_table.html @@ -50,8 +50,8 @@

{{ base_name }}

- years: {{ baseline_yrs }}

{% else %}
-

Test Case:

{{ case1 }}

- years: {{ case_yrs }}


-

Baseline Case:

{{ case2 }}

- years: {{ baseline_yrs }}

+

Test Case:

{{ case_name }}

- years: {{ case_yrs }}


+

Baseline Case:

{{ base_name }}

- years: {{ baseline_yrs }}

{% endif %}
diff --git a/lib/website_templates/template_var.html b/lib/website_templates/template_var.html index 1ff0aa029..b63d2cca0 100644 --- a/lib/website_templates/template_var.html +++ b/lib/website_templates/template_var.html @@ -35,8 +35,8 @@
-

Test Case:

{{ case1 }}

- years: {{ case_yrs }}


-

Baseline Case:

{{ case2 }}

- years: {{ baseline_yrs }}

+

Test Case:

{{ case_name }}

- years: {{ case_yrs }}


+

Baseline Case:

{{ base_name }}

- years: {{ baseline_yrs }}

From d96021b2b6a8cb21ce287184817f9fbf3c76246c Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 30 Mar 2023 11:05:43 -0600 Subject: [PATCH 09/12] Clean up source_code_tests warnings --- lib/adf_web.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/adf_web.py b/lib/adf_web.py index c2e0cae2e..30d5f5314 100644 --- a/lib/adf_web.py +++ b/lib/adf_web.py @@ -395,9 +395,9 @@ def create_website(self): #var_list should be a list of all vars for each plot map extentions #var is iterative for all plot map extensions for multi_var_list in [multi_case_plots[ext] for ext in multi_case_plots]: - for multi_var in multi_var_list: - if ((self.compare_obs) and (multi_var in self.var_obs_dict)) or (not self.compare_obs): - mvars.append(multi_var) + for var in multi_var_list: + if (self.compare_obs and var in self.var_obs_dict) or (not self.compare_obs): + mvars.append(var) #Create multi-case site: #Make a dictionary for plot type extensions for given plot type @@ -672,8 +672,8 @@ def create_website(self): float_format='{:6g}'.format) #Construct amwg_table.html - rend_kwarg_dict = {"title": main_title, "case_name": case1, - "base_name": data_name, + rend_kwarg_dict = {"title": main_title, + "case_name": case1, "case_yrs": case_yrs, "base_name": data_name, "baseline_yrs": baseline_yrs, @@ -722,7 +722,7 @@ def create_website(self): #Reuse the rend_kwarg_dict, but ignore certain keys #since all others are the same - new_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'table_name', 'table_html'}} + new_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'table_name', 'table_html'}} if main_site_path: plot_types = multi_plot_type_html @@ -779,9 +779,9 @@ def create_website(self): #Construct individual plot type mean_diag html files, if they don't #already exist: mean_tmpl = jinenv.get_template('template_mean_diag.html') - + #Remove keys from main dictionary for this html page - templ_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'imgs', 'var_title', 'season_title'}} + templ_rend_kwarg_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'imgs', 'var_title', 'season_title'}} mean_rndr = mean_tmpl.render(templ_rend_kwarg_dict) @@ -1034,7 +1034,7 @@ def create_website(self): multimean = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" if not (img_pages_dir / multimean).exists(): - + tmpl = jinenv.get_template('template_multi_case.html') rndr = tmpl.render(rend_kwarg_dict) From d3f4f431077cca236c3d90679990d3a607b20595 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 30 Mar 2023 12:12:22 -0600 Subject: [PATCH 10/12] Update adf_web.py Need second set of parenthesis to get correct if comparison --- lib/adf_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/adf_web.py b/lib/adf_web.py index 30d5f5314..6549e1566 100644 --- a/lib/adf_web.py +++ b/lib/adf_web.py @@ -396,7 +396,7 @@ def create_website(self): #var is iterative for all plot map extensions for multi_var_list in [multi_case_plots[ext] for ext in multi_case_plots]: for var in multi_var_list: - if (self.compare_obs and var in self.var_obs_dict) or (not self.compare_obs): + if ((self.compare_obs) and (var in self.var_obs_dict)) or (not self.compare_obs): mvars.append(var) #Create multi-case site: From 793cd79899ecf2f988d6e90ea6742e7281d6d642 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 21 Apr 2023 10:45:17 -0600 Subject: [PATCH 11/12] Fix minor issues found during second code review. --- lib/adf_info.py | 25 ++++++++++++------------- lib/adf_web.py | 12 ++++++------ lib/plotting_functions.py | 25 +++++++++++++++---------- scripts/analysis/amwg_table.py | 34 +++++++++++++++++----------------- 4 files changed, 50 insertions(+), 46 deletions(-) diff --git a/lib/adf_info.py b/lib/adf_info.py index 7b06daa95..ddda253d9 100644 --- a/lib/adf_info.py +++ b/lib/adf_info.py @@ -123,10 +123,12 @@ def __init__(self, config_file, debug=False): case_names = self.get_cam_info('cam_case_name', required=True) #Grab test case nickname(s) - test_nicknames = self.get_cam_info('case_nickname', required=True) + test_nicknames = self.get_cam_info('case_nickname') if test_nicknames is None: for idx,case_name in enumerate(case_names): test_nicknames[idx] = case_name + #End for + #End if #Check if a CAM vs AMWG obs comparison is being performed: if self.__compare_obs: @@ -180,6 +182,7 @@ def __init__(self, config_file, debug=False): base_nickname = self.get_baseline_info('case_nickname') if base_nickname == None: base_nickname = data_name + #End if data_name += f"_{syear_baseline}_{eyear_baseline}" #End if @@ -233,7 +236,7 @@ def __init__(self, config_file, debug=False): eyears[case_idx] = int(case_climo_yrs[-1]) #End if #End if - + #Append climo years to case directory case_name += f"_{syears[case_idx]}_{eyears[case_idx]}" @@ -246,7 +249,7 @@ def __init__(self, config_file, debug=False): first_case_dir = direc_name #End if #End for - + #Initialize case climo years: self.__syears = syears self.__eyears = eyears @@ -407,24 +410,20 @@ def climo_yrs(self): @property def case_nicknames(self): """Return the test case and baseline nicknames to the user if requested.""" - - #Note that copies are needed in order to avoid having a script mistakenly - #modify these variables, as they are mutable and thus passed by reference: + #Note that a copy is needed in order to avoid having a script mistakenly + #modify this variable, as it is mutable and thus passed by reference: test_nicknames = copy.copy(self.__test_nicknames) - base_nickname = copy.copy(self.__base_nickname) - return {"test_nicknames":test_nicknames,"base_nickname":base_nickname} + return {"test_nicknames":test_nicknames,"base_nickname":self.__base_nickname} # Create property needed to return the multi-case directories to scripts: @property def main_site_paths(self): """Return the directories for multi-case diags if applicable.""" - main_site_path = copy.copy(self.__main_site_path) #Send copies so a script doesn't modify the original - main_site_assets_path = copy.copy(self.__main_site_assets_path) - main_site_img_path = copy.copy(self.__main_site_img_path) - return {"main_site_path":main_site_path, "main_site_assets_path":main_site_assets_path, - "main_site_img_path":main_site_img_path} + return {"main_site_path":self.__main_site_path, + "main_site_assets_path":self.__main_site_assets_path, + "main_site_img_path":self.__main_site_img_path} ######### diff --git a/lib/adf_web.py b/lib/adf_web.py index 6549e1566..e6e9ec064 100644 --- a/lib/adf_web.py +++ b/lib/adf_web.py @@ -391,8 +391,8 @@ def create_website(self): if multi_case_plots: #Grab all variables for each multi-case plot type mvars = [] - #ext are plot type extentions (keys for multi-case plots) - #var_list should be a list of all vars for each plot map extentions + #ext are plot type extensions (keys for multi-case plots) + #var_list should be a list of all vars for each plot map extensions #var is iterative for all plot map extensions for multi_var_list in [multi_case_plots[ext] for ext in multi_case_plots]: for var in multi_var_list: @@ -722,7 +722,7 @@ def create_website(self): #Reuse the rend_kwarg_dict, but ignore certain keys #since all others are the same - new_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'table_name', 'table_html'}} + new_dict = {k: rend_kwarg_dict[k] for k in rend_kwarg_dict.keys() - {'table_name', 'table_html'}} if main_site_path: plot_types = multi_plot_type_html @@ -1030,7 +1030,7 @@ def create_website(self): "mydata": multi_plot_html_info[ptype], "plot_types": multi_plot_type_html, "multi": multi_layout, - "case_sites": case_sites} + "case_sites": case_sites} multimean = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" if not (img_pages_dir / multimean).exists(): @@ -1111,7 +1111,7 @@ def create_website(self): "mydata": multi_mean_html_info[ptype], "plot_types": multi_plot_type_html, "multi": multi_layout, - "case_sites": case_sites} + "case_sites": case_sites} multimean = f"plot_page_multi_case_{var}_{season}_{ptype}_Mean.html" if not (img_pages_dir / multimean).exists(): @@ -1204,4 +1204,4 @@ def create_website(self): #++++++++++++++++++++ #End Class definition -#++++++++++++++++++++ \ No newline at end of file +#++++++++++++++++++++ diff --git a/lib/plotting_functions.py b/lib/plotting_functions.py index 397b58d86..7ae6db8e6 100644 --- a/lib/plotting_functions.py +++ b/lib/plotting_functions.py @@ -1658,7 +1658,7 @@ def multi_latlon_plots(wks, ptype, case_names, nicknames, multi_dict, web_catego multi_dict: ordered dictionary of difference data for each var, test case, and season multi_dict[var][case_name][s] - + web_category: variable category @@ -1666,31 +1666,38 @@ def multi_latlon_plots(wks, ptype, case_names, nicknames, multi_dict, web_catego Needed to test if redo_plot is called in config yaml file """ - + nplots = len(nicknames[0]) if nplots > 2: ncols = 3 else: ncols = 2 + #End if - # Check redo_plot. If set to True: remove old plot, if it already exists: + #Check redo_plot. If set to True: remove old plot, if it already exists: redo_plot = adfobj.get_basic_info('redo_plot') + #Determine needed matplotlib normalization function: + normfunc,_ = use_this_norm() + #Try and format spacing based on number of cases # NOTE: ** this will have to change if figsize or dpi change ** if nplots < 4: hspace = -1.0 else: - hspace = -0.85 + hspace = -0.85 + #End if nrows = int(np.ceil(nplots/ncols)) if nrows == 1: y_title = 0.265 else: y_title = 0.315 + #End if if nrows < 2: nrows = 2 + #End if # specify the central longitude for the plot central_longitude = get_central_longitude(adfobj) @@ -1719,8 +1726,6 @@ def multi_latlon_plots(wks, ptype, case_names, nicknames, multi_dict, web_catego #Set figure title plt.suptitle(f'All Case Comparison (Test - Baseline) {var}: {season}\n', fontsize=16, y=y_title)#y=0.325 y=0.225 - - normfunc,_ = use_this_norm() count = 0 img = [] @@ -1746,8 +1751,8 @@ def multi_latlon_plots(wks, ptype, case_names, nicknames, multi_dict, web_catego cmap = multi_dict[var][case_names[count]][season]["vres"]['diff_colormap'] - img.append(axs[r,c].contourf(lons, lats, mwrap, levels=levelsdiff, - cmap=cmap, norm=normdiff, + img.append(axs[r,c].contourf(lons, lats, mwrap, levels=levelsdiff, + cmap=cmap, norm=normdiff, transform=proj)) #Set individual plot titles (case name/nickname) @@ -1772,10 +1777,10 @@ def multi_latlon_plots(wks, ptype, case_names, nicknames, multi_dict, web_catego fig.colorbar(img[-1], ax=axs.ravel().tolist(), orientation='horizontal', aspect=20, shrink=.5, location="bottom", anchor=(0.5,-0.3), extend='both') - + #Clean up the spacing a bit plt.subplots_adjust(wspace=0.3, hspace=hspace) - + fig.savefig(wks / file_name, bbox_inches='tight', dpi=300) adfobj.add_website_data(wks / file_name, file_name, case_names[0], plot_ext="global_latlon_map", diff --git a/scripts/analysis/amwg_table.py b/scripts/analysis/amwg_table.py index 99a8a7088..cdd860d65 100644 --- a/scripts/analysis/amwg_table.py +++ b/scripts/analysis/amwg_table.py @@ -54,7 +54,7 @@ def amwg_table(adf): #Import necessary modules: from adf_base import AdfError - + #Additional information: @@ -157,7 +157,7 @@ def amwg_table(adf): #Generate input file path: input_location = Path(input_ts_locs[case_idx]) - + #Add output paths for csv files csv_locs.append(output_locs[case_idx]) @@ -238,12 +238,12 @@ def amwg_table(adf): # In order to get correct statistics, average to annual or seasonal data = data.groupby('time.year').mean(dim='time') # this should be fast b/c time series should be in memory # NOTE: data will now have a 'year' dimension instead of 'time' - + # create a dataframe: cols = ['variable', 'unit', 'mean', 'sample size', 'standard dev.', 'standard error', '95% CI', 'trend', 'trend p-value'] - - # These get written to our output file: + + # These get written to our output file: stats_list = _get_row_vals(data) row_values = [var, unit_str] + stats_list @@ -271,8 +271,8 @@ def amwg_table(adf): # In order to get correct statistics, average to annual or seasonal data = data.groupby('time.year').mean(dim='time') # this should be fast b/c time series should be in memory # NOTE: data will now have a 'year' dimension instead of 'time' - # These get written to our output file: - stats_list = _get_row_vals(data) + # These get written to our output file: + stats_list = _get_row_vals(data) row_values = [var, restom_units] + stats_list # col (column) values declared above @@ -377,16 +377,16 @@ def _spatial_average(indata): ##### -def _get_row_vals(data): +def _get_row_vals(data): # Now that data is (time,), we can do our simple stats: - + data_mean = data.data.mean() #Conditional Formatting depending on type of float if np.abs(data_mean) < 1: formatter = ".3g" else: formatter = ".3f" - + data_sample = len(data) data_std = data.std() data_sem = data_std / data_sample @@ -407,10 +407,10 @@ def _df_comp_table(adf, output_location, base_output_location, case_names): """ Function to build case vs baseline AMWG table ----- - - Read in table data and create side by side comaprison table + - Read in table data and create side by side comparison table - Write output to csv file and add to website """ - + output_csv_file_comp = output_location / "amwg_table_comp.csv" case = output_location/f"amwg_table_{case_names[0]}.csv" @@ -428,7 +428,7 @@ def _df_comp_table(adf, output_location, base_output_location, case_names): df_comp = pd.DataFrame(dtype=object) df_comp[['variable','unit','case']] = df_merge[['variable','unit_x','mean_x']] df_comp['baseline'] = df_merge[['mean_y']] - + diffs = df_comp['case'].values-df_comp['baseline'].values df_comp['diff'] = [f'{i:.3g}' if np.abs(i) < 1 else f'{i:.3f}' for i in diffs] @@ -443,7 +443,7 @@ def _df_comp_table(adf, output_location, base_output_location, case_names): def _df_multi_comp_table(adf, csv_locs, case_names, test_nicknames): """ - Function to build all case comparison AMWG table + Function to build comparison AMWG table for all cases ------ - Read in each previously made table from file and compile full comparison. @@ -464,10 +464,10 @@ def _df_multi_comp_table(adf, csv_locs, case_names, test_nicknames): df_base = pd.read_csv(baseline) #Read all test cases and add to table - for i,val in enumerate(csv_locs[:-1]): + for i,val in enumerate(csv_locs[:-1]): case = str(val)+f"/amwg_table_{case_names[i]}.csv" df_case = pd.read_csv(case) - + #If no custom nicknames, shorten column name to case number if test_nicknames[i] == case_names[i]: df_comp[['variable','unit',f"case {i+1}"]] = df_case[['variable','unit','mean']] @@ -501,7 +501,7 @@ def _df_multi_comp_table(adf, csv_locs, case_names, test_nicknames): formatter = ".3f" #Replace value in dataframe df_comp.at[idx,col]= f'{df_comp[col][idx]:{formatter}} ({(df_comp[col][idx]-df_base["mean"][idx]):{formatter}})' - + #Finally, write data to csv df_comp.to_csv(output_csv_file_comp, header=cols_comp, index=False) From b5a8280f02c6fd7a4977c64c742f12897987c80b Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 25 Apr 2023 08:39:30 -0600 Subject: [PATCH 12/12] Fix bugs found during testing. --- config_cam_baseline_example.yaml | 6 +++--- lib/adf_info.py | 5 ++--- lib/website_templates/template.html | 8 ++++---- lib/website_templates/template_index.html | 4 ++-- lib/website_templates/template_mean_diag.html | 4 ++-- lib/website_templates/template_mean_tables.html | 10 +++++----- lib/website_templates/template_multi_case.html | 6 +++--- lib/website_templates/template_multi_case_index.html | 2 +- .../template_multi_case_mean_diag.html | 4 ++-- lib/website_templates/template_multi_case_var.html | 6 +++--- lib/website_templates/template_table.html | 10 +++++----- lib/website_templates/template_var.html | 8 ++++---- 12 files changed, 36 insertions(+), 37 deletions(-) diff --git a/config_cam_baseline_example.yaml b/config_cam_baseline_example.yaml index 59c9dbff1..2293b9a97 100644 --- a/config_cam_baseline_example.yaml +++ b/config_cam_baseline_example.yaml @@ -65,10 +65,10 @@ user: 'USER-NAME-NOT-SET' diag_basic_info: #History file string to match (eg. cam.h0 or ocn.pop.h.ecosys.nday1) - # Only affects timeseries as everything else uses timeseries + # Only affects timeseries as everything else uses timeseries # Leave off trailing '.' #Default: cam.h0 - hist_str: cam.h0 + hist_str: cam.h0 #Is this a model vs observations comparison? #If "false" or missing, then a model-model comparison is assumed: @@ -206,7 +206,7 @@ diag_cam_climo: # - b.e20.BHIST.f09_g17.20thC.297_05 # - b1850.f19_g17.validation_mct.004 - #case_nickname: + #case_nickname: # - nickname 1 # - "026c" diff --git a/lib/adf_info.py b/lib/adf_info.py index 9a857c1d1..0ade67b94 100644 --- a/lib/adf_info.py +++ b/lib/adf_info.py @@ -122,9 +122,8 @@ def __init__(self, config_file, debug=False): #Grab test case nickname(s) test_nicknames = self.get_cam_info('case_nickname') if test_nicknames is None: - for idx,case_name in enumerate(case_names): - test_nicknames[idx] = case_name - #End for + #If no nicknames exist, then just set to case names + test_nicknames = case_names #End if #Check if a CAM vs AMWG obs comparison is being performed: diff --git a/lib/website_templates/template.html b/lib/website_templates/template.html index 025810630..be92a6d7b 100644 --- a/lib/website_templates/template.html +++ b/lib/website_templates/template.html @@ -11,9 +11,9 @@
@@ -76,4 +76,4 @@

Baseline Case: