From 622b64f3808b02de47d683ff512a6b38a719454a Mon Sep 17 00:00:00 2001 From: Benjamin Berger Date: Wed, 27 Oct 2021 10:34:47 +0200 Subject: [PATCH 1/7] add method sepList Add sepList to get separation between GRB position and positions of catalog, saved in the report file --- morgoth/utils/result_reader.py | 63 +++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/morgoth/utils/result_reader.py b/morgoth/utils/result_reader.py index 7a9372f..7b5b5e5 100644 --- a/morgoth/utils/result_reader.py +++ b/morgoth/utils/result_reader.py @@ -18,6 +18,11 @@ from astropy.coordinates import SkyCoord import astropy.units as unit +from astropy.table import Table +import pandas as pd +import os, ssl + + base_dir = get_env_value("GBM_TRIGGER_DATA_DIR") @@ -295,6 +300,60 @@ def _read_post_equal_weights_file(self, post_equal_weights_file): if dec_err is not None: self._dec_err = dec_err + + + def sepList(self): + + #for the case of certification errror + if (not os.environ.get('PYTHONHTTPSVERIFY', '') and + getattr(ssl, '_create_unverified_context', None)): + ssl._create_default_https_context = ssl._create_unverified_context + + #read in table from website + table_MN = pd.read_html("https://swift.gsfc.nasa.gov/results/transients/BAT_current.html") + df = table_MN[0] + + #delete unwanted string + df.rename(columns={'Peak*': 'Peak'}, inplace=True) + + #filter by peak value + df = df.drop(df[df.Peak=="-"].index) + df = df.astype({"Peak": int}) + df_filtered = df[df['Peak']>400] + + #for table of catalog + table = Table.from_pandas(df_filtered) + #table.show_in_browser(jsviewer=True) + + #transform input in SkyCoord + position=SkyCoord(self._ra*unit.deg, self._dec*unit.deg, frame="icrs") + + #transform table data in SkyCoord + coords = [] + for i in range(len(df_filtered['RA J2000 Degs'])): + ra = table[i]['RA J2000 Degs'] + dec = table[i]['Dec J2000 Degs'] + coords.append(SkyCoord(ra*unit.deg, dec*unit.deg, frame="icrs")) + + #get separation value + separations = [] + for i in coords: + z = i.separation(position) + separations.append(z.to(unit.deg)) + + #for table of separations + table["Separation Degs"] = separations + table.round(3) + table.sort("Separation Degs") + #table.show_in_browser(jsviewer=True) + + #create dictionary + dic = {} + for i in range(len(table["Source Name"])): + dic[table[i]['Source Name']]={"ra":table[i]['RA J2000 Degs'], "dec":table[i]['Dec J2000 Degs'], "separation":table[i]["Separation Degs"]} + + return dic + def _build_report(self): self._report = { @@ -349,7 +408,9 @@ def _build_report(self): "active_time_stop": self._active_time_stop, "used_detectors": self._used_detectors, }, - } + "source comparison": self.sepList() + } + def save_result_yml(self, file_path): with open(file_path, "w") as f: From da8a9c215467c66740ac85b9d23e89c7fceb3423 Mon Sep 17 00:00:00 2001 From: Benjamin Berger Date: Thu, 4 Nov 2021 13:40:43 +0100 Subject: [PATCH 2/7] add method _sep_bright_sources Renamed the previous method and changed code style --- morgoth/utils/result_reader.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/morgoth/utils/result_reader.py b/morgoth/utils/result_reader.py index 7b5b5e5..7d3623c 100644 --- a/morgoth/utils/result_reader.py +++ b/morgoth/utils/result_reader.py @@ -79,6 +79,9 @@ def __init__( trigger_number=self._trigger_number, grb_name=self.grb_name ) + # Check catalog of bright gamma sources and get separation to GRB position + self._sep_bright_sources() + # Create a report containing all the results of the pipeline self._build_report() @@ -302,57 +305,57 @@ def _read_post_equal_weights_file(self, post_equal_weights_file): self._dec_err = dec_err - def sepList(self): - - #for the case of certification errror + def _sep_bright_sources(self): + + #for the case of certification errror if (not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl, '_create_unverified_context', None)): ssl._create_default_https_context = ssl._create_unverified_context - #read in table from website + #read in table from website table_MN = pd.read_html("https://swift.gsfc.nasa.gov/results/transients/BAT_current.html") df = table_MN[0] - #delete unwanted string + #delete unwanted string df.rename(columns={'Peak*': 'Peak'}, inplace=True) - #filter by peak value + #filter by peak value df = df.drop(df[df.Peak=="-"].index) df = df.astype({"Peak": int}) df_filtered = df[df['Peak']>400] - #for table of catalog + #for table of catalog table = Table.from_pandas(df_filtered) #table.show_in_browser(jsviewer=True) - #transform input in SkyCoord + #transform input in SkyCoord position=SkyCoord(self._ra*unit.deg, self._dec*unit.deg, frame="icrs") - #transform table data in SkyCoord + #transform table data in SkyCoord coords = [] for i in range(len(df_filtered['RA J2000 Degs'])): ra = table[i]['RA J2000 Degs'] dec = table[i]['Dec J2000 Degs'] coords.append(SkyCoord(ra*unit.deg, dec*unit.deg, frame="icrs")) - #get separation value + #get separation value separations = [] for i in coords: z = i.separation(position) separations.append(z.to(unit.deg)) - #for table of separations + #for table of separations table["Separation Degs"] = separations table.round(3) table.sort("Separation Degs") #table.show_in_browser(jsviewer=True) - #create dictionary + #create dictionary dic = {} for i in range(len(table["Source Name"])): dic[table[i]['Source Name']]={"ra":table[i]['RA J2000 Degs'], "dec":table[i]['Dec J2000 Degs'], "separation":table[i]["Separation Degs"]} - return dic + self.dic_bright_sources = dic def _build_report(self): @@ -408,7 +411,7 @@ def _build_report(self): "active_time_stop": self._active_time_stop, "used_detectors": self._used_detectors, }, - "source comparison": self.sepList() + "bright sources": self.dic_bright_sources } From b5fa0102bb8ed0a8dc9edef5d0e1a75f77c0f016 Mon Sep 17 00:00:00 2001 From: Benjamin Berger Date: Thu, 4 Nov 2021 14:06:01 +0100 Subject: [PATCH 3/7] Add method _sep_SGRs Method checks catalog of SGRs and calculates the separation to GRB position. Results are saved in _build_report. --- morgoth/utils/result_reader.py | 52 +++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/morgoth/utils/result_reader.py b/morgoth/utils/result_reader.py index 7d3623c..ed5a731 100644 --- a/morgoth/utils/result_reader.py +++ b/morgoth/utils/result_reader.py @@ -19,8 +19,10 @@ import astropy.units as unit from astropy.table import Table +from astropy.coordinates import Angle import pandas as pd import os, ssl +import requests base_dir = get_env_value("GBM_TRIGGER_DATA_DIR") @@ -82,6 +84,9 @@ def __init__( # Check catalog of bright gamma sources and get separation to GRB position self._sep_bright_sources() + # Check catalog of SGRs and get separation to GRB position + self._sep_SGRs() + # Create a report containing all the results of the pipeline self._build_report() @@ -356,6 +361,50 @@ def _sep_bright_sources(self): dic[table[i]['Source Name']]={"ra":table[i]['RA J2000 Degs'], "dec":table[i]['Dec J2000 Degs'], "separation":table[i]["Separation Degs"]} self.dic_bright_sources = dic + + + def _sep_SGRs(self): + + #get csv data from website + url = 'http://www.physics.mcgill.ca/~pulsar/magnetar/TabO1.csv' + r = requests.get(url, allow_redirects=True) + open('SGRList.csv', 'wb').write(r.content) + df = pd.read_csv('SGRList.csv') + + #for table of catalog + table = Table.from_pandas(df) + + #transform table data in SkyCoord + coords = [] + for i in range(len(df['RA'])): + hour_ra = Angle(table[i]['RA']+" hours") + ra = hour_ra.to(unit.deg) + arc_dec = table[i]['Decl'] + dec = Angle(tuple(map(float,arc_dec.split(' '))), unit=unit.deg) + coords.append(SkyCoord(ra, dec, frame="icrs")) + + #transform input in SkyCoord + position=SkyCoord(self._ra*unit.deg, self._dec*unit.deg, frame="icrs") + + #get separation value + separations = [] + for i in coords: + z = i.separation(position) + separations.append(z.to(unit.deg)) + + #for table of separations + table["Separation Degs"] = separations + table["Coords Degs"] = coords + table.round(3) + table.sort("Separation Degs") + #table.show_in_browser(jsviewer=True) + + #create dictionary + dic = {} + for i in range(len(table["Name"])): + dic[table[i]['Name']]={"ra":round(table[i]['Coords Degs'].ra.degree,3), "dec":round(table[i]['Coords Degs'].dec.degree,3), "separation":table[i]["Separation Degs"]} + + self.dic_SGRs = dic def _build_report(self): @@ -411,7 +460,8 @@ def _build_report(self): "active_time_stop": self._active_time_stop, "used_detectors": self._used_detectors, }, - "bright sources": self.dic_bright_sources + "bright sources": self.dic_bright_sources, + "SGRs": self.dic_SGRs } From d28333cad3c51b206f2568d8a634b6651cdf9d0b Mon Sep 17 00:00:00 2001 From: Benjamin Berger Date: Thu, 11 Nov 2021 13:01:55 +0100 Subject: [PATCH 4/7] Minor corrections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Schönheitskorrektur --- morgoth/utils/result_reader.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/morgoth/utils/result_reader.py b/morgoth/utils/result_reader.py index ed5a731..dd8eb5c 100644 --- a/morgoth/utils/result_reader.py +++ b/morgoth/utils/result_reader.py @@ -318,7 +318,8 @@ def _sep_bright_sources(self): ssl._create_default_https_context = ssl._create_unverified_context #read in table from website - table_MN = pd.read_html("https://swift.gsfc.nasa.gov/results/transients/BAT_current.html") + url = "https://swift.gsfc.nasa.gov/results/transients/BAT_current.html" + table_MN = pd.read_html(url) df = table_MN[0] #delete unwanted string @@ -334,7 +335,7 @@ def _sep_bright_sources(self): #table.show_in_browser(jsviewer=True) #transform input in SkyCoord - position=SkyCoord(self._ra*unit.deg, self._dec*unit.deg, frame="icrs") + position = SkyCoord(self._ra*unit.deg, self._dec*unit.deg, frame="icrs") #transform table data in SkyCoord coords = [] @@ -358,9 +359,10 @@ def _sep_bright_sources(self): #create dictionary dic = {} for i in range(len(table["Source Name"])): - dic[table[i]['Source Name']]={"ra":table[i]['RA J2000 Degs'], "dec":table[i]['Dec J2000 Degs'], "separation":table[i]["Separation Degs"]} + dic[table[i]['Source Name']]={"ra":table[i]['RA J2000 Degs'], + "dec":table[i]['Dec J2000 Degs'], "separation":table[i]["Separation Degs"]} - self.dic_bright_sources = dic + self._dic_bright_sources = dic def _sep_SGRs(self): @@ -402,9 +404,11 @@ def _sep_SGRs(self): #create dictionary dic = {} for i in range(len(table["Name"])): - dic[table[i]['Name']]={"ra":round(table[i]['Coords Degs'].ra.degree,3), "dec":round(table[i]['Coords Degs'].dec.degree,3), "separation":table[i]["Separation Degs"]} + dic[table[i]['Name']]={"ra":round(table[i]['Coords Degs'].ra.degree,3), + "dec":round(table[i]['Coords Degs'].dec.degree,3), + "separation":table[i]["Separation Degs"]} - self.dic_SGRs = dic + self._dic_SGRs = dic def _build_report(self): @@ -460,8 +464,8 @@ def _build_report(self): "active_time_stop": self._active_time_stop, "used_detectors": self._used_detectors, }, - "bright sources": self.dic_bright_sources, - "SGRs": self.dic_SGRs + "bright sources": self._dic_bright_sources, + "SGRs": self._dic_SGRs } From cb6fcd1f483e53d2f9cefebf2bf9038fe5fd2ec2 Mon Sep 17 00:00:00 2001 From: Benjamin Berger Date: Fri, 12 Nov 2021 15:20:08 +0100 Subject: [PATCH 5/7] Update upload_utils Implementing separation_values in report --- morgoth/utils/result_reader.py | 6 ++++-- morgoth/utils/upload_utils.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/morgoth/utils/result_reader.py b/morgoth/utils/result_reader.py index dd8eb5c..af0c569 100644 --- a/morgoth/utils/result_reader.py +++ b/morgoth/utils/result_reader.py @@ -464,9 +464,11 @@ def _build_report(self): "active_time_stop": self._active_time_stop, "used_detectors": self._used_detectors, }, - "bright sources": self._dic_bright_sources, - "SGRs": self._dic_SGRs + "separation_values": { + "bright_sources": self._dic_bright_sources, + "SGRs": self._dic_SGRs } + } def save_result_yml(self, file_path): diff --git a/morgoth/utils/upload_utils.py b/morgoth/utils/upload_utils.py index 1139280..7e619bc 100644 --- a/morgoth/utils/upload_utils.py +++ b/morgoth/utils/upload_utils.py @@ -105,8 +105,10 @@ def create_report_from_result(result): "balrog_two_sig_err_circle": result["fit_result"][ "balrog_two_sig_err_circle" ], + "bright_sources": result["separation_values"]["bright_sources"], + "SGRs": result["separation_values"]["SGRs"] } - ], + ] } return report From 63377f0efd1f768ed66346a55988bf2ec9e77c60 Mon Sep 17 00:00:00 2001 From: Benjamin Berger Date: Sun, 2 Jan 2022 19:15:02 +0100 Subject: [PATCH 6/7] Make tables shorter Make separation tables shorter, only the three nearest --- morgoth/utils/result_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/morgoth/utils/result_reader.py b/morgoth/utils/result_reader.py index af0c569..7cf4efc 100644 --- a/morgoth/utils/result_reader.py +++ b/morgoth/utils/result_reader.py @@ -358,7 +358,7 @@ def _sep_bright_sources(self): #create dictionary dic = {} - for i in range(len(table["Source Name"])): + for i in range(3): dic[table[i]['Source Name']]={"ra":table[i]['RA J2000 Degs'], "dec":table[i]['Dec J2000 Degs'], "separation":table[i]["Separation Degs"]} @@ -403,7 +403,7 @@ def _sep_SGRs(self): #create dictionary dic = {} - for i in range(len(table["Name"])): + for i in range(3): dic[table[i]['Name']]={"ra":round(table[i]['Coords Degs'].ra.degree,3), "dec":round(table[i]['Coords Degs'].dec.degree,3), "separation":table[i]["Separation Degs"]} From c09ed6178fd4a963e367cd76a7728226a98edfe9 Mon Sep 17 00:00:00 2001 From: Benjamin Berger Date: Tue, 25 Jan 2022 17:38:58 +0100 Subject: [PATCH 7/7] Fix types in YAML file --- morgoth/utils/result_reader.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/morgoth/utils/result_reader.py b/morgoth/utils/result_reader.py index 7cf4efc..a33f65a 100644 --- a/morgoth/utils/result_reader.py +++ b/morgoth/utils/result_reader.py @@ -359,8 +359,9 @@ def _sep_bright_sources(self): #create dictionary dic = {} for i in range(3): - dic[table[i]['Source Name']]={"ra":table[i]['RA J2000 Degs'], - "dec":table[i]['Dec J2000 Degs'], "separation":table[i]["Separation Degs"]} + dic[str(table[i]['Source Name'])]={"ra":float(table[i]['RA J2000 Degs']), + "dec":float(table[i]['Dec J2000 Degs']), + "separation":float(table[i]["Separation Degs"])} self._dic_bright_sources = dic @@ -404,9 +405,9 @@ def _sep_SGRs(self): #create dictionary dic = {} for i in range(3): - dic[table[i]['Name']]={"ra":round(table[i]['Coords Degs'].ra.degree,3), - "dec":round(table[i]['Coords Degs'].dec.degree,3), - "separation":table[i]["Separation Degs"]} + dic[str(table[i]['Name'])]={"ra":float(round(table[i]['Coords Degs'].ra.degree,3)), + "dec":float(round(table[i]['Coords Degs'].dec.degree,3)), + "separation":float(table[i]["Separation Degs"])} self._dic_SGRs = dic