From a4cee4bdfacdd5e6fd54007cd92274867d10cb44 Mon Sep 17 00:00:00 2001 From: "Golovich, Nathan Ryan" Date: Tue, 28 Jan 2020 16:16:40 -0800 Subject: [PATCH 001/125] basic structure for pbh func. to run and synthetic.py --- popsycle/run.py | 31 ++- popsycle/synthetic.py | 485 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 515 insertions(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index 8947cad4..166c1c7c 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -103,7 +103,14 @@ def run(): optional.add_argument('--overwrite', help="Overwrite all output files.", action='store_true') - args = parser.parse_args() + + + optional.add_argument('--pbh-config-filename', type=str, + help='Name of configuration file containing ' + 'pbh inputs. Default if needed is: ' + 'pbh_config.yaml', + default=None) + # Check for field config file. Exit if not present. if not os.path.exists(args.field_config_filename): @@ -183,6 +190,28 @@ def run(): args.overwrite): sys.exit(1) + # only do stuff if optional config file for pbhs was provided + if args.pbh_config_filename is not None: + # Check for pbh config file. Exit if not present. + if not os.path.exists(args.pbh_config_filename): + print("""Error: PBH configuration file {0} missing, + cannot continue. In order to execute run.py, generate a + PBH configuration file using + popsycle.synthetic.generate_pbh_config_file. + Exiting...""".format(args.pbh_config_filename)) + sys.exit(1) + + # TODO: check if .h5 file exists from perform popsyn, use as input for following function + if not os.path.exists({0}+'.h5'.format(args.output_root)): + print("""Error: H5 file was not created properly by + synthetic.perform_pop_syn""") + sys.exit(1) + + synthetic.add_pbh(hdf5_file=filename_dict['hdf5_filename'], + output_root2=args.output_root, + overwrite=args.overwrite, + seed=args.seed) + # Run calc_events print('-- Executing calc_events') diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 57280374..c483f107 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3512,3 +3512,488 @@ def generate_slurm_scripts(slurm_config_filename, popsycle_config_filename, print(stderr) print('Submitted job {0} to {1}'.format(script_filename, resource)) + + +def add_pbh(hdf5_file, output_root2, overwrite = False, seed = None): + """ + + TODO: change comments below for add pbh functionality + + Given some hdf5 file from perform pop syn output, creates compact objects. Sorts the stars and + compact objects into latitude/longitude bins, and saves them in an HDF5 file. + + Parameters + ---------- + ebf_file : str or ebf file + str : name of the ebf file from Galaxia + ebf file : actually the ebf file from Galaxia + + output_root : str + The thing you want the output files to be named + Examples: + 'myout' + '/some/path/to/myout' + '../back/to/some/path/myout' + + iso_dir : filepath + Where are the isochrones stored (for PopStar) + + Optional Parameters + ------------------- + bin_edges_number : int + Number of edges for the bins (bins = bin_edges_number - 1) + Total number of bins is (bin_edges_number - 1)**2 + + BH_kick_speed_mean : float + Mean of the birth kick speed of BH (in km/s) maxwellian distrubution. + Defaults to 50 km/s. + + NS_kick_speed_mean : float + Mean of the birth kick speed of NS (in km/s) maxwellian distrubution. + Defaults to 400 km/s based on distributions found by + Hobbs et al 2005 'A statistical study of 233 pulsar proper motions'. + https://ui.adsabs.harvard.edu/abs/2005MNRAS.360..974H/abstract + + overwrite : bool + If set to True, overwrites output files. If set to False, exists the + function if output files are already on disk. + Default is False. + + seed : int + If set to non-None, all random sampling will be seeded with the + specified seed, forcing identical output for PyPopStar and PopSyCLE. + Default None. + + Outputs + ------- + .h5 : hdf5 file + NOTE: This is what _bin_lb_hdf5 returns. + An hdf5 file with datasets that correspond to the longitude bin edges, + latitude bin edges, and the compact objects + and stars sorted into those bins. + + _label.fits : fits file + NOTE: This is what make_label_file returns. + A fits file that shows the correspondence between dataset name, + latitude, longitude, and number of objects in that bin. + """ + ########## + # Error handling: check whether files exist and + # whether input types are correct. + ########## + + if not overwrite: + # Check if HDF5 file exists already. If it does, throw an error message + # to complain and exit. + if os.path.isfile(output_root + '.h5'): + raise Exception( + 'That .h5 file name is taken! Either delete the .h5 file, ' + 'or pick a new name.') + + # Ditto with the fits label file. + if os.path.isfile(output_root + '_label.fits'): + raise Exception( + 'That .fits file name is taken! Either delete the .fits file, ' + 'or pick a new name.') + + # Error handling/complaining if input types are not right. + if ebf_file[-4:] != '.ebf': + raise Exception('ebf_file must be an ebf file.') + + if type(output_root) != str: + raise Exception('output_root must be a string.') + + if bin_edges_number is not None: + if type(bin_edges_number) != int: + raise Exception('bin_edges_number must be an integer.') + + if type(BH_kick_speed_mean) != int: + if type(BH_kick_speed_mean) != float: + raise Exception('BH_kick_speed_mean must be an integer or a float.') + + if type(NS_kick_speed_mean) != int: + if type(NS_kick_speed_mean) != float: + raise Exception('NS_kick_speed_mean must be an integer or a float.') + + if type(iso_dir) != str: + raise Exception('iso_dir must be a string.') + + if seed is not None: + if type(seed) != int: + raise Exception('seed must be an integer.') + + ########## + # Start of code + ######### + + # Set random seed + np.random.seed(seed) + + t0 = time.time() + + ######### + # Read in only what you need + # Control yourself... take only what you need from it + # i.e. the log file, popid, and age + ######## + t = ebf.read_ind(ebf_file, '/log', 0) + popid_array = ebf.read(ebf_file, '/popid') + age_array = ebf.read(ebf_file, '/age') + + # Just some counting/error checking things + n_stars = len(popid_array) # How many stars from Galaxia + comp_counter = 0 # Number of compact objects made + + # Convert log to useful dictionary. + ebf_log = make_ebf_log(t) + + ########## + ### Fetch the center point and area of the survey. + ########## + b = float(ebf_log['latitude']) + l = float(ebf_log['longitude']) + surveyArea = float(ebf_log['surveyArea']) + + ########## + ### Make the bins for the latitude and longitude. All the information + ### will be sorted into each of these bins in order to + # handle large arrays, etc. + ########## + # Extend the edges a bit, that's what the * 1.1 is for + # (to potentially catch any edge cases.) + # make bins of size ~1/2 arcmin + radius = np.sqrt(surveyArea / np.pi) # degrees + + # Define bin_edges_number, if not given in input. + if bin_edges_number is None: + # st the widths are 1/2 arcmin + bin_edges_number = int(60 * 2 * radius) + 1 + + # Make sure we have enough bin edges (minimum is 3) + if bin_edges_number < 2: + bin_edges_number = 3 + lat_bin_edges = np.linspace(b - 1.1 * radius, b + 1.1 * radius, + bin_edges_number) + long_bin_edges = np.linspace(l - 1.1 * radius, l + 1.1 * radius, + bin_edges_number) + # Angle wrapping for longitude + wrap_id = np.where(long_bin_edges > 180)[0] + long_bin_edges[wrap_id] -= 360 + + ########## + # Create h5py file to store lat/long binned output + ########## + h5file = h5py.File(output_root + '.h5', 'w') + dataset = h5file.create_dataset('lat_bin_edges', data=lat_bin_edges) + dataset = h5file.create_dataset('long_bin_edges', data=long_bin_edges) + h5file.close() + + ########## + # Reassign ages for stars that are less than logage 6 + # or greater than logage 10.01, since those are the limits of + # PopStar evolution. Justification: in writeup/paper. + ########## + young_id = np.where(age_array <= 5.01)[0] + age_array[young_id] = 5.0101 + old_id = np.where(age_array >= 10.14)[0] + age_array[old_id] = 10.1399 + + # Initialize a running counter for the "unique ID". + next_id = n_stars # for compact objects... + n_binned_stars = 0 # for stars... + + ########## + # Loop through population ID (i.e. bulge, disk, halo, ...) in order to + # design bin sizes appropriate for each population. Dividing by population + # is convenient because each has very different + # age ranges and radius ranges. + ########## + for pid in range(10): + print('*********************** Starting popid ' + str(pid)) + if np.sum(popid_array == pid) == 0: + print('No stars with this pid. Skipping!') + continue + popid_idx = np.where(popid_array == pid)[0] + if len(popid_idx) == 0: + print('No stars with this pid. Skipping!') + continue + + logage_min = np.min(age_array[popid_idx]) + logage_max = np.max(age_array[popid_idx]) + + # For single age populations (popID = 7-9), have a single age bin... + if logage_min == logage_max: + logt_bins = np.array([logage_min * 0.99, logage_min * 1.01]) + + # ...for multiple age populations (popID = 1-6, + # break ages into 4 age bins. + else: + logt_bins = np.log10( + np.logspace(logage_min, logage_max * 1.001, 5)) + + # HARDCODED: Special handling for the youngest bin, + # popID = 0 (Thin Disk < 150 Myr). + if pid == 0: + logt_bins = np.array([logage_min, 6.3, 7.0, 7.7, 8.0, 8.2]) + + ########## + # Loop through age bins + # -- note, don't slice tables as we don't want + # duplicated things carried in memory. + ########## + for aa in range(len(logt_bins) - 1): + print('Starting age bin ', logt_bins[aa]) + # Mid-point age of bin. + age_of_bin = (logt_bins[aa] + logt_bins[aa + 1]) / 2.0 + + # Fetch the stars in this age bin. + age_idx = np.where((age_array[popid_idx] >= logt_bins[aa]) & + (age_array[popid_idx] < logt_bins[aa + 1]))[0] + len_adx = len(age_idx) + + # Figure out how many random bins we will need. + # -- breaking up into managable chunks of 2 million stars each. + num_stars_in_bin = 2e6 + num_rand_bins = int(math.ceil(len_adx / num_stars_in_bin)) + + ########## + # Loop through random bins of 2 million stars at a time. + ########## + for nn in range(num_rand_bins): + print('Starting sub-bin ', nn) + n_start = int(nn * num_stars_in_bin) + n_stop = int((nn + 1) * num_stars_in_bin) + + bin_idx = popid_idx[age_idx[n_start:n_stop]] + + exbv = ebf.read_ind(ebf_file, '/exbv_schlegel', bin_idx) + + star_dict = {} + star_dict['zams_mass'] = ebf.read_ind(ebf_file, '/smass', + bin_idx) + star_dict['mass'] = ebf.read_ind(ebf_file, '/mact', bin_idx) + star_dict['px'] = ebf.read_ind(ebf_file, '/px', bin_idx) + star_dict['py'] = ebf.read_ind(ebf_file, '/py', bin_idx) + star_dict['pz'] = ebf.read_ind(ebf_file, '/pz', bin_idx) + star_dict['vx'] = ebf.read_ind(ebf_file, '/vx', bin_idx) + star_dict['vy'] = ebf.read_ind(ebf_file, '/vy', bin_idx) + star_dict['vz'] = ebf.read_ind(ebf_file, '/vz', bin_idx) + star_dict['age'] = age_array[bin_idx] + star_dict['popid'] = popid_array[bin_idx] + star_dict['ubv_k'] = ebf.read_ind(ebf_file, '/ubv_k', bin_idx) + star_dict['ubv_i'] = ebf.read_ind(ebf_file, '/ubv_i', bin_idx) + star_dict['ubv_j'] = ebf.read_ind(ebf_file, '/ubv_j', bin_idx) + star_dict['ubv_u'] = ebf.read_ind(ebf_file, '/ubv_u', bin_idx) + star_dict['ubv_r'] = ebf.read_ind(ebf_file, '/ubv_r', bin_idx) + star_dict['ubv_b'] = ebf.read_ind(ebf_file, '/ubv_b', bin_idx) + star_dict['ubv_h'] = ebf.read_ind(ebf_file, '/ubv_h', bin_idx) + star_dict['ubv_v'] = ebf.read_ind(ebf_file, '/ubv_v', bin_idx) + star_dict['exbv'] = exbv + star_dict['glat'] = ebf.read_ind(ebf_file, '/glat', bin_idx) + star_dict['glon'] = ebf.read_ind(ebf_file, '/glon', bin_idx) + # Angle wrapping for longitude + wrap_idx = np.where(star_dict['glon'] > 180)[0] + star_dict['glon'][wrap_idx] -= 360 + star_dict['rad'] = ebf.read_ind(ebf_file, '/rad', bin_idx) + star_dict['rem_id'] = np.zeros(len(bin_idx)) + star_dict['obj_id'] = np.arange(len(bin_idx)) + n_binned_stars + + n_binned_stars += len(bin_idx) + + ########## + # Add spherical velocities vr, mu_b, mu_lcosb + ########## + vr, mu_b, mu_lcosb = calc_sph_motion(star_dict['vx'], + star_dict['vy'], + star_dict['vz'], + star_dict['rad'], + star_dict['glat'], + star_dict['glon']) + ######### + # Add precision to r, b, l, vr, mu_b, mu_lcosb + ######### + star_dict['rad'] = add_precision64(star_dict['rad'], -4) + star_dict['glat'] = add_precision64(star_dict['glat'], -4) + star_dict['glon'] = add_precision64(star_dict['glon'], -4) + star_dict['vr'] = add_precision64(vr, -4) + star_dict['mu_b'] = add_precision64(mu_b, -4) + star_dict['mu_lcosb'] = add_precision64(mu_lcosb, -4) + + ########## + # Perform population synthesis. + ########## + mass_in_bin = np.sum(star_dict['zams_mass']) + + stars_in_bin = {} + for key, val in star_dict.items(): + stars_in_bin[key] = val + + comp_dict, next_id = _make_comp_dict(iso_dir, age_of_bin, + mass_in_bin, stars_in_bin, next_id, + BH_kick_speed_mean=BH_kick_speed_mean, + NS_kick_speed_mean=NS_kick_speed_mean, + seed=seed) + + ########## + # Bin in l, b all stars and compact objects. + ########## + if comp_dict is not None: + comp_counter += len(comp_dict['mass']) + _bin_lb_hdf5(lat_bin_edges, long_bin_edges, comp_dict, + output_root) + _bin_lb_hdf5(lat_bin_edges, long_bin_edges, stars_in_bin, + output_root) + ########## + # Done with galaxia output in dictionary t and ebf_log. + # Garbage collect in order to save space. + ########## + del star_dict + gc.collect() + + t1 = time.time() + print('Total run time is {0:f} s'.format(t1 - t0)) + + ########## + # Figure out how much stuff got binned. + ########## + binned_counter = 0 + hf = h5py.File(output_root + '.h5', 'r') + for key in list(hf.keys()): + if len(hf[key].shape) > 1: + binned_counter += (hf[key].shape)[1] + + ########## + # Make label file containing information about l,b bins + ########## + make_label_file(output_root) + + ########## + # Make log file + ########## + now = datetime.datetime.now() + microlens_path = os.path.split(os.path.abspath(__file__))[0] + popstar_path = os.path.split(os.path.abspath(imf.__file__))[0] + microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=microlens_path).decode('ascii').strip() + popstar_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=popstar_path).decode('ascii').strip() + dash_line = '-----------------------------' + '\n' + empty_line = '\n' + + line0 = 'FUNCTION INPUT PARAMETERS' + '\n' + line1 = 'ebf_file , ' + ebf_file + '\n' + line2 = 'output_root , ' + output_root + '\n' + line3 = 'bin_edges_number , ' + str(bin_edges_number) + '\n' + line4 = 'BH_kick_speed_mean , ' + str(BH_kick_speed_mean) + ' , (km/s)' + '\n' + line5 = 'NS_kick_speed_mean , ' + str(NS_kick_speed_mean) + ' , (km/s)' + '\n' + line6 = 'iso_dir , ' + iso_dir + '\n' + + line7 = 'VERSION INFORMATION' + '\n' + line8 = str(now) + ' : creation date' + '\n' + line9 = popstar_hash + ' : PopStar commit' + '\n' + line10 = microlens_hash + ' : microlens commit' + '\n' + + line11 = 'OTHER INFORMATION' + '\n' + line12 = str(t1 - t0) + ' : total runtime (s)' + '\n' + line13 = str(n_stars) + ' : total stars from Galaxia' + '\n' + line14 = str(comp_counter) + ' : total compact objects made' + '\n' + line15 = str(binned_counter) + ' : total things binned' + '\n' + + line16 = 'FILES CREATED' + '\n' + line17 = output_root + '.h5 : HDF5 file' + '\n' + line18 = output_root + '_label.fits : label file' + '\n' + + with open(output_root + '_perform_pop_syn.log', 'w') as out: + out.writelines([line0, dash_line, line1, line2, line3, line4, line5, + line6, empty_line, line7, dash_line, line8, line9, + line10, empty_line, line11, dash_line, line12, line13, + line14, line15, empty_line, line16, dash_line, line17, + line18]) + + ########## + # Informative print statements. + ########## + if (n_stars + comp_counter) != binned_counter: + print('***************** WARNING ******************') + print('Number of things in != number of things out.') + print('********************************************') + + print('******************** INFO **********************') + print('Total number of stars from Galaxia: ' + str(n_stars)) + print('Total number of compact objects made: ' + str(comp_counter)) + print('Total number of things binned: ' + str(binned_counter)) + + + return + +# TODO, write this for pbhs +def generate_pbh_config_file(config_filename, radius_cut, obs_time, + n_obs, theta_frac, blend_rad, + isochrones_dir, + bin_edges_number, + BH_kick_speed_mean, NS_kick_speed_mean, + filter_name, red_law): + """ + Save popsycle configuration parameters from a dictionary into a yaml file + + Parameters + ---------- + config_filename : str + Name of the configuration file + + radius_cut : float + Initial radius cut, in ARCSECONDS. + + obs_time : float + Survey duration, in DAYS. + + n_obs : float + Number of observations. + + theta_frac : float + Another cut, in multiples of Einstein radii. + + blend_rad : float + Stars within this distance of the lens are said to be blended. + Units are in ARCSECONDS. + + isochrones_dir : str + Directory for PyPopStar isochrones + + bin_edges_number : int + Number of edges for the bins (bins = bin_edges_number - 1) + Total number of bins is (bin_edges_number - 1)**2 + + BH_kick_speed_mean : float + Mean of the birth kick speed of BH (in km/s) maxwellian distrubution. + + NS_kick_speed_mean : float + Mean of the birth kick speed of NS (in km/s) maxwellian distrubution. + + filter_name : str + The name of the filter in which to calculate all the + microlensing events. The filter name convention is set + in the global filt_dict parameter at the top of this module. + + red_law : str + The name of the reddening law to use from PopStar. + + Output + ------ + None + """ + + config = {'radius_cut': radius_cut, + 'obs_time': obs_time, + 'n_obs': n_obs, + 'theta_frac': theta_frac, + 'blend_rad': blend_rad, + 'isochrones_dir': isochrones_dir, + 'bin_edges_number': bin_edges_number, + 'BH_kick_speed_mean': BH_kick_speed_mean, + 'NS_kick_speed_mean': NS_kick_speed_mean, + 'filter_name': filter_name, + 'red_law': red_law} + generate_config_file(config_filename, config) + + From 2e75aa0ef4fbbcab8ce654d2096c837cc8784d8d Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Thu, 30 Jan 2020 16:19:19 -0800 Subject: [PATCH 002/125] Added .csv file, edited run.py and synthetic.py to include PBHs --- popsycle/data/pbh_config.yaml | 8 + .../radial_velocity_dispersion_digitized.csv | 59 ++ popsycle/synthetic.py | 702 ++++++++---------- 3 files changed, 370 insertions(+), 399 deletions(-) create mode 100644 popsycle/data/pbh_config.yaml create mode 100644 popsycle/data/radial_velocity_dispersion_digitized.csv diff --git a/popsycle/data/pbh_config.yaml b/popsycle/data/pbh_config.yaml new file mode 100644 index 00000000..d856bd66 --- /dev/null +++ b/popsycle/data/pbh_config.yaml @@ -0,0 +1,8 @@ +# Example configuration parameters for PBHs + +galaxia_area: .001 #How many square degrees are you running galaxia on +fdm: 1 #Fraction of dark matter that consists of PBHs +pbh_mass: 40 #Msun +r_max: 8.3 #kpc, How far from the galactic center that you want to go out +c: 10 #concentration index +r_vir: 200 #kpc diff --git a/popsycle/data/radial_velocity_dispersion_digitized.csv b/popsycle/data/radial_velocity_dispersion_digitized.csv new file mode 100644 index 00000000..4864069f --- /dev/null +++ b/popsycle/data/radial_velocity_dispersion_digitized.csv @@ -0,0 +1,59 @@ +radius, dispersion +0.0007360446241274024, 0.07906914418438438 +0.0008534023145816314, 0.0783151688404502 +0.0009352493660181735, 0.09928939204443932 +0.0010616920597239583, 0.0948487710252941 +0.001131186315865826, 0.07058692156512414 +0.0012223307362665052, 0.08917926811441179 +0.0014072747508446864, 0.10059171309305304 +0.0014006809548853252, 0.09339467571913518 +0.0016481597082506775, 0.0995264956080697 +0.00191865661207353, 0.10597235532022009 +0.0022518088984294273, 0.10372756509168868 +0.002236359182974952, 0.11113665803108796 +0.002652983994959231, 0.10806292331931067 +0.0030977317671075488, 0.11191847905533803 +0.003216334330268018, 0.12074722569630558 +0.0036221371499138544, 0.10862497766660706 +0.0037732018429818596, 0.12255981288677364 +0.004077224522720631, 0.11224405931749148 +0.004431684223884757, 0.12149072916046288 +0.00507711867557682, 0.12835958416876758 +0.006006722605063362, 0.13438351714392444 +0.006933859783391445, 0.13789754680086547 +0.007850528352299446, 0.14344664349407954 +0.008160676858521945, 0.15484366624977663 +0.009166595468236206, 0.14295141877954093 +0.010703288837072599, 0.14740672763006152 +0.01249759437151708, 0.1496001104487793 +0.01459269832408766, 0.15175922166095468 +0.01703902671566554, 0.15295872788994097 +0.019895459014453068, 0.15350707359462046 +0.023230745276774503, 0.15511783910211635 +0.02712516085818071, 0.15532346874137115 +0.031672438521280974, 0.15583754283950818 +0.03672240983289494, 0.1546811426756397 +0.04237806984137696, 0.1474672741349532 +0.049715295366751476, 0.1448363571393766 +0.05836502438103976, 0.14088223391957533 +0.0680202364114164, 0.14458331844440475 +0.06929852326934813, 0.13362464585619843 +0.07947915979047598, 0.13753993210648552 +0.09241165369047062, 0.13308119609531072 +0.1079035967920762, 0.131333344161645 +0.12182981941605571, 0.12177314770275272 +0.1430264132617661, 0.11525996069322841 +0.16700344394773997, 0.11056475059691062 +0.19499999793297998, 0.1044986762388942 +0.22768991042939973, 0.0982269722416228 +0.26214039495405517, 0.09037877434339814 +0.3018033892545949, 0.08338736660873514 +0.3523978840773408, 0.07845225526662003 +0.41147406929028135, 0.07620936901622977 +0.46710403451832827, 0.07134269208844868 +0.5071166622055777, 0.0619601870049431 +0.5935221015515242, 0.05919161238975412 +0.6930204901160147, 0.06107655074958973 +0.8091988461173554, 0.06330420517485014 +0.9537702647588352, 0.05996843547138331 +0.9926153602554509, 0.053433982490619925 diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index c483f107..2db49206 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3514,45 +3514,51 @@ def generate_slurm_scripts(slurm_config_filename, popsycle_config_filename, print('Submitted job {0} to {1}'.format(script_filename, resource)) -def add_pbh(hdf5_file, output_root2, overwrite = False, seed = None): +def add_pbh(hdf5_file, output_root2, galaxia_area, fdm, pbh_mass, r_max, c, r_vir, overwrite = False, seed = None): """ - - TODO: change comments below for add pbh functionality - - Given some hdf5 file from perform pop syn output, creates compact objects. Sorts the stars and - compact objects into latitude/longitude bins, and saves them in an HDF5 file. + Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, + and saves them in a new HDF5 file with the PBHs added. Parameters ---------- - ebf_file : str or ebf file - str : name of the ebf file from Galaxia - ebf file : actually the ebf file from Galaxia + hdf5_file : str or hdf5 file + str : name of the hdf5 file from the output of perform_pop_syn - output_root : str + output_root2 : str The thing you want the output files to be named Examples: 'myout' '/some/path/to/myout' '../back/to/some/path/myout' - iso_dir : filepath - Where are the isochrones stored (for PopStar) - Optional Parameters ------------------- - bin_edges_number : int - Number of edges for the bins (bins = bin_edges_number - 1) - Total number of bins is (bin_edges_number - 1)**2 + galaxia_area : float + How many square degrees did you run galaxia on. + This goes into calculating the field of view area, which is used to cut + a rectangular mask down into a circular mask for determining positions of + PBHs. - BH_kick_speed_mean : float - Mean of the birth kick speed of BH (in km/s) maxwellian distrubution. - Defaults to 50 km/s. + fdm : float + Fraction of dark matter. + The fraction of dark matter that you want to consist of PBHs. + Defaults to 1. - NS_kick_speed_mean : float - Mean of the birth kick speed of NS (in km/s) maxwellian distrubution. - Defaults to 400 km/s based on distributions found by - Hobbs et al 2005 'A statistical study of 233 pulsar proper motions'. - https://ui.adsabs.harvard.edu/abs/2005MNRAS.360..974H/abstract + pbh_mass : int + The single mass that all PBHs will have. + Defaults to 40 Msun (from LIGO detections thought to be primordial) + + r_max : float + The maximum radius from the galactic center that you want to find PBHs at. + Defaults to 8.3 kpc (Where Earth is located) + + c : float + Concentration index. + Defaults to 10 (concentration index of the Milky Way, multiple papers show it to be anywhere from 10-12) + + r_vir : float + The virial radius. + Defaults to 200 kpc (The virial radius of the Milky Way) overwrite : bool If set to True, overwrites output files. If set to False, exists the @@ -3566,16 +3572,8 @@ def add_pbh(hdf5_file, output_root2, overwrite = False, seed = None): Outputs ------- - .h5 : hdf5 file - NOTE: This is what _bin_lb_hdf5 returns. - An hdf5 file with datasets that correspond to the longitude bin edges, - latitude bin edges, and the compact objects - and stars sorted into those bins. - - _label.fits : fits file - NOTE: This is what make_label_file returns. - A fits file that shows the correspondence between dataset name, - latitude, longitude, and number of objects in that bin. + .h5 : hdf5 file + The new .h5 file with PBHs injected in. """ ########## # Error handling: check whether files exist and @@ -3585,38 +3583,39 @@ def add_pbh(hdf5_file, output_root2, overwrite = False, seed = None): if not overwrite: # Check if HDF5 file exists already. If it does, throw an error message # to complain and exit. - if os.path.isfile(output_root + '.h5'): + if os.path.isfile(output_root2 + '.h5'): raise Exception( 'That .h5 file name is taken! Either delete the .h5 file, ' 'or pick a new name.') - # Ditto with the fits label file. - if os.path.isfile(output_root + '_label.fits'): - raise Exception( - 'That .fits file name is taken! Either delete the .fits file, ' - 'or pick a new name.') - # Error handling/complaining if input types are not right. - if ebf_file[-4:] != '.ebf': - raise Exception('ebf_file must be an ebf file.') - if type(output_root) != str: + if type(output_root2) != str: raise Exception('output_root must be a string.') - if bin_edges_number is not None: - if type(bin_edges_number) != int: - raise Exception('bin_edges_number must be an integer.') + if type(galaxia_area) != float: + if type(galaxia_area) != int: + raise Exception('galaxia_area must be a float or an integer.') - if type(BH_kick_speed_mean) != int: - if type(BH_kick_speed_mean) != float: - raise Exception('BH_kick_speed_mean must be an integer or a float.') + if type(fdm) != float: + if type(fdm) != int: + raise Exception('fdm must be a float or an integer.') - if type(NS_kick_speed_mean) != int: - if type(NS_kick_speed_mean) != float: - raise Exception('NS_kick_speed_mean must be an integer or a float.') + if type(pbh_mass) != int: + if type(pbh_mass) != float: + raise Exception('pbh_mass must be an integer or a float.') - if type(iso_dir) != str: - raise Exception('iso_dir must be a string.') + if type(r_max) != float: + if type(r_max) != int: + raise Exception('r_max must be a float or an integer.') + + if type(c) != int: + if type(c) != float: + raise Exception('c must be an integer or a float.') + + if type(r_vir) != float: + if type(r_vir) != int: + raise Exception('r_vir must be a float or an integer.') if seed is not None: if type(seed) != int: @@ -3631,369 +3630,274 @@ def add_pbh(hdf5_file, output_root2, overwrite = False, seed = None): t0 = time.time() - ######### - # Read in only what you need - # Control yourself... take only what you need from it - # i.e. the log file, popid, and age - ######## - t = ebf.read_ind(ebf_file, '/log', 0) - popid_array = ebf.read(ebf_file, '/popid') - age_array = ebf.read(ebf_file, '/age') - - # Just some counting/error checking things - n_stars = len(popid_array) # How many stars from Galaxia - comp_counter = 0 # Number of compact objects made - - # Convert log to useful dictionary. - ebf_log = make_ebf_log(t) - - ########## - ### Fetch the center point and area of the survey. - ########## - b = float(ebf_log['latitude']) - l = float(ebf_log['longitude']) - surveyArea = float(ebf_log['surveyArea']) - - ########## - ### Make the bins for the latitude and longitude. All the information - ### will be sorted into each of these bins in order to - # handle large arrays, etc. - ########## - # Extend the edges a bit, that's what the * 1.1 is for - # (to potentially catch any edge cases.) - # make bins of size ~1/2 arcmin - radius = np.sqrt(surveyArea / np.pi) # degrees - - # Define bin_edges_number, if not given in input. - if bin_edges_number is None: - # st the widths are 1/2 arcmin - bin_edges_number = int(60 * 2 * radius) + 1 - - # Make sure we have enough bin edges (minimum is 3) - if bin_edges_number < 2: - bin_edges_number = 3 - lat_bin_edges = np.linspace(b - 1.1 * radius, b + 1.1 * radius, - bin_edges_number) - long_bin_edges = np.linspace(l - 1.1 * radius, l + 1.1 * radius, - bin_edges_number) - # Angle wrapping for longitude - wrap_id = np.where(long_bin_edges > 180)[0] - long_bin_edges[wrap_id] -= 360 - - ########## - # Create h5py file to store lat/long binned output - ########## - h5file = h5py.File(output_root + '.h5', 'w') - dataset = h5file.create_dataset('lat_bin_edges', data=lat_bin_edges) - dataset = h5file.create_dataset('long_bin_edges', data=long_bin_edges) - h5file.close() - - ########## - # Reassign ages for stars that are less than logage 6 - # or greater than logage 10.01, since those are the limits of - # PopStar evolution. Justification: in writeup/paper. - ########## - young_id = np.where(age_array <= 5.01)[0] - age_array[young_id] = 5.0101 - old_id = np.where(age_array >= 10.14)[0] - age_array[old_id] = 10.1399 - - # Initialize a running counter for the "unique ID". - next_id = n_stars # for compact objects... - n_binned_stars = 0 # for stars... - - ########## - # Loop through population ID (i.e. bulge, disk, halo, ...) in order to - # design bin sizes appropriate for each population. Dividing by population - # is convenient because each has very different - # age ranges and radius ranges. - ########## - for pid in range(10): - print('*********************** Starting popid ' + str(pid)) - if np.sum(popid_array == pid) == 0: - print('No stars with this pid. Skipping!') - continue - popid_idx = np.where(popid_array == pid)[0] - if len(popid_idx) == 0: - print('No stars with this pid. Skipping!') - continue - - logage_min = np.min(age_array[popid_idx]) - logage_max = np.max(age_array[popid_idx]) - - # For single age populations (popID = 7-9), have a single age bin... - if logage_min == logage_max: - logt_bins = np.array([logage_min * 0.99, logage_min * 1.01]) - - # ...for multiple age populations (popID = 1-6, - # break ages into 4 age bins. + #Define parameters for NFW profile calculations + r_s = r_vir/c #kpc, scale radius + g = 4.3*(10**-3) #(pc*km^2)/(Msun*s^2) + h = 70 #km/(s*Mpc) + + #Read in the hdf5 file that doesn't have PBHs. Product of perform_pop_syn. + no_pbh_hdf5_file = h5py.File(hdf5_file) + key_list = list(no_pbh_hdf5_file) + key_list = key_list[:len(key_list)-2] + lat_bin = pd.DataFrame(np.array(no_pbh['lat_bin_edges'])) + long_bin = pd.DataFrame(np.array(no_pbh['long_bin_edges'])) + no_pbh_hdf5_file.close() + + field_of_view_area = 2*((galaxia_area/np.pi)**(1/2)) + + #NFW Profile calculations to determine mass of dark matter within given distance of galactic center + rho_crit = ((3*(h**2))/(8*np.pi*g))*(10**6) #Msun/Mpc^3 + rho_knot = (200/3)*(rho_crit)*((c**3)/(np.log(c+1)-(c/(1+c)))) # Msun/Mpc^3 + mass_8_kpc = (4*np.pi*rho_knot*(r_s**3)*(np.log((r_s + r_max)/r_s) - (r_max/(r_s+r_max))))*(((10**3)**3)/((10**6)**3))*fdm #Msun + + #Determine the number of PBHs within that distance + num_pbh_8kpc = (mass_8_kpc/pbh_mass) + num_pbh_8kpc = round(num_pbh_8kpc, -1) + + #Defining needed functions from the python package "NFWdist". + #Used to calculate PBH radii from galactic center assuming NFW profile. + #https://github.com/CullanHowlett/NFWdist + def pnfwunorm(q, con=5): + if hasattr(con, '__len__'): + y = np.outer(q,con) else: - logt_bins = np.log10( - np.logspace(logage_min, logage_max * 1.001, 5)) - - # HARDCODED: Special handling for the youngest bin, - # popID = 0 (Thin Disk < 150 Myr). - if pid == 0: - logt_bins = np.array([logage_min, 6.3, 7.0, 7.7, 8.0, 8.2]) - - ########## - # Loop through age bins - # -- note, don't slice tables as we don't want - # duplicated things carried in memory. - ########## - for aa in range(len(logt_bins) - 1): - print('Starting age bin ', logt_bins[aa]) - # Mid-point age of bin. - age_of_bin = (logt_bins[aa] + logt_bins[aa + 1]) / 2.0 - - # Fetch the stars in this age bin. - age_idx = np.where((age_array[popid_idx] >= logt_bins[aa]) & - (age_array[popid_idx] < logt_bins[aa + 1]))[0] - len_adx = len(age_idx) - - # Figure out how many random bins we will need. - # -- breaking up into managable chunks of 2 million stars each. - num_stars_in_bin = 2e6 - num_rand_bins = int(math.ceil(len_adx / num_stars_in_bin)) - - ########## - # Loop through random bins of 2 million stars at a time. - ########## - for nn in range(num_rand_bins): - print('Starting sub-bin ', nn) - n_start = int(nn * num_stars_in_bin) - n_stop = int((nn + 1) * num_stars_in_bin) - - bin_idx = popid_idx[age_idx[n_start:n_stop]] - - exbv = ebf.read_ind(ebf_file, '/exbv_schlegel', bin_idx) - - star_dict = {} - star_dict['zams_mass'] = ebf.read_ind(ebf_file, '/smass', - bin_idx) - star_dict['mass'] = ebf.read_ind(ebf_file, '/mact', bin_idx) - star_dict['px'] = ebf.read_ind(ebf_file, '/px', bin_idx) - star_dict['py'] = ebf.read_ind(ebf_file, '/py', bin_idx) - star_dict['pz'] = ebf.read_ind(ebf_file, '/pz', bin_idx) - star_dict['vx'] = ebf.read_ind(ebf_file, '/vx', bin_idx) - star_dict['vy'] = ebf.read_ind(ebf_file, '/vy', bin_idx) - star_dict['vz'] = ebf.read_ind(ebf_file, '/vz', bin_idx) - star_dict['age'] = age_array[bin_idx] - star_dict['popid'] = popid_array[bin_idx] - star_dict['ubv_k'] = ebf.read_ind(ebf_file, '/ubv_k', bin_idx) - star_dict['ubv_i'] = ebf.read_ind(ebf_file, '/ubv_i', bin_idx) - star_dict['ubv_j'] = ebf.read_ind(ebf_file, '/ubv_j', bin_idx) - star_dict['ubv_u'] = ebf.read_ind(ebf_file, '/ubv_u', bin_idx) - star_dict['ubv_r'] = ebf.read_ind(ebf_file, '/ubv_r', bin_idx) - star_dict['ubv_b'] = ebf.read_ind(ebf_file, '/ubv_b', bin_idx) - star_dict['ubv_h'] = ebf.read_ind(ebf_file, '/ubv_h', bin_idx) - star_dict['ubv_v'] = ebf.read_ind(ebf_file, '/ubv_v', bin_idx) - star_dict['exbv'] = exbv - star_dict['glat'] = ebf.read_ind(ebf_file, '/glat', bin_idx) - star_dict['glon'] = ebf.read_ind(ebf_file, '/glon', bin_idx) - # Angle wrapping for longitude - wrap_idx = np.where(star_dict['glon'] > 180)[0] - star_dict['glon'][wrap_idx] -= 360 - star_dict['rad'] = ebf.read_ind(ebf_file, '/rad', bin_idx) - star_dict['rem_id'] = np.zeros(len(bin_idx)) - star_dict['obj_id'] = np.arange(len(bin_idx)) + n_binned_stars - - n_binned_stars += len(bin_idx) - - ########## - # Add spherical velocities vr, mu_b, mu_lcosb - ########## - vr, mu_b, mu_lcosb = calc_sph_motion(star_dict['vx'], - star_dict['vy'], - star_dict['vz'], - star_dict['rad'], - star_dict['glat'], - star_dict['glon']) - ######### - # Add precision to r, b, l, vr, mu_b, mu_lcosb - ######### - star_dict['rad'] = add_precision64(star_dict['rad'], -4) - star_dict['glat'] = add_precision64(star_dict['glat'], -4) - star_dict['glon'] = add_precision64(star_dict['glon'], -4) - star_dict['vr'] = add_precision64(vr, -4) - star_dict['mu_b'] = add_precision64(mu_b, -4) - star_dict['mu_lcosb'] = add_precision64(mu_lcosb, -4) - - ########## - # Perform population synthesis. - ########## - mass_in_bin = np.sum(star_dict['zams_mass']) - - stars_in_bin = {} - for key, val in star_dict.items(): - stars_in_bin[key] = val - - comp_dict, next_id = _make_comp_dict(iso_dir, age_of_bin, - mass_in_bin, stars_in_bin, next_id, - BH_kick_speed_mean=BH_kick_speed_mean, - NS_kick_speed_mean=NS_kick_speed_mean, - seed=seed) - - ########## - # Bin in l, b all stars and compact objects. - ########## - if comp_dict is not None: - comp_counter += len(comp_dict['mass']) - _bin_lb_hdf5(lat_bin_edges, long_bin_edges, comp_dict, - output_root) - _bin_lb_hdf5(lat_bin_edges, long_bin_edges, stars_in_bin, - output_root) - ########## - # Done with galaxia output in dictionary t and ebf_log. - # Garbage collect in order to save space. - ########## - del star_dict - gc.collect() - - t1 = time.time() - print('Total run time is {0:f} s'.format(t1 - t0)) - - ########## - # Figure out how much stuff got binned. - ########## - binned_counter = 0 - hf = h5py.File(output_root + '.h5', 'r') - for key in list(hf.keys()): - if len(hf[key].shape) > 1: - binned_counter += (hf[key].shape)[1] - - ########## - # Make label file containing information about l,b bins - ########## - make_label_file(output_root) - - ########## - # Make log file - ########## - now = datetime.datetime.now() - microlens_path = os.path.split(os.path.abspath(__file__))[0] - popstar_path = os.path.split(os.path.abspath(imf.__file__))[0] - microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=microlens_path).decode('ascii').strip() - popstar_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=popstar_path).decode('ascii').strip() - dash_line = '-----------------------------' + '\n' - empty_line = '\n' - - line0 = 'FUNCTION INPUT PARAMETERS' + '\n' - line1 = 'ebf_file , ' + ebf_file + '\n' - line2 = 'output_root , ' + output_root + '\n' - line3 = 'bin_edges_number , ' + str(bin_edges_number) + '\n' - line4 = 'BH_kick_speed_mean , ' + str(BH_kick_speed_mean) + ' , (km/s)' + '\n' - line5 = 'NS_kick_speed_mean , ' + str(NS_kick_speed_mean) + ' , (km/s)' + '\n' - line6 = 'iso_dir , ' + iso_dir + '\n' - - line7 = 'VERSION INFORMATION' + '\n' - line8 = str(now) + ' : creation date' + '\n' - line9 = popstar_hash + ' : PopStar commit' + '\n' - line10 = microlens_hash + ' : microlens commit' + '\n' - - line11 = 'OTHER INFORMATION' + '\n' - line12 = str(t1 - t0) + ' : total runtime (s)' + '\n' - line13 = str(n_stars) + ' : total stars from Galaxia' + '\n' - line14 = str(comp_counter) + ' : total compact objects made' + '\n' - line15 = str(binned_counter) + ' : total things binned' + '\n' - - line16 = 'FILES CREATED' + '\n' - line17 = output_root + '.h5 : HDF5 file' + '\n' - line18 = output_root + '_label.fits : label file' + '\n' - - with open(output_root + '_perform_pop_syn.log', 'w') as out: - out.writelines([line0, dash_line, line1, line2, line3, line4, line5, - line6, empty_line, line7, dash_line, line8, line9, - line10, empty_line, line11, dash_line, line12, line13, - line14, line15, empty_line, line16, dash_line, line17, - line18]) - - ########## - # Informative print statements. - ########## - if (n_stars + comp_counter) != binned_counter: - print('***************** WARNING ******************') - print('Number of things in != number of things out.') - print('********************************************') - - print('******************** INFO **********************') - print('Total number of stars from Galaxia: ' + str(n_stars)) - print('Total number of compact objects made: ' + str(comp_counter)) - print('Total number of things binned: ' + str(binned_counter)) - + y = q*con + return np.log(1.0 + y)-y/(1.0 + y) + + def pnfw(q, con=5, logp=False): + p = pnfwunorm(q, con=con)/pnfwunorm(1, con=con) + if hasattr(q, '__len__'): + p[q>1] = 1 + p[q<=0] = 0 + else: + if (q > 1): + p = 1 + elif (q <= 0): + p = 0 + if(logp): + return np.log(p) + else: + return p + + def qnfw(p, con=5, logp=False): + if (logp): + p = np.exp(p) + if hasattr(p, '__len__'): + p[p>1] = 1 + p[p<=0] = 0 + else: + if (p > 1): + p = 1 + elif (p <= 0): + p = 0 + if hasattr(con, '__len__'): + p = np.outer(p,pnfwunorm(1, con=con)) + else: + p *= pnfwunorm(1, con=con) + return (-(1.0/np.real(special.lambertw(-np.exp(-p-1))))-1)/con + + #Calculating radius values for all PBHs within 8.3kpc of the galactic center + r_values = (qnfw(np.random.rand(int(num_pbh_8kpc)) * pnfw(r_max/r_vir,con=c, logp=False), con=c)*r_vir) + + #Sample PBH lattitude and longitude to have full spherical coordinates. + sin_lats = np.random.uniform(-1, 1, int(num_pbh_8kpc)) + lats=np.arcsin(sin_lats) + longs = np.random.uniform(0, 2*np.pi, int(num_pbh_8kpc)) + + #Converting spherical galactocentric coordinates to cartesian galactocentric coordinates + cart = astropy.coordinates.spherical_to_cartesian(r_values, lats, longs) + + #Defining galactocentric coordinates + c = coord.Galactocentric(x=cart[0] * u.kpc, y=cart[1] * u.kpc, z=cart[2] * u.kpc) + + #Transforming from galactocentric to galactic (heliocentric) coordinates + #outputs l, b, dist in degrees. + c = c.transform_to(coord.Galactic(representation_type='cartesian')) + + #Defining function to use in PBH mask (this portion of the mask creates circular shape + #from a rectangular shape because our field of view is circular) + def angdist(ra1, dec1, ra2, dec2): + from numpy import pi, sin, cos, arccos + d2r = pi / 180.0 + cosdist = (cos(np.absolute(ra1 - ra2) * d2r) * cos(np.absolute(dec1 * d2r)) * cos(np.absolute(dec2 * d2r)) + + sin(np.absolute(dec1 * d2r)) * sin(np.absolute(dec2 * d2r))) + distance = arccos(cosdist) / d2r + return distance + + #Determining the center of the field of view circle. + l_mid = np.average((l_min, l_max)) + b_mid = np.average((b_min, b_max)) + + dists = angdist(l_mid, b_mid, c.l.deg, c.b.deg) + + #Set minimum and maximum l, b, and r values for PBH mask. + l_min = np.min(long_bin).values + l_max = np.max(long_bin).values + b_min = np.min(lat_bin).values + b_max = np.max(lat_bin).values + r_max_mask = 16.6 + + #Masking the full PBH data for our field of view + mask = (c.l.deg >= l_min) & (c.l.deg <= l_max) & (c.b.deg >= b_min) & (c.b.deg <= b_max) & (c.distance.kpc <= r_max_mask) & (dists < field_of_view_area) + data_in_field = c[mask] + + #Obtain the radius values for all of the PBHs in our field of view. + r = data_in_field.distance.kpc + + #Getting radial dispersion vs. radius distribution from the M. Hoeft 2018 paper. + disp_data = pd.read_csv('data/radial_velocity_dispersion_digitized.csv') + plot_r_val = disp_data['radius'].values + plot_dispersion = disp_data.iloc[:,1].values + + #Adjusting for the correct units + plot_r_val_kpc = plot_r_val*(10**3) + plot_dispersion_fixed = plot_dispersion*1000 + + #Get radial dispersion value for each PBH, given PBH radius. + pbh_dispersion = np.interp(r, plot_r_val_kpc, plot_dispersion_fixed) + + #Defining some parameters that are used in the velocity distribution equations + v_esc = 525 #km/s + v_knot = ((2/3)**(1/2))*pbh_dispersion + z=v_esc/v_knot + n_esc=scipy.special.erf(z)-(2*(np.pi**(-1/2))*z*(np.exp(-z**2))) + v=np.arange(0,525, 525/len(n_esc)) + + #Getting velocity for each PBH using formulas from Ranjan Laha 2018. + for disp in pbh_dispersion: + f = ((3/(2*np.pi*(disp**2)))**(3/2))*(np.exp((-(v**2))/(2*(disp**2)))) + cdf = (3*(3**(1/2))*(1/(disp**2))*disp*scipy.special.erf(v/((2**(1/2))*disp)))/(4*np.pi) + rand_cdf = np.random.uniform(0, np.amax(cdf), 1) + interpreted_rms_velocities = np.interp(rand_cdf, cdf, v) + + #Sampling random lattitude and longitude values for velocity to complete the spherical velocities + sin_lat_vel = np.random.uniform(-1, 1, len(data_in_field)) + lat_vel = np.arcsin(sin_lat_vel) + long_vel = np.random.uniform(0, 2*np.pi, len(data_in_field)) + + #Transforming velocities to cartesian to get vx, vy, vz values + cart_vel = astropy.coordinates.spherical_to_cartesian(interpreted_rms_velocities, lat_vel, long_vel) + vx= cart_vel[0] + vy= cart_vel[1] + vz= cart_vel[2] + + #Getting rest of data for PBHs for the combined .h5 file + mass = np.full((len(data_in_field), 1), pbh_mass) + mass = np.reshape(mass, (len(data_in_field),)) + zams_mass = mass + age = np.full((len(data_in_field), 1), 14*(10**9)) + age = np.reshape(age, (len(data_in_field),)) + pop_id = np.full((len(data_in_field), 1), 10) + pop_id = np.reshape(pop_id, (len(data_in_field),)) + rem_id = np.full((len(data_in_field), 1), 104) + rem_id = np.reshape(rem_id, (len(data_in_field),)) + + l = data_in_field.l.deg + b = data_in_field.b.deg + b_rad = np.radians(b) + l_rad = np.radians(l) + + cart_helio = astropy.coordinates.spherical_to_cartesian(r, b_rad, l_rad) + + px = cart_helio[0] + py = cart_helio[1] + pz = cart_helio[2] + + calc_sph_motion(vx, vy, vz, r, b, l) + + exbv = np.full((len(data_in_field), 1), 0) + exbv = np.reshape(exbv, (len(data_in_field),)) + ubv_k = np.full((len(data_in_field), 1), np.nan) + ubv_k = np.reshape(ubv_k, (len(data_in_field),)) + ubv_j = np.full((len(data_in_field), 1), np.nan) + ubv_j = np.reshape(ubv_j, (len(data_in_field),)) + obj_id = np.arange(1, (len(data_in_field)+1)) + ubv_i = np.full((len(data_in_field), 1), np.nan) + ubv_i = np.reshape(ubv_i, (len(data_in_field),)) + ubv_u = np.full((len(data_in_field), 1), np.nan) + ubv_u = np.reshape(ubv_u, (len(data_in_field),)) + ubv_r = np.full((len(data_in_field), 1), np.nan) + ubv_r = np.reshape(ubv_r, (len(data_in_field),)) + ubv_b = np.full((len(data_in_field), 1), np.nan) + ubv_b = np.reshape(ubv_b, (len(data_in_field),)) + ubv_h = np.full((len(data_in_field), 1), np.nan) + ubv_h = np.reshape(ubv_h, (len(data_in_field),)) + ubv_v = np.full((len(data_in_field), 1), np.nan) + ubv_v = np.reshape(ubv_v, (len(data_in_field),)) + + pbh_data = pd.DataFrame({'zams_mass':zams_mass, 'rem_id':rem_id, 'mass':mass, 'px':px, 'py':py, 'pz':pz, 'vx':vx, 'vy':vy, 'vz':vz, 'rad':r, 'glat':b, 'glon':l, 'vr':vr, 'mu_b':mu_b, 'mu_lcosb':mu_lcosb, 'age':age, 'popid':pop_id, 'ubv_k':ubv_k, 'ubv_i':ubv_i, 'exbv':exbv, 'obj_id':obj_id, 'ubv_j':ubv_j, 'ubv_u':ubv_u, 'ubv_r':ubv_r, 'ubv_b':ubv_b, 'ubv_h':ubv_h, 'ubv_v':ubv_v}) + + #Getting information out of .h5 file before PBHs + no_pbh_hdf5_file = h5py.File(hdf5_file) + + for key in key_list: + keys = pd.DataFrame(np.array(no_pbh['{}'.format(key)])) + vars()[key+'_max_l'] = np.amax(keys.T[11]) + vars()[key+'_min_l'] = np.amin(keys.T[11]) + vars()[key+'_max_b'] = np.amax(keys.T[10]) + vars()[key+'_min_b'] = np.amin(keys.T[10]) + vars()[key+'_mask'] = (pbh_data['glon'] >= vars()[key+'_min_l']) & (pbh_data['glon'] <= vars()[key+'_max_l']) & (pbh_data['glat'] >= vars()[key+'_min_b']) & (pbh_data['glat'] <= vars()[key+'_max_b']) + vars()['pbh_'+key] = (pbh_data[vars()[key+'_mask']]).T + vars()['pbh_'+key].reset_index(drop=True, inplace=True) + vars()['full_'+key]=pd.concat([keys, vars()['pbh_'+key]], axis=1) + no_pbh_hdf5_file.close() + + #Creating final .h5 file with PBHs injected + f = h5py.File(run_key + '_with_PBH.h5', 'w') + + for key in key_list: + vars()['d_'+key]=f.create_dataset(key, (vars()['full_'+key].shape[0], vars()['full_'+key].shape[1]), data=vars()['full_'+key]) + + d_lat = f.create_dataset('lat_bin_edges', (len(lat_bin), 1), data=lat_bin) + d_long = f.create_dataset('long_bin_edges', (len(lat_bin), 1), data=long_bin) + f.close() return -# TODO, write this for pbhs -def generate_pbh_config_file(config_filename, radius_cut, obs_time, - n_obs, theta_frac, blend_rad, - isochrones_dir, - bin_edges_number, - BH_kick_speed_mean, NS_kick_speed_mean, - filter_name, red_law): +def generate_pbh_config_file(config_filename, galaxia_area, fdm, pbh_mass, r_max, c, r_vir): """ - Save popsycle configuration parameters from a dictionary into a yaml file + Save PBH configuration parameters into a yaml file Parameters ---------- - config_filename : str - Name of the configuration file - - radius_cut : float - Initial radius cut, in ARCSECONDS. - - obs_time : float - Survey duration, in DAYS. + galaxia_area : float + How many square degrees did you run galaxia on. + This goes into calculating the field of view area, which is used to cut + a rectangular mask down into a circular mask for determining positions of + PBHs. - n_obs : float - Number of observations. - - theta_frac : float - Another cut, in multiples of Einstein radii. - - blend_rad : float - Stars within this distance of the lens are said to be blended. - Units are in ARCSECONDS. - - isochrones_dir : str - Directory for PyPopStar isochrones + fdm : float + Fraction of dark matter. + The fraction of dark matter that you want to consist of PBHs. + Defaults to 1. - bin_edges_number : int - Number of edges for the bins (bins = bin_edges_number - 1) - Total number of bins is (bin_edges_number - 1)**2 + pbh_mass : int + The single mass that all PBHs will have. + Defaults to 40 Msun (from LIGO detections thought to be primordial) - BH_kick_speed_mean : float - Mean of the birth kick speed of BH (in km/s) maxwellian distrubution. - - NS_kick_speed_mean : float - Mean of the birth kick speed of NS (in km/s) maxwellian distrubution. + r_max : float + The maximum radius from the galactic center that you want to find PBHs at. + Defaults to 8.3 kpc (Where Earth is located) - filter_name : str - The name of the filter in which to calculate all the - microlensing events. The filter name convention is set - in the global filt_dict parameter at the top of this module. + c : float + Concentration index. + Defaults to 10 (concentration index of the Milky Way, multiple papers show it to be anywhere from 10-12) - red_law : str - The name of the reddening law to use from PopStar. + r_vir : float + The virial radius. + Defaults to 200 kpc (The virial radius of the Milky Way) Output ------ None """ - config = {'radius_cut': radius_cut, - 'obs_time': obs_time, - 'n_obs': n_obs, - 'theta_frac': theta_frac, - 'blend_rad': blend_rad, - 'isochrones_dir': isochrones_dir, - 'bin_edges_number': bin_edges_number, - 'BH_kick_speed_mean': BH_kick_speed_mean, - 'NS_kick_speed_mean': NS_kick_speed_mean, - 'filter_name': filter_name, - 'red_law': red_law} + config = {'galaxia_area': galaxia_area, + 'fdm': fdm, + 'pbh_mass': pbh_mass, + 'r_max': r_max, + 'c': c, + 'r_vir': r_vir} generate_config_file(config_filename, config) From 00002f3c02e2730ca687f7ab163497ac90c8f8c3 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Tue, 4 Feb 2020 13:22:25 -0800 Subject: [PATCH 003/125] small bug fixes and typo fixes --- popsycle/synthetic.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 2db49206..5a871b08 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1,6 +1,8 @@ import numpy as np +import pandas as pd import h5py import math +import astropy from astropy import units from scipy.stats import maxwell import astropy.coordinates as coord @@ -11,8 +13,10 @@ from astropy.table import vstack from popstar.imf import imf from popstar import synthetic, evolution, reddening, ifmr +import scipy from scipy.interpolate import interp1d from scipy.spatial import cKDTree +from scipy import special import time import datetime from popsycle import ebf @@ -26,6 +30,7 @@ import inspect import sys + ########## # Conversions. ########## @@ -1233,7 +1238,7 @@ def calc_events(hdf5_file, output_root2, ######### # Set random seed - np.random.seed(seed + 1) + np.random.seed(seed) t0 = time.time() @@ -1727,7 +1732,7 @@ def _calc_blends(bigpatch, c, event_lbt, blend_rad): # Query ball against the existing (cached) tree. # NOTE: results is an array of lists. - results = kdtree_cache.query_ball_point(flatxyz1.T, r_kdt) + results = kdtree_cache.query_ball_point(flatxyz1.T.copy(order='C'), r_kdt) # Figure out the number of blends for each lens. blend_lens_obj_id = [] @@ -3639,8 +3644,8 @@ def add_pbh(hdf5_file, output_root2, galaxia_area, fdm, pbh_mass, r_max, c, r_vi no_pbh_hdf5_file = h5py.File(hdf5_file) key_list = list(no_pbh_hdf5_file) key_list = key_list[:len(key_list)-2] - lat_bin = pd.DataFrame(np.array(no_pbh['lat_bin_edges'])) - long_bin = pd.DataFrame(np.array(no_pbh['long_bin_edges'])) + lat_bin = pd.DataFrame(np.array(no_pbh_hdf5_file['lat_bin_edges'])) + long_bin = pd.DataFrame(np.array(no_pbh_hdf5_file['long_bin_edges'])) no_pbh_hdf5_file.close() field_of_view_area = 2*((galaxia_area/np.pi)**(1/2)) @@ -3708,9 +3713,9 @@ def qnfw(p, con=5, logp=False): cart = astropy.coordinates.spherical_to_cartesian(r_values, lats, longs) #Defining galactocentric coordinates - c = coord.Galactocentric(x=cart[0] * u.kpc, y=cart[1] * u.kpc, z=cart[2] * u.kpc) + c = coord.Galactocentric(x=cart[0] * units.kpc, y=cart[1] * units.kpc, z=cart[2] * units.kpc) - #Transforming from galactocentric to galactic (heliocentric) coordinates + #Transforming from galactocentric to galactic coordinates #outputs l, b, dist in degrees. c = c.transform_to(coord.Galactic(representation_type='cartesian')) @@ -3724,17 +3729,16 @@ def angdist(ra1, dec1, ra2, dec2): distance = arccos(cosdist) / d2r return distance + #Set minimum and maximum l, b, and r values for PBH mask. #Determining the center of the field of view circle. - l_mid = np.average((l_min, l_max)) - b_mid = np.average((b_min, b_max)) - - dists = angdist(l_mid, b_mid, c.l.deg, c.b.deg) - - #Set minimum and maximum l, b, and r values for PBH mask. l_min = np.min(long_bin).values l_max = np.max(long_bin).values b_min = np.min(lat_bin).values b_max = np.max(lat_bin).values + l_mid = np.average((l_min, l_max)) + b_mid = np.average((b_min, b_max)) + + dists = angdist(l_mid, b_mid, c.l.deg, c.b.deg) r_max_mask = 16.6 #Masking the full PBH data for our field of view @@ -3803,7 +3807,7 @@ def angdist(ra1, dec1, ra2, dec2): py = cart_helio[1] pz = cart_helio[2] - calc_sph_motion(vx, vy, vz, r, b, l) + vr, mu_b, mu_lcosb = calc_sph_motion(vx, vy, vz, r, b, l) exbv = np.full((len(data_in_field), 1), 0) exbv = np.reshape(exbv, (len(data_in_field),)) @@ -3831,7 +3835,7 @@ def angdist(ra1, dec1, ra2, dec2): no_pbh_hdf5_file = h5py.File(hdf5_file) for key in key_list: - keys = pd.DataFrame(np.array(no_pbh['{}'.format(key)])) + keys = pd.DataFrame(np.array(no_pbh_hdf5_file['{}'.format(key)])) vars()[key+'_max_l'] = np.amax(keys.T[11]) vars()[key+'_min_l'] = np.amin(keys.T[11]) vars()[key+'_max_b'] = np.amax(keys.T[10]) @@ -3843,7 +3847,7 @@ def angdist(ra1, dec1, ra2, dec2): no_pbh_hdf5_file.close() #Creating final .h5 file with PBHs injected - f = h5py.File(run_key + '_with_PBH.h5', 'w') + f = h5py.File(output_root2 + '.h5', 'w') for key in key_list: vars()['d_'+key]=f.create_dataset(key, (vars()['full_'+key].shape[0], vars()['full_'+key].shape[1]), data=vars()['full_'+key]) From 42f110e354b972b8d45eb276a244c3d0f889703e Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Tue, 4 Feb 2020 16:09:38 -0800 Subject: [PATCH 004/125] Fixed comments --- popsycle/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index 166c1c7c..dc0a8096 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -201,7 +201,7 @@ def run(): Exiting...""".format(args.pbh_config_filename)) sys.exit(1) - # TODO: check if .h5 file exists from perform popsyn, use as input for following function + # Check if .h5 file exists from perform popsyn, use as input for following function if not os.path.exists({0}+'.h5'.format(args.output_root)): print("""Error: H5 file was not created properly by synthetic.perform_pop_syn""") From ce3bd672b28286112eba2aff8ac8be5df3c5ec48 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Tue, 4 Feb 2020 16:10:56 -0800 Subject: [PATCH 005/125] fixed comments --- popsycle/synthetic.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 5a871b08..22ed4c01 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3536,8 +3536,6 @@ def add_pbh(hdf5_file, output_root2, galaxia_area, fdm, pbh_mass, r_max, c, r_vi '/some/path/to/myout' '../back/to/some/path/myout' - Optional Parameters - ------------------- galaxia_area : float How many square degrees did you run galaxia on. This goes into calculating the field of view area, which is used to cut From db6b3e5e85a219fcca6c6cb21488e4561b635b5f Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Wed, 5 Feb 2020 13:24:15 -0800 Subject: [PATCH 006/125] moved ang_dist, added pbh to generate_slurm, minor fixes --- popsycle/data/pbh_config.yaml | 1 - popsycle/run.py | 8 ++++- popsycle/synthetic.py | 58 +++++++++++++++++++---------------- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/popsycle/data/pbh_config.yaml b/popsycle/data/pbh_config.yaml index d856bd66..6f813f9c 100644 --- a/popsycle/data/pbh_config.yaml +++ b/popsycle/data/pbh_config.yaml @@ -1,6 +1,5 @@ # Example configuration parameters for PBHs -galaxia_area: .001 #How many square degrees are you running galaxia on fdm: 1 #Fraction of dark matter that consists of PBHs pbh_mass: 40 #Msun r_max: 8.3 #kpc, How far from the galactic center that you want to go out diff --git a/popsycle/run.py b/popsycle/run.py index dc0a8096..07bc32d1 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -109,7 +109,7 @@ def run(): help='Name of configuration file containing ' 'pbh inputs. Default if needed is: ' 'pbh_config.yaml', - default=None) + default='pbh_config.yaml') # Check for field config file. Exit if not present. @@ -209,6 +209,12 @@ def run(): synthetic.add_pbh(hdf5_file=filename_dict['hdf5_filename'], output_root2=args.output_root, + galaxia_area=field_config['area'], + fdm=pbh_config['fdm'], + pbh_mass=pbh_config['pbh_mass'], + r_max=pbh_config['r_max'], + c=pbh_config['c'], + r_vir=pbh_config['r_vir'], overwrite=args.overwrite, seed=args.seed) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 22ed4c01..3a2b46c9 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3343,7 +3343,7 @@ def generate_config_file(config_filename, config): def generate_slurm_scripts(slurm_config_filename, popsycle_config_filename, - path_run, output_root, + pbh_config_filename, path_run, output_root, longitude, latitude, area, n_cores_calc_events, walltime, @@ -3362,6 +3362,11 @@ def generate_slurm_scripts(slurm_config_filename, popsycle_config_filename, that will be passed along to the run_on_slurm.py command in the slurm script. + pbh_config_filename : str + Name of pbh_config.yaml file containing the PBH parameters + that will be passed along to the run_on_slurm.py command in the + slurm script. + path_run : str Directory containing the parameter file and PopSyCLE output files @@ -3518,8 +3523,25 @@ def generate_slurm_scripts(slurm_config_filename, popsycle_config_filename, print('Submitted job {0} to {1}'.format(script_filename, resource)) +def angdist(ra1, dec1, ra2, dec2): + ''' + Takes two spherical coordinates and determines the angluar distance between + them. + Input: + ra1,dec1 [degrees] angular coordinates of the first point + ra2,dec2 [degrees] angular coordinates of the second point + Output: + distance [degrees] angular distance between the two points + ''' + from numpy import pi, sin, cos, arccos + d2r = pi / 180.0 + cosdist = (cos(np.absolute(ra1 - ra2) * d2r) * cos(np.absolute(dec1 * d2r)) * cos(np.absolute(dec2 * d2r)) + + sin(np.absolute(dec1 * d2r)) * sin(np.absolute(dec2 * d2r))) + distance = arccos(cosdist) / d2r + return distance + -def add_pbh(hdf5_file, output_root2, galaxia_area, fdm, pbh_mass, r_max, c, r_vir, overwrite = False, seed = None): +def add_pbh(hdf5_file, output_root2, area, fdm, pbh_mass, r_max, c, r_vir, overwrite = False, seed = None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, and saves them in a new HDF5 file with the PBHs added. @@ -3536,11 +3558,8 @@ def add_pbh(hdf5_file, output_root2, galaxia_area, fdm, pbh_mass, r_max, c, r_vi '/some/path/to/myout' '../back/to/some/path/myout' - galaxia_area : float - How many square degrees did you run galaxia on. - This goes into calculating the field of view area, which is used to cut - a rectangular mask down into a circular mask for determining positions of - PBHs. + area : float + Area of the sky that will be generated, in square degrees. fdm : float Fraction of dark matter. @@ -3563,6 +3582,9 @@ def add_pbh(hdf5_file, output_root2, galaxia_area, fdm, pbh_mass, r_max, c, r_vi The virial radius. Defaults to 200 kpc (The virial radius of the Milky Way) + Optional Parameters + ------------------- + overwrite : bool If set to True, overwrites output files. If set to False, exists the function if output files are already on disk. @@ -3646,7 +3668,7 @@ def add_pbh(hdf5_file, output_root2, galaxia_area, fdm, pbh_mass, r_max, c, r_vi long_bin = pd.DataFrame(np.array(no_pbh_hdf5_file['long_bin_edges'])) no_pbh_hdf5_file.close() - field_of_view_area = 2*((galaxia_area/np.pi)**(1/2)) + field_of_view_area = 2*((area/np.pi)**(1/2)) #NFW Profile calculations to determine mass of dark matter within given distance of galactic center rho_crit = ((3*(h**2))/(8*np.pi*g))*(10**6) #Msun/Mpc^3 @@ -3717,16 +3739,6 @@ def qnfw(p, con=5, logp=False): #outputs l, b, dist in degrees. c = c.transform_to(coord.Galactic(representation_type='cartesian')) - #Defining function to use in PBH mask (this portion of the mask creates circular shape - #from a rectangular shape because our field of view is circular) - def angdist(ra1, dec1, ra2, dec2): - from numpy import pi, sin, cos, arccos - d2r = pi / 180.0 - cosdist = (cos(np.absolute(ra1 - ra2) * d2r) * cos(np.absolute(dec1 * d2r)) * cos(np.absolute(dec2 * d2r)) + - sin(np.absolute(dec1 * d2r)) * sin(np.absolute(dec2 * d2r))) - distance = arccos(cosdist) / d2r - return distance - #Set minimum and maximum l, b, and r values for PBH mask. #Determining the center of the field of view circle. l_min = np.min(long_bin).values @@ -3856,17 +3868,12 @@ def angdist(ra1, dec1, ra2, dec2): return -def generate_pbh_config_file(config_filename, galaxia_area, fdm, pbh_mass, r_max, c, r_vir): +def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir): """ Save PBH configuration parameters into a yaml file Parameters ---------- - galaxia_area : float - How many square degrees did you run galaxia on. - This goes into calculating the field of view area, which is used to cut - a rectangular mask down into a circular mask for determining positions of - PBHs. fdm : float Fraction of dark matter. @@ -3894,8 +3901,7 @@ def generate_pbh_config_file(config_filename, galaxia_area, fdm, pbh_mass, r_max None """ - config = {'galaxia_area': galaxia_area, - 'fdm': fdm, + config = {'fdm': fdm, 'pbh_mass': pbh_mass, 'r_max': r_max, 'c': c, From edb57403bc37ab66c5ca8c593c4aaf8c10210836 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Wed, 5 Feb 2020 20:31:05 -0800 Subject: [PATCH 007/125] typo fix --- popsycle/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index c8d2bccd..e2e67756 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3508,7 +3508,7 @@ def generate_slurm_scripts(slurm_config_filename, popsycle_config_filename, if pbh_config_filename: pbh_cmd = '--pbh-config-filename={0}'.format(pbh_config_filename) else: - seed_cmd = '' + pbh_cmd = '' # Populate the mpi_template specified inputs job_script = slurm_template.format(**locals()) From cd6bbdf76e864c7683d1a0391843a9817357fccf Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 14 Feb 2020 14:29:18 -0800 Subject: [PATCH 008/125] Changed way PBH velocities are calculated --- popsycle/data/blue_line.csv | 66 ++++++++++++++++++++++++++++++++++++ popsycle/data/green_line.csv | 62 +++++++++++++++++++++++++++++++++ popsycle/data/red_line.csv | 64 ++++++++++++++++++++++++++++++++++ popsycle/run.py | 2 ++ popsycle/synthetic.py | 58 ++++++++++++++++--------------- 5 files changed, 225 insertions(+), 27 deletions(-) create mode 100644 popsycle/data/blue_line.csv create mode 100644 popsycle/data/green_line.csv create mode 100644 popsycle/data/red_line.csv diff --git a/popsycle/data/blue_line.csv b/popsycle/data/blue_line.csv new file mode 100644 index 00000000..43364660 --- /dev/null +++ b/popsycle/data/blue_line.csv @@ -0,0 +1,66 @@ +x,y +0.010792642776929309, 204.26643023097807 +0.012590321940393834, 210.92487127516114 +0.014687431970009386, 218.17494681231219 +0.01713384764066691, 225.4158429807519 +0.019987751131241467, 232.89705353032554 +0.023317015749352955, 240.21341604790845 +0.02720082013657757, 247.6177427638289 +0.031731531344141896, 255.39654951292243 +0.03701690156357718, 262.5159561489671 +0.04318263075634799, 269.98842786464206 +0.05037535612850712, 277.3556810971669 +0.05876613954792163, 284.43477324360026 +0.0685545358439959, 291.3605743829068 +0.07997333874472501, 298.11329956878745 +0.0932941173219056, 304.6732953099007 +0.10883367461568658, 311.0211336366553 +0.1269615820414861, 316.5932048851325 +0.14810896876722457, 322.0805612681069 +0.17277877509531006, 326.913144325661 +0.20155771370168304, 331.43832299459757 +0.235130223201544, 335.44920768526305 +0.2742947458941446, 338.73163728292803 +0.3199827168140894, 342.046186032515 +0.3732807156985778, 343.8140575474295 +0.4354563087024429, 346.3837933152392 +0.5079881944447303, 346.3837933152392 +0.5926013713388398, 346.3837933152392 +0.6913081625775506, 346.3837933152392 +0.8064560744546262, 343.61717701711524 +0.9407836262193813, 341.654561829232 +1.0974854792445174, 338.15005999198166 +1.2802884144496132, 334.10688186636213 +1.4935399649225864, 329.73408677356304 +1.7423118116552716, 324.4878564913184 +2.03252040141478, 318.41185653421763 +2.3710676553598873, 312.4496290180272 +2.7660051148222395, 306.07263557365945 +3.2267254280694635, 299.1396191737312 +3.764185732107431, 292.02890586026 +4.391168242126657, 285.9048791942064 +5.122584246093038, 278.7891986399261 +5.975828734271417, 271.383867634092 +6.971194097701775, 265.692780180841 +8.132352734462563, 258.3391991378944 +9.48692004136333, 253.79240406567155 +11.067111180485346, 247.90157653062977 +12.970922127990187, 241.5318118152786 +15.060949011985775, 236.1208479753841 +17.618863818498795, 229.54628085055066 +20.496060936083154, 223.2314835842758 +23.909992300202614, 217.3019029730845 +27.892565970532186, 211.4086969647845 +32.53849799081271, 206.147098218244 +37.958280805597894, 199.9828369269898 +44.28081106028432, 193.7807781680185 +51.65645510129202, 186.37803645873726 +60.26062508202531, 179.36080012164197 +70.29795072379233, 172.80561961785477 +82.0071459470937, 165.2548659942107 +95.6666861145325, 158.0340430777453 +111.60143090506897, 149.06535054405288 +130.19035032893146, 139.90937104204863 +151.8755376280755, 130.51589873409813 +177.1727234125981, 121.49765534647908 +195.4228471568372, 116.70016866901982 diff --git a/popsycle/data/green_line.csv b/popsycle/data/green_line.csv new file mode 100644 index 00000000..b79e6068 --- /dev/null +++ b/popsycle/data/green_line.csv @@ -0,0 +1,62 @@ +x,y +0.010792642776929309, 111.71180517031452 +0.012590321940393834, 118.88165941281024 +0.014687431970009386, 126.54067684534101 +0.01713384764066691, 134.0005466632754 +0.01993184060733583, 142.11168012051704 +0.023448011680987736, 150.78286434254127 +0.027455988964315874, 159.3571491494456 +0.03213404568533922, 168.4661816124074 +0.03764430047370784, 177.73468233692108 +0.043859894720648336, 186.54417763189218 +0.05108586704256386, 196.01349797292718 +0.05934511962521002, 204.67635482054385 +0.07083192093472244, 215.1824749096273 +0.08253367166054373, 223.90379647052103 +0.09684442649610563, 232.571184061413 +0.11350401149180162, 242.54850927105662 +0.13427738741960332, 251.89354878059038 +0.1586056689369005, 261.0497880274381 +0.186179188876288, 269.5935014641335 +0.21617856025114549, 276.95879153868964 +0.2526914812832889, 283.38932791498735 +0.2962593087128887, 290.36094394136614 +0.35792137580603484, 297.77197612125576 +0.4309060654759373, 304.89151975943804 +0.5059594624841087, 309.244695083941 +0.5778483816986392, 312.37804867591933 +0.6816933312568133, 314.7048135621079 +0.927699047785235, 320.24095532420716 +1.0671697449159139, 319.8132280123578 +1.4727675574625494, 314.96530138192946 +1.7718422284532012, 311.9846498190986 +2.1197411537756987, 307.83085241102907 +2.4598627413134806, 301.78571980162417 +2.8847016095688685, 295.5628794127563 +3.369909155131431, 289.9952727173605 +4.037236963747592, 283.1343596020251 +4.820932000343341, 279.00220082094125 +5.650248971118326, 275.5083826876096 +6.703101332022676, 271.60158332331133 +7.770476112914143, 265.4265828672343 +9.096562802640236, 259.22858778705637 +10.611733993556404, 252.9216664462353 +12.3792800416127, 246.90957929498413 +14.44123782613885, 240.62655413150787 +16.846646109472406, 235.17599223163336 +19.652711807301024, 229.3228663966612 +22.766181747138315, 223.97434631744974 +26.744873675700347, 218.17494681231219 +31.068822348129252, 212.83026716581725 +36.396415667350134, 206.61996451113927 +43.05764653615527, 199.86831942509124 +50.22955388134484, 193.3372953786207 +58.59605171407531, 186.69858418697999 +68.54786206569099, 179.3094384837924 +79.24703527090092, 171.75301799473309 +93.02409457669958, 163.93495203048826 +108.51867546639197, 155.9656915244779 +126.5941149824405, 147.0301382654459 +147.68029446830633, 138.0518741736095 +172.27869855775518, 128.43932707311797 +192.7048730125345, 121.55334325539347 diff --git a/popsycle/data/red_line.csv b/popsycle/data/red_line.csv new file mode 100644 index 00000000..f082fbec --- /dev/null +++ b/popsycle/data/red_line.csv @@ -0,0 +1,64 @@ +x,y +0.010792642776929309, 51.189721234746486 +0.012590321940393834, 56.2637730438137 +0.014687431970009386, 61.73460135767326 +0.01713384764066691, 67.58236521833585 +0.019987751131241467, 73.81473571195039 +0.023317015749352955, 80.43733893511352 +0.02720082013657757, 87.40343466110791 +0.031731531344141896, 94.80975218695623 +0.03701690156357718, 102.66708492081379 +0.04318263075634799, 110.8576402505859 +0.05037535612850712, 119.0179284411037 +0.05876613954792163, 127.63259980070472 +0.0685545358439959, 136.4012214489796 +0.07997333874472501, 145.5219851562832 +0.0932941173219056, 155.0748727428563 +0.10883367461568658, 164.21690036223077 +0.1269615820414861, 173.59930211335876 +0.14810896876722457, 183.3076420995835 +0.17277877509531006, 193.00534947186358 +0.20155771370168304, 202.28701613851908 +0.235130223201544, 211.89363246804209 +0.2742947458941446, 221.19495574296548 +0.3199827168140894, 230.37612812090563 +0.3732807156985778, 239.11517900904093 +0.4354563087024429, 247.7596190021874 +0.5079881944447303, 255.5428827333543 +0.5926013713388398, 263.2688804002836 +0.6913081625775506, 269.98842786464206 +0.8064560744546262, 275.92953461668526 +0.9407836262193813, 282.1629525216277 +1.0974854792445174, 284.43477324360026 +1.2802884144496132, 287.7119979375535 +1.4935399649225864, 289.6964297280955 +1.7423118116552716, 290.02849651765854 +2.03252040141478, 292.53116033258596 +2.3710676553598873, 288.7025088087155 +2.7660051148222395, 288.5371871710757 +3.2267254280694635, 285.9048791942064 +3.764185732107431, 282.1629525216277 +4.340214672149436, 278.4168360406884 +6.424289627691652, 275.5083826876096 +7.582335742589992, 269.21628633919846 +8.942171302597005, 263.2521253122571 +10.318607100674031, 258.0434148632248 +12.037328396676813, 252.19833425421467 +14.042328922474109, 246.06245599323356 +16.381292847455335, 240.62655413150787 +19.109846866257, 236.1208479753841 +22.29288314740797, 229.84889395002654 +26.00610263923727, 224.89996081862702 +30.337815436904652, 218.4250315448684 +35.39104102800242, 213.35512334606335 +41.285958366076336, 207.68788330277627 +48.16276404123605, 200.44156345555936 +56.185006522648195, 193.66981219227398 +65.54347576993862, 187.55607596607996 +76.46074071866175, 180.18459004092205 +89.19644255313948, 172.31141178905165 +104.05346965457406, 163.6534877832804 +121.38516108089176, 156.32345022691644 +141.603709895957, 146.35792791879018 +165.18996620135306, 135.46686778654006 +187.38180510452284, 127.19470448022776 diff --git a/popsycle/run.py b/popsycle/run.py index 019ee74d..dafe913f 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -207,6 +207,8 @@ def run(): r_max=pbh_config['r_max'], c=pbh_config['c'], r_vir=pbh_config['r_vir'], + inner_slope=args.inner_slope, + v_esc=args.v_esc, overwrite=args.overwrite, seed=args.seed) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index e2e67756..62adcfb6 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3546,7 +3546,7 @@ def angdist(ra1, dec1, ra2, dec2): return distance -def add_pbh(hdf5_file, output_root2, area, fdm, pbh_mass, r_max, c, r_vir, overwrite = False, seed = None): +def add_pbh(hdf5_file, output_root2, area, fdm, pbh_mass, r_max, c, r_vir, inner_slope = .5, v_esc = 550, overwrite = False, seed = None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, and saves them in a new HDF5 file with the PBHs added. @@ -3589,6 +3589,14 @@ def add_pbh(hdf5_file, output_root2, area, fdm, pbh_mass, r_max, c, r_vir, overw Optional Parameters ------------------- + inner_slope: float + The inner slope of the MW halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. + Inner_slope goes into the determination of the velocities and each value returns a slightly different distribution. + The default value is .5 because it is in the middle of the options. More investigation is needed. + v_esc: int + The escape velocity of the Milky Way. + v_esc is used in calculating the velocities. + Default is 550 because most papers cite values of 515-575, with a lot being around 550. overwrite : bool If set to True, overwrites output files. If set to False, exists the @@ -3762,32 +3770,30 @@ def qnfw(p, con=5, logp=False): #Obtain the radius values for all of the PBHs in our field of view. r = data_in_field.distance.kpc + l = data_in_field.l.deg + b = data_in_field.b.deg + + c_pbh = coord.Galactic(l=l * u.deg, b=b * u.deg, distance=r * u.kpc) + c_pbh = c_pbh.transform_to(coord.Galactocentric(representation_type='spherical')) + cart_pbh = astropy.coordinates.cartesian_to_spherical(c_pbh.x, c_pbh.y, c_pbh.z) + pbh_r_galacto = cart[0] + + if inner_slope == 1: + vel_data = pd.read_csv('data/red_line.csv') + elif inner_slope == .25: + vel_data = pd.read_csv('data/blue_line.csv') + else: + vel_data = pd.read_csv('data/green_line.csv') + + pbh_vrms = np.interp(pbh_r_galacto, vel_data['x'], vel_data['y']) + v_vals = np.arange(0, v_esc) #Goes from v to v_esc + a = (1/2)*pbh_vrms*((np.pi/2)**(1/2)) - #Getting radial dispersion vs. radius distribution from the M. Hoeft 2018 paper. - disp_data = pd.read_csv('data/radial_velocity_dispersion_digitized.csv') - plot_r_val = disp_data['radius'].values - plot_dispersion = disp_data.iloc[:,1].values - - #Adjusting for the correct units - plot_r_val_kpc = plot_r_val*(10**3) - plot_dispersion_fixed = plot_dispersion*1000 - - #Get radial dispersion value for each PBH, given PBH radius. - pbh_dispersion = np.interp(r, plot_r_val_kpc, plot_dispersion_fixed) - - #Defining some parameters that are used in the velocity distribution equations - v_esc = 525 #km/s - v_knot = ((2/3)**(1/2))*pbh_dispersion - z=v_esc/v_knot - n_esc=scipy.special.erf(z)-(2*(np.pi**(-1/2))*z*(np.exp(-z**2))) - v=np.arange(0,525, 525/len(n_esc)) - - #Getting velocity for each PBH using formulas from Ranjan Laha 2018. - for disp in pbh_dispersion: - f = ((3/(2*np.pi*(disp**2)))**(3/2))*(np.exp((-(v**2))/(2*(disp**2)))) - cdf = (3*(3**(1/2))*(1/(disp**2))*disp*scipy.special.erf(v/((2**(1/2))*disp)))/(4*np.pi) + for a_val in a: + pdf = ((2/np.pi)**(1/2))*((v_vals**2*np.exp(-v_vals**2/(2*a_val**2)))/a_val**3) + cdf = scipy.special.erf(v_vals/(a_val*2**(1/2)))-(((2/np.pi)**(1/2))*((v_vals*np.exp(-v**2/2*a_val**2))/a_val)) rand_cdf = np.random.uniform(0, np.amax(cdf), 1) - interpreted_rms_velocities = np.interp(rand_cdf, cdf, v) + interpreted_rms_velocities = np.interp(rand_cdf, cdf, v_vals) #Sampling random lattitude and longitude values for velocity to complete the spherical velocities sin_lat_vel = np.random.uniform(-1, 1, len(data_in_field)) @@ -3811,8 +3817,6 @@ def qnfw(p, con=5, logp=False): rem_id = np.full((len(data_in_field), 1), 104) rem_id = np.reshape(rem_id, (len(data_in_field),)) - l = data_in_field.l.deg - b = data_in_field.b.deg b_rad = np.radians(b) l_rad = np.radians(l) From 16dbb1fa7afea03683058554500a154fe21df2a9 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 14 Feb 2020 14:30:52 -0800 Subject: [PATCH 009/125] Delete radial_velocity_dispersion_digitized.csv don't need anymore --- .../radial_velocity_dispersion_digitized.csv | 59 ------------------- 1 file changed, 59 deletions(-) delete mode 100644 popsycle/data/radial_velocity_dispersion_digitized.csv diff --git a/popsycle/data/radial_velocity_dispersion_digitized.csv b/popsycle/data/radial_velocity_dispersion_digitized.csv deleted file mode 100644 index 4864069f..00000000 --- a/popsycle/data/radial_velocity_dispersion_digitized.csv +++ /dev/null @@ -1,59 +0,0 @@ -radius, dispersion -0.0007360446241274024, 0.07906914418438438 -0.0008534023145816314, 0.0783151688404502 -0.0009352493660181735, 0.09928939204443932 -0.0010616920597239583, 0.0948487710252941 -0.001131186315865826, 0.07058692156512414 -0.0012223307362665052, 0.08917926811441179 -0.0014072747508446864, 0.10059171309305304 -0.0014006809548853252, 0.09339467571913518 -0.0016481597082506775, 0.0995264956080697 -0.00191865661207353, 0.10597235532022009 -0.0022518088984294273, 0.10372756509168868 -0.002236359182974952, 0.11113665803108796 -0.002652983994959231, 0.10806292331931067 -0.0030977317671075488, 0.11191847905533803 -0.003216334330268018, 0.12074722569630558 -0.0036221371499138544, 0.10862497766660706 -0.0037732018429818596, 0.12255981288677364 -0.004077224522720631, 0.11224405931749148 -0.004431684223884757, 0.12149072916046288 -0.00507711867557682, 0.12835958416876758 -0.006006722605063362, 0.13438351714392444 -0.006933859783391445, 0.13789754680086547 -0.007850528352299446, 0.14344664349407954 -0.008160676858521945, 0.15484366624977663 -0.009166595468236206, 0.14295141877954093 -0.010703288837072599, 0.14740672763006152 -0.01249759437151708, 0.1496001104487793 -0.01459269832408766, 0.15175922166095468 -0.01703902671566554, 0.15295872788994097 -0.019895459014453068, 0.15350707359462046 -0.023230745276774503, 0.15511783910211635 -0.02712516085818071, 0.15532346874137115 -0.031672438521280974, 0.15583754283950818 -0.03672240983289494, 0.1546811426756397 -0.04237806984137696, 0.1474672741349532 -0.049715295366751476, 0.1448363571393766 -0.05836502438103976, 0.14088223391957533 -0.0680202364114164, 0.14458331844440475 -0.06929852326934813, 0.13362464585619843 -0.07947915979047598, 0.13753993210648552 -0.09241165369047062, 0.13308119609531072 -0.1079035967920762, 0.131333344161645 -0.12182981941605571, 0.12177314770275272 -0.1430264132617661, 0.11525996069322841 -0.16700344394773997, 0.11056475059691062 -0.19499999793297998, 0.1044986762388942 -0.22768991042939973, 0.0982269722416228 -0.26214039495405517, 0.09037877434339814 -0.3018033892545949, 0.08338736660873514 -0.3523978840773408, 0.07845225526662003 -0.41147406929028135, 0.07620936901622977 -0.46710403451832827, 0.07134269208844868 -0.5071166622055777, 0.0619601870049431 -0.5935221015515242, 0.05919161238975412 -0.6930204901160147, 0.06107655074958973 -0.8091988461173554, 0.06330420517485014 -0.9537702647588352, 0.05996843547138331 -0.9926153602554509, 0.053433982490619925 From 744c8c7d7ee24c006c946b9cca1a3ba1670c6d90 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 14:19:14 -0800 Subject: [PATCH 010/125] Add files via upload minor fixes --- popsycle/run.py | 2 +- popsycle/synthetic.py | 160 ++++++++++++++++++++++-------------------- 2 files changed, 85 insertions(+), 77 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index dafe913f..aee8e0e9 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -200,8 +200,8 @@ def run(): sys.exit(1) synthetic.add_pbh(hdf5_file=filename_dict['hdf5_filename'], + ebf_file=filename_dict['ebf_filename'], output_root2=args.output_root, - galaxia_area=field_config['area'], fdm=pbh_config['fdm'], pbh_mass=pbh_config['pbh_mass'], r_max=pbh_config['r_max'], diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 62adcfb6..4fd4f29c 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3539,14 +3539,14 @@ def angdist(ra1, dec1, ra2, dec2): distance [degrees] angular distance between the two points ''' from numpy import pi, sin, cos, arccos - d2r = pi / 180.0 - cosdist = (cos(np.absolute(ra1 - ra2) * d2r) * cos(np.absolute(dec1 * d2r)) * cos(np.absolute(dec2 * d2r)) + - sin(np.absolute(dec1 * d2r)) * sin(np.absolute(dec2 * d2r))) - distance = arccos(cosdist) / d2r - return distance + d2r = pi / 180.0 + cosdist = (cos(np.absolute(ra1 - ra2) * d2r) * cos(np.absolute(dec1 * d2r)) * cos(np.absolute(dec2 * d2r)) + + sin(np.absolute(dec1 * d2r)) * sin(np.absolute(dec2 * d2r))) + distance = arccos(cosdist) / d2r + return distance -def add_pbh(hdf5_file, output_root2, area, fdm, pbh_mass, r_max, c, r_vir, inner_slope = .5, v_esc = 550, overwrite = False, seed = None): +def add_pbh(hdf5_file, ebf_file, output_root2, fdm, pbh_mass, r_max, c, r_vir, inner_slope = .5, v_esc = 550, overwrite = False, seed = None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, and saves them in a new HDF5 file with the PBHs added. @@ -3556,6 +3556,10 @@ def add_pbh(hdf5_file, output_root2, area, fdm, pbh_mass, r_max, c, r_vir, inner hdf5_file : str or hdf5 file str : name of the hdf5 file from the output of perform_pop_syn + ebf_file : str or ebf file + str : name of the ebf file from Galaxia + ebf file : actually the ebf file from Galaxia + output_root2 : str The thing you want the output files to be named Examples: @@ -3563,9 +3567,6 @@ def add_pbh(hdf5_file, output_root2, area, fdm, pbh_mass, r_max, c, r_vir, inner '/some/path/to/myout' '../back/to/some/path/myout' - area : float - Area of the sky that will be generated, in square degrees. - fdm : float Fraction of dark matter. The fraction of dark matter that you want to consist of PBHs. @@ -3593,6 +3594,7 @@ def add_pbh(hdf5_file, output_root2, area, fdm, pbh_mass, r_max, c, r_vir, inner The inner slope of the MW halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. Inner_slope goes into the determination of the velocities and each value returns a slightly different distribution. The default value is .5 because it is in the middle of the options. More investigation is needed. + v_esc: int The escape velocity of the Milky Way. v_esc is used in calculating the velocities. @@ -3628,13 +3630,12 @@ def add_pbh(hdf5_file, output_root2, area, fdm, pbh_mass, r_max, c, r_vir, inner # Error handling/complaining if input types are not right. + if ebf_file[-4:] != '.ebf': + raise Exception('ebf_file must be an ebf file.') + if type(output_root2) != str: raise Exception('output_root must be a string.') - if type(galaxia_area) != float: - if type(galaxia_area) != int: - raise Exception('galaxia_area must be a float or an integer.') - if type(fdm) != float: if type(fdm) != int: raise Exception('fdm must be a float or an integer.') @@ -3674,27 +3675,47 @@ def add_pbh(hdf5_file, output_root2, area, fdm, pbh_mass, r_max, c, r_vir, inner h = 70 #km/(s*Mpc) #Read in the hdf5 file that doesn't have PBHs. Product of perform_pop_syn. - no_pbh_hdf5_file = h5py.File(hdf5_file) + no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') key_list = list(no_pbh_hdf5_file) - key_list = key_list[:len(key_list)-2] + key_list = [key for key in key_list if 'bin_edges' not in key] lat_bin = pd.DataFrame(np.array(no_pbh_hdf5_file['lat_bin_edges'])) long_bin = pd.DataFrame(np.array(no_pbh_hdf5_file['long_bin_edges'])) no_pbh_hdf5_file.close() - field_of_view_area = 2*((area/np.pi)**(1/2)) + t = ebf.read_ind(ebf_file, '/log', 0) + # Convert log to useful dictionary. + ebf_log = make_ebf_log(t) + + b = float(ebf_log['latitude']) + l = float(ebf_log['longitude']) + surveyArea = float(ebf_log['surveyArea']) + + field_of_view_diameter = 2*((surveyArea/np.pi)**(1/2)) #NFW Profile calculations to determine mass of dark matter within given distance of galactic center rho_crit = ((3*(h**2))/(8*np.pi*g))*(10**6) #Msun/Mpc^3 rho_knot = (200/3)*(rho_crit)*((c**3)/(np.log(c+1)-(c/(1+c)))) # Msun/Mpc^3 - mass_8_kpc = (4*np.pi*rho_knot*(r_s**3)*(np.log((r_s + r_max)/r_s) - (r_max/(r_s+r_max))))*(((10**3)**3)/((10**6)**3))*fdm #Msun + mass_within_r_max = (4*np.pi*rho_knot*(r_s**3)*(np.log((r_s + r_max)/r_s) - (r_max/(r_s+r_max))))*(((10**3)**3)/((10**6)**3))*fdm #Msun #Determine the number of PBHs within that distance - num_pbh_8kpc = (mass_8_kpc/pbh_mass) - num_pbh_8kpc = round(num_pbh_8kpc, -1) + #num_pbh_within_r_max = (mass_within_r_max/pbh_mass) + #num_pbh_within_r_max = round(num_pbh_within_r_max) + num_pbh_within_r_max = 5e7 + + """ + Defining needed functions from the python package "NFWdist". + Used to calculate PBH radii from galactic center assuming NFW profile. + https://github.com/CullanHowlett/NFWdist + + x, q: array_like + Vector of quantiles. This is scaled such that x=R/Rvir for NFW. This means the PDF is only defined between 0 and 1. + p: array_like + Vector of probabilities + + pnfw: distribution function + qnfw: Quantile function (CDF inversion) + """ - #Defining needed functions from the python package "NFWdist". - #Used to calculate PBH radii from galactic center assuming NFW profile. - #https://github.com/CullanHowlett/NFWdist def pnfwunorm(q, con=5): if hasattr(con, '__len__'): y = np.outer(q,con) @@ -3734,13 +3755,13 @@ def qnfw(p, con=5, logp=False): p *= pnfwunorm(1, con=con) return (-(1.0/np.real(special.lambertw(-np.exp(-p-1))))-1)/con - #Calculating radius values for all PBHs within 8.3kpc of the galactic center - r_values = (qnfw(np.random.rand(int(num_pbh_8kpc)) * pnfw(r_max/r_vir,con=c, logp=False), con=c)*r_vir) + #Calculating radius values for all PBHs within r_max of the galactic center, using NFWDist functions + r_values = (qnfw(np.random.rand(int(num_pbh_within_r_max)) * pnfw(r_max/r_vir,con=c, logp=False), con=c)*r_vir) - #Sample PBH lattitude and longitude to have full spherical coordinates. - sin_lats = np.random.uniform(-1, 1, int(num_pbh_8kpc)) + #Sample PBH latitude and longitude to have full spherical coordinates. + sin_lats = np.random.uniform(-1, 1, int(num_pbh_within_r_max)) lats=np.arcsin(sin_lats) - longs = np.random.uniform(0, 2*np.pi, int(num_pbh_8kpc)) + longs = np.random.uniform(0, 2*np.pi, int(num_pbh_within_r_max)) #Converting spherical galactocentric coordinates to cartesian galactocentric coordinates cart = astropy.coordinates.spherical_to_cartesian(r_values, lats, longs) @@ -3751,6 +3772,7 @@ def qnfw(p, con=5, logp=False): #Transforming from galactocentric to galactic coordinates #outputs l, b, dist in degrees. c = c.transform_to(coord.Galactic(representation_type='cartesian')) + print(c.l.deg) #Set minimum and maximum l, b, and r values for PBH mask. #Determining the center of the field of view circle. @@ -3758,44 +3780,43 @@ def qnfw(p, con=5, logp=False): l_max = np.max(long_bin).values b_min = np.min(lat_bin).values b_max = np.max(lat_bin).values - l_mid = np.average((l_min, l_max)) - b_mid = np.average((b_min, b_max)) - dists = angdist(l_mid, b_mid, c.l.deg, c.b.deg) - r_max_mask = 16.6 + dists = angdist(l, b, c.l.deg, c.b.deg) #Masking the full PBH data for our field of view - mask = (c.l.deg >= l_min) & (c.l.deg <= l_max) & (c.b.deg >= b_min) & (c.b.deg <= b_max) & (c.distance.kpc <= r_max_mask) & (dists < field_of_view_area) + mask = (c.distance.kpc <= 2*r_max) & (dists < field_of_view_diameter) data_in_field = c[mask] #Obtain the radius values for all of the PBHs in our field of view. - r = data_in_field.distance.kpc - l = data_in_field.l.deg - b = data_in_field.b.deg + r_in_field = data_in_field.distance.kpc + l_in_field = data_in_field.l.deg + b_in_field = data_in_field.b.deg - c_pbh = coord.Galactic(l=l * u.deg, b=b * u.deg, distance=r * u.kpc) + c_pbh = coord.Galactic(l=l_in_field * units.deg, b=b_in_field * units.deg, distance=r_in_field * units.kpc) c_pbh = c_pbh.transform_to(coord.Galactocentric(representation_type='spherical')) cart_pbh = astropy.coordinates.cartesian_to_spherical(c_pbh.x, c_pbh.y, c_pbh.z) pbh_r_galacto = cart[0] if inner_slope == 1: - vel_data = pd.read_csv('data/red_line.csv') + vel_data = pd.read_csv('data/inner_slope_one.csv') elif inner_slope == .25: - vel_data = pd.read_csv('data/blue_line.csv') + vel_data = pd.read_csv('data/inner_slope_quarter.csv') else: - vel_data = pd.read_csv('data/green_line.csv') + vel_data = pd.read_csv('data/inner_slope_half.csv') pbh_vrms = np.interp(pbh_r_galacto, vel_data['x'], vel_data['y']) v_vals = np.arange(0, v_esc) #Goes from v to v_esc a = (1/2)*pbh_vrms*((np.pi/2)**(1/2)) + rand_cdf = [] + for a_val in a: pdf = ((2/np.pi)**(1/2))*((v_vals**2*np.exp(-v_vals**2/(2*a_val**2)))/a_val**3) - cdf = scipy.special.erf(v_vals/(a_val*2**(1/2)))-(((2/np.pi)**(1/2))*((v_vals*np.exp(-v**2/2*a_val**2))/a_val)) - rand_cdf = np.random.uniform(0, np.amax(cdf), 1) - interpreted_rms_velocities = np.interp(rand_cdf, cdf, v_vals) + cdf = scipy.special.erf(v_vals/(a_val*2**(1/2)))-(((2/np.pi)**(1/2))*((v_vals*np.exp(-v_vals**2/2*a_val**2))/a_val)) + rand_cdf.append(np.random.uniform(0, np.amax(cdf), 1)) + interpreted_rms_velocities = np.interp(rand_cdf, cdf, v_vals) - #Sampling random lattitude and longitude values for velocity to complete the spherical velocities + #Sampling random latitude and longitude values for velocity to complete the spherical velocities sin_lat_vel = np.random.uniform(-1, 1, len(data_in_field)) lat_vel = np.arcsin(sin_lat_vel) long_vel = np.random.uniform(0, 2*np.pi, len(data_in_field)) @@ -3807,51 +3828,38 @@ def qnfw(p, con=5, logp=False): vz= cart_vel[2] #Getting rest of data for PBHs for the combined .h5 file - mass = np.full((len(data_in_field), 1), pbh_mass) - mass = np.reshape(mass, (len(data_in_field),)) + mass = np.full(len(data_in_field), pbh_mass) zams_mass = mass - age = np.full((len(data_in_field), 1), 14*(10**9)) - age = np.reshape(age, (len(data_in_field),)) - pop_id = np.full((len(data_in_field), 1), 10) - pop_id = np.reshape(pop_id, (len(data_in_field),)) - rem_id = np.full((len(data_in_field), 1), 104) - rem_id = np.reshape(rem_id, (len(data_in_field),)) + age = np.full(len(data_in_field), 14*(10**9)) + pop_id = np.full(len(data_in_field), 10) + rem_id = np.full(len(data_in_field), 104) - b_rad = np.radians(b) - l_rad = np.radians(l) + b_rad = np.radians(b_in_field) + l_rad = np.radians(l_in_field) - cart_helio = astropy.coordinates.spherical_to_cartesian(r, b_rad, l_rad) + cart_helio = astropy.coordinates.spherical_to_cartesian(r_in_field, b_rad, l_rad) px = cart_helio[0] py = cart_helio[1] pz = cart_helio[2] - vr, mu_b, mu_lcosb = calc_sph_motion(vx, vy, vz, r, b, l) + vr, mu_b, mu_lcosb = calc_sph_motion(vx, vy, vz, r_in_field, b_in_field, l_in_field) - exbv = np.full((len(data_in_field), 1), 0) - exbv = np.reshape(exbv, (len(data_in_field),)) - ubv_k = np.full((len(data_in_field), 1), np.nan) - ubv_k = np.reshape(ubv_k, (len(data_in_field),)) - ubv_j = np.full((len(data_in_field), 1), np.nan) - ubv_j = np.reshape(ubv_j, (len(data_in_field),)) + exbv = np.full(len(data_in_field), 0) + ubv_k = np.full(len(data_in_field), np.nan) + ubv_j = np.full(len(data_in_field), np.nan) obj_id = np.arange(1, (len(data_in_field)+1)) - ubv_i = np.full((len(data_in_field), 1), np.nan) - ubv_i = np.reshape(ubv_i, (len(data_in_field),)) - ubv_u = np.full((len(data_in_field), 1), np.nan) - ubv_u = np.reshape(ubv_u, (len(data_in_field),)) - ubv_r = np.full((len(data_in_field), 1), np.nan) - ubv_r = np.reshape(ubv_r, (len(data_in_field),)) - ubv_b = np.full((len(data_in_field), 1), np.nan) - ubv_b = np.reshape(ubv_b, (len(data_in_field),)) - ubv_h = np.full((len(data_in_field), 1), np.nan) - ubv_h = np.reshape(ubv_h, (len(data_in_field),)) - ubv_v = np.full((len(data_in_field), 1), np.nan) - ubv_v = np.reshape(ubv_v, (len(data_in_field),)) - - pbh_data = pd.DataFrame({'zams_mass':zams_mass, 'rem_id':rem_id, 'mass':mass, 'px':px, 'py':py, 'pz':pz, 'vx':vx, 'vy':vy, 'vz':vz, 'rad':r, 'glat':b, 'glon':l, 'vr':vr, 'mu_b':mu_b, 'mu_lcosb':mu_lcosb, 'age':age, 'popid':pop_id, 'ubv_k':ubv_k, 'ubv_i':ubv_i, 'exbv':exbv, 'obj_id':obj_id, 'ubv_j':ubv_j, 'ubv_u':ubv_u, 'ubv_r':ubv_r, 'ubv_b':ubv_b, 'ubv_h':ubv_h, 'ubv_v':ubv_v}) + ubv_i = np.full(len(data_in_field), np.nan) + ubv_u = np.full(len(data_in_field), np.nan) + ubv_r = np.full(len(data_in_field), np.nan) + ubv_b = np.full(len(data_in_field), np.nan) + ubv_h = np.full(len(data_in_field), np.nan) + ubv_v = np.full(len(data_in_field), np.nan) + + pbh_data = pd.DataFrame({'zams_mass':zams_mass, 'rem_id':rem_id, 'mass':mass, 'px':px, 'py':py, 'pz':pz, 'vx':vx, 'vy':vy, 'vz':vz, 'rad':r_in_field, 'glat':b_in_field, 'glon':l_in_field, 'vr':vr, 'mu_b':mu_b, 'mu_lcosb':mu_lcosb, 'age':age, 'popid':pop_id, 'ubv_k':ubv_k, 'ubv_i':ubv_i, 'exbv':exbv, 'obj_id':obj_id, 'ubv_j':ubv_j, 'ubv_u':ubv_u, 'ubv_r':ubv_r, 'ubv_b':ubv_b, 'ubv_h':ubv_h, 'ubv_v':ubv_v}) #Getting information out of .h5 file before PBHs - no_pbh_hdf5_file = h5py.File(hdf5_file) + no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') for key in key_list: keys = pd.DataFrame(np.array(no_pbh_hdf5_file['{}'.format(key)])) From c94c6a5304f18af546f1e2f8cb42584d9fb2bf9c Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 14:19:44 -0800 Subject: [PATCH 011/125] Delete blue_line.csv --- popsycle/data/blue_line.csv | 66 ------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 popsycle/data/blue_line.csv diff --git a/popsycle/data/blue_line.csv b/popsycle/data/blue_line.csv deleted file mode 100644 index 43364660..00000000 --- a/popsycle/data/blue_line.csv +++ /dev/null @@ -1,66 +0,0 @@ -x,y -0.010792642776929309, 204.26643023097807 -0.012590321940393834, 210.92487127516114 -0.014687431970009386, 218.17494681231219 -0.01713384764066691, 225.4158429807519 -0.019987751131241467, 232.89705353032554 -0.023317015749352955, 240.21341604790845 -0.02720082013657757, 247.6177427638289 -0.031731531344141896, 255.39654951292243 -0.03701690156357718, 262.5159561489671 -0.04318263075634799, 269.98842786464206 -0.05037535612850712, 277.3556810971669 -0.05876613954792163, 284.43477324360026 -0.0685545358439959, 291.3605743829068 -0.07997333874472501, 298.11329956878745 -0.0932941173219056, 304.6732953099007 -0.10883367461568658, 311.0211336366553 -0.1269615820414861, 316.5932048851325 -0.14810896876722457, 322.0805612681069 -0.17277877509531006, 326.913144325661 -0.20155771370168304, 331.43832299459757 -0.235130223201544, 335.44920768526305 -0.2742947458941446, 338.73163728292803 -0.3199827168140894, 342.046186032515 -0.3732807156985778, 343.8140575474295 -0.4354563087024429, 346.3837933152392 -0.5079881944447303, 346.3837933152392 -0.5926013713388398, 346.3837933152392 -0.6913081625775506, 346.3837933152392 -0.8064560744546262, 343.61717701711524 -0.9407836262193813, 341.654561829232 -1.0974854792445174, 338.15005999198166 -1.2802884144496132, 334.10688186636213 -1.4935399649225864, 329.73408677356304 -1.7423118116552716, 324.4878564913184 -2.03252040141478, 318.41185653421763 -2.3710676553598873, 312.4496290180272 -2.7660051148222395, 306.07263557365945 -3.2267254280694635, 299.1396191737312 -3.764185732107431, 292.02890586026 -4.391168242126657, 285.9048791942064 -5.122584246093038, 278.7891986399261 -5.975828734271417, 271.383867634092 -6.971194097701775, 265.692780180841 -8.132352734462563, 258.3391991378944 -9.48692004136333, 253.79240406567155 -11.067111180485346, 247.90157653062977 -12.970922127990187, 241.5318118152786 -15.060949011985775, 236.1208479753841 -17.618863818498795, 229.54628085055066 -20.496060936083154, 223.2314835842758 -23.909992300202614, 217.3019029730845 -27.892565970532186, 211.4086969647845 -32.53849799081271, 206.147098218244 -37.958280805597894, 199.9828369269898 -44.28081106028432, 193.7807781680185 -51.65645510129202, 186.37803645873726 -60.26062508202531, 179.36080012164197 -70.29795072379233, 172.80561961785477 -82.0071459470937, 165.2548659942107 -95.6666861145325, 158.0340430777453 -111.60143090506897, 149.06535054405288 -130.19035032893146, 139.90937104204863 -151.8755376280755, 130.51589873409813 -177.1727234125981, 121.49765534647908 -195.4228471568372, 116.70016866901982 From b637a4e15231ecafccabcc5dc1a1be8e9e23d51e Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 14:19:51 -0800 Subject: [PATCH 012/125] Delete green_line.csv --- popsycle/data/green_line.csv | 62 ------------------------------------ 1 file changed, 62 deletions(-) delete mode 100644 popsycle/data/green_line.csv diff --git a/popsycle/data/green_line.csv b/popsycle/data/green_line.csv deleted file mode 100644 index b79e6068..00000000 --- a/popsycle/data/green_line.csv +++ /dev/null @@ -1,62 +0,0 @@ -x,y -0.010792642776929309, 111.71180517031452 -0.012590321940393834, 118.88165941281024 -0.014687431970009386, 126.54067684534101 -0.01713384764066691, 134.0005466632754 -0.01993184060733583, 142.11168012051704 -0.023448011680987736, 150.78286434254127 -0.027455988964315874, 159.3571491494456 -0.03213404568533922, 168.4661816124074 -0.03764430047370784, 177.73468233692108 -0.043859894720648336, 186.54417763189218 -0.05108586704256386, 196.01349797292718 -0.05934511962521002, 204.67635482054385 -0.07083192093472244, 215.1824749096273 -0.08253367166054373, 223.90379647052103 -0.09684442649610563, 232.571184061413 -0.11350401149180162, 242.54850927105662 -0.13427738741960332, 251.89354878059038 -0.1586056689369005, 261.0497880274381 -0.186179188876288, 269.5935014641335 -0.21617856025114549, 276.95879153868964 -0.2526914812832889, 283.38932791498735 -0.2962593087128887, 290.36094394136614 -0.35792137580603484, 297.77197612125576 -0.4309060654759373, 304.89151975943804 -0.5059594624841087, 309.244695083941 -0.5778483816986392, 312.37804867591933 -0.6816933312568133, 314.7048135621079 -0.927699047785235, 320.24095532420716 -1.0671697449159139, 319.8132280123578 -1.4727675574625494, 314.96530138192946 -1.7718422284532012, 311.9846498190986 -2.1197411537756987, 307.83085241102907 -2.4598627413134806, 301.78571980162417 -2.8847016095688685, 295.5628794127563 -3.369909155131431, 289.9952727173605 -4.037236963747592, 283.1343596020251 -4.820932000343341, 279.00220082094125 -5.650248971118326, 275.5083826876096 -6.703101332022676, 271.60158332331133 -7.770476112914143, 265.4265828672343 -9.096562802640236, 259.22858778705637 -10.611733993556404, 252.9216664462353 -12.3792800416127, 246.90957929498413 -14.44123782613885, 240.62655413150787 -16.846646109472406, 235.17599223163336 -19.652711807301024, 229.3228663966612 -22.766181747138315, 223.97434631744974 -26.744873675700347, 218.17494681231219 -31.068822348129252, 212.83026716581725 -36.396415667350134, 206.61996451113927 -43.05764653615527, 199.86831942509124 -50.22955388134484, 193.3372953786207 -58.59605171407531, 186.69858418697999 -68.54786206569099, 179.3094384837924 -79.24703527090092, 171.75301799473309 -93.02409457669958, 163.93495203048826 -108.51867546639197, 155.9656915244779 -126.5941149824405, 147.0301382654459 -147.68029446830633, 138.0518741736095 -172.27869855775518, 128.43932707311797 -192.7048730125345, 121.55334325539347 From 2aa1532cc8196f8e36aa7962fe9def9f69bfccc0 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 14:20:00 -0800 Subject: [PATCH 013/125] Delete red_line.csv --- popsycle/data/red_line.csv | 64 -------------------------------------- 1 file changed, 64 deletions(-) delete mode 100644 popsycle/data/red_line.csv diff --git a/popsycle/data/red_line.csv b/popsycle/data/red_line.csv deleted file mode 100644 index f082fbec..00000000 --- a/popsycle/data/red_line.csv +++ /dev/null @@ -1,64 +0,0 @@ -x,y -0.010792642776929309, 51.189721234746486 -0.012590321940393834, 56.2637730438137 -0.014687431970009386, 61.73460135767326 -0.01713384764066691, 67.58236521833585 -0.019987751131241467, 73.81473571195039 -0.023317015749352955, 80.43733893511352 -0.02720082013657757, 87.40343466110791 -0.031731531344141896, 94.80975218695623 -0.03701690156357718, 102.66708492081379 -0.04318263075634799, 110.8576402505859 -0.05037535612850712, 119.0179284411037 -0.05876613954792163, 127.63259980070472 -0.0685545358439959, 136.4012214489796 -0.07997333874472501, 145.5219851562832 -0.0932941173219056, 155.0748727428563 -0.10883367461568658, 164.21690036223077 -0.1269615820414861, 173.59930211335876 -0.14810896876722457, 183.3076420995835 -0.17277877509531006, 193.00534947186358 -0.20155771370168304, 202.28701613851908 -0.235130223201544, 211.89363246804209 -0.2742947458941446, 221.19495574296548 -0.3199827168140894, 230.37612812090563 -0.3732807156985778, 239.11517900904093 -0.4354563087024429, 247.7596190021874 -0.5079881944447303, 255.5428827333543 -0.5926013713388398, 263.2688804002836 -0.6913081625775506, 269.98842786464206 -0.8064560744546262, 275.92953461668526 -0.9407836262193813, 282.1629525216277 -1.0974854792445174, 284.43477324360026 -1.2802884144496132, 287.7119979375535 -1.4935399649225864, 289.6964297280955 -1.7423118116552716, 290.02849651765854 -2.03252040141478, 292.53116033258596 -2.3710676553598873, 288.7025088087155 -2.7660051148222395, 288.5371871710757 -3.2267254280694635, 285.9048791942064 -3.764185732107431, 282.1629525216277 -4.340214672149436, 278.4168360406884 -6.424289627691652, 275.5083826876096 -7.582335742589992, 269.21628633919846 -8.942171302597005, 263.2521253122571 -10.318607100674031, 258.0434148632248 -12.037328396676813, 252.19833425421467 -14.042328922474109, 246.06245599323356 -16.381292847455335, 240.62655413150787 -19.109846866257, 236.1208479753841 -22.29288314740797, 229.84889395002654 -26.00610263923727, 224.89996081862702 -30.337815436904652, 218.4250315448684 -35.39104102800242, 213.35512334606335 -41.285958366076336, 207.68788330277627 -48.16276404123605, 200.44156345555936 -56.185006522648195, 193.66981219227398 -65.54347576993862, 187.55607596607996 -76.46074071866175, 180.18459004092205 -89.19644255313948, 172.31141178905165 -104.05346965457406, 163.6534877832804 -121.38516108089176, 156.32345022691644 -141.603709895957, 146.35792791879018 -165.18996620135306, 135.46686778654006 -187.38180510452284, 127.19470448022776 From 4879bb085beb8eb86e2db41e4eace7d0cc2241fa Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 14:20:25 -0800 Subject: [PATCH 014/125] Add files via upload --- popsycle/data/inner_slop_half.csv | 62 +++++++++++++++++++++++++ popsycle/data/inner_slope_one.csv | 64 ++++++++++++++++++++++++++ popsycle/data/inner_slope_quarter.csv | 66 +++++++++++++++++++++++++++ 3 files changed, 192 insertions(+) create mode 100644 popsycle/data/inner_slop_half.csv create mode 100644 popsycle/data/inner_slope_one.csv create mode 100644 popsycle/data/inner_slope_quarter.csv diff --git a/popsycle/data/inner_slop_half.csv b/popsycle/data/inner_slop_half.csv new file mode 100644 index 00000000..b79e6068 --- /dev/null +++ b/popsycle/data/inner_slop_half.csv @@ -0,0 +1,62 @@ +x,y +0.010792642776929309, 111.71180517031452 +0.012590321940393834, 118.88165941281024 +0.014687431970009386, 126.54067684534101 +0.01713384764066691, 134.0005466632754 +0.01993184060733583, 142.11168012051704 +0.023448011680987736, 150.78286434254127 +0.027455988964315874, 159.3571491494456 +0.03213404568533922, 168.4661816124074 +0.03764430047370784, 177.73468233692108 +0.043859894720648336, 186.54417763189218 +0.05108586704256386, 196.01349797292718 +0.05934511962521002, 204.67635482054385 +0.07083192093472244, 215.1824749096273 +0.08253367166054373, 223.90379647052103 +0.09684442649610563, 232.571184061413 +0.11350401149180162, 242.54850927105662 +0.13427738741960332, 251.89354878059038 +0.1586056689369005, 261.0497880274381 +0.186179188876288, 269.5935014641335 +0.21617856025114549, 276.95879153868964 +0.2526914812832889, 283.38932791498735 +0.2962593087128887, 290.36094394136614 +0.35792137580603484, 297.77197612125576 +0.4309060654759373, 304.89151975943804 +0.5059594624841087, 309.244695083941 +0.5778483816986392, 312.37804867591933 +0.6816933312568133, 314.7048135621079 +0.927699047785235, 320.24095532420716 +1.0671697449159139, 319.8132280123578 +1.4727675574625494, 314.96530138192946 +1.7718422284532012, 311.9846498190986 +2.1197411537756987, 307.83085241102907 +2.4598627413134806, 301.78571980162417 +2.8847016095688685, 295.5628794127563 +3.369909155131431, 289.9952727173605 +4.037236963747592, 283.1343596020251 +4.820932000343341, 279.00220082094125 +5.650248971118326, 275.5083826876096 +6.703101332022676, 271.60158332331133 +7.770476112914143, 265.4265828672343 +9.096562802640236, 259.22858778705637 +10.611733993556404, 252.9216664462353 +12.3792800416127, 246.90957929498413 +14.44123782613885, 240.62655413150787 +16.846646109472406, 235.17599223163336 +19.652711807301024, 229.3228663966612 +22.766181747138315, 223.97434631744974 +26.744873675700347, 218.17494681231219 +31.068822348129252, 212.83026716581725 +36.396415667350134, 206.61996451113927 +43.05764653615527, 199.86831942509124 +50.22955388134484, 193.3372953786207 +58.59605171407531, 186.69858418697999 +68.54786206569099, 179.3094384837924 +79.24703527090092, 171.75301799473309 +93.02409457669958, 163.93495203048826 +108.51867546639197, 155.9656915244779 +126.5941149824405, 147.0301382654459 +147.68029446830633, 138.0518741736095 +172.27869855775518, 128.43932707311797 +192.7048730125345, 121.55334325539347 diff --git a/popsycle/data/inner_slope_one.csv b/popsycle/data/inner_slope_one.csv new file mode 100644 index 00000000..f082fbec --- /dev/null +++ b/popsycle/data/inner_slope_one.csv @@ -0,0 +1,64 @@ +x,y +0.010792642776929309, 51.189721234746486 +0.012590321940393834, 56.2637730438137 +0.014687431970009386, 61.73460135767326 +0.01713384764066691, 67.58236521833585 +0.019987751131241467, 73.81473571195039 +0.023317015749352955, 80.43733893511352 +0.02720082013657757, 87.40343466110791 +0.031731531344141896, 94.80975218695623 +0.03701690156357718, 102.66708492081379 +0.04318263075634799, 110.8576402505859 +0.05037535612850712, 119.0179284411037 +0.05876613954792163, 127.63259980070472 +0.0685545358439959, 136.4012214489796 +0.07997333874472501, 145.5219851562832 +0.0932941173219056, 155.0748727428563 +0.10883367461568658, 164.21690036223077 +0.1269615820414861, 173.59930211335876 +0.14810896876722457, 183.3076420995835 +0.17277877509531006, 193.00534947186358 +0.20155771370168304, 202.28701613851908 +0.235130223201544, 211.89363246804209 +0.2742947458941446, 221.19495574296548 +0.3199827168140894, 230.37612812090563 +0.3732807156985778, 239.11517900904093 +0.4354563087024429, 247.7596190021874 +0.5079881944447303, 255.5428827333543 +0.5926013713388398, 263.2688804002836 +0.6913081625775506, 269.98842786464206 +0.8064560744546262, 275.92953461668526 +0.9407836262193813, 282.1629525216277 +1.0974854792445174, 284.43477324360026 +1.2802884144496132, 287.7119979375535 +1.4935399649225864, 289.6964297280955 +1.7423118116552716, 290.02849651765854 +2.03252040141478, 292.53116033258596 +2.3710676553598873, 288.7025088087155 +2.7660051148222395, 288.5371871710757 +3.2267254280694635, 285.9048791942064 +3.764185732107431, 282.1629525216277 +4.340214672149436, 278.4168360406884 +6.424289627691652, 275.5083826876096 +7.582335742589992, 269.21628633919846 +8.942171302597005, 263.2521253122571 +10.318607100674031, 258.0434148632248 +12.037328396676813, 252.19833425421467 +14.042328922474109, 246.06245599323356 +16.381292847455335, 240.62655413150787 +19.109846866257, 236.1208479753841 +22.29288314740797, 229.84889395002654 +26.00610263923727, 224.89996081862702 +30.337815436904652, 218.4250315448684 +35.39104102800242, 213.35512334606335 +41.285958366076336, 207.68788330277627 +48.16276404123605, 200.44156345555936 +56.185006522648195, 193.66981219227398 +65.54347576993862, 187.55607596607996 +76.46074071866175, 180.18459004092205 +89.19644255313948, 172.31141178905165 +104.05346965457406, 163.6534877832804 +121.38516108089176, 156.32345022691644 +141.603709895957, 146.35792791879018 +165.18996620135306, 135.46686778654006 +187.38180510452284, 127.19470448022776 diff --git a/popsycle/data/inner_slope_quarter.csv b/popsycle/data/inner_slope_quarter.csv new file mode 100644 index 00000000..43364660 --- /dev/null +++ b/popsycle/data/inner_slope_quarter.csv @@ -0,0 +1,66 @@ +x,y +0.010792642776929309, 204.26643023097807 +0.012590321940393834, 210.92487127516114 +0.014687431970009386, 218.17494681231219 +0.01713384764066691, 225.4158429807519 +0.019987751131241467, 232.89705353032554 +0.023317015749352955, 240.21341604790845 +0.02720082013657757, 247.6177427638289 +0.031731531344141896, 255.39654951292243 +0.03701690156357718, 262.5159561489671 +0.04318263075634799, 269.98842786464206 +0.05037535612850712, 277.3556810971669 +0.05876613954792163, 284.43477324360026 +0.0685545358439959, 291.3605743829068 +0.07997333874472501, 298.11329956878745 +0.0932941173219056, 304.6732953099007 +0.10883367461568658, 311.0211336366553 +0.1269615820414861, 316.5932048851325 +0.14810896876722457, 322.0805612681069 +0.17277877509531006, 326.913144325661 +0.20155771370168304, 331.43832299459757 +0.235130223201544, 335.44920768526305 +0.2742947458941446, 338.73163728292803 +0.3199827168140894, 342.046186032515 +0.3732807156985778, 343.8140575474295 +0.4354563087024429, 346.3837933152392 +0.5079881944447303, 346.3837933152392 +0.5926013713388398, 346.3837933152392 +0.6913081625775506, 346.3837933152392 +0.8064560744546262, 343.61717701711524 +0.9407836262193813, 341.654561829232 +1.0974854792445174, 338.15005999198166 +1.2802884144496132, 334.10688186636213 +1.4935399649225864, 329.73408677356304 +1.7423118116552716, 324.4878564913184 +2.03252040141478, 318.41185653421763 +2.3710676553598873, 312.4496290180272 +2.7660051148222395, 306.07263557365945 +3.2267254280694635, 299.1396191737312 +3.764185732107431, 292.02890586026 +4.391168242126657, 285.9048791942064 +5.122584246093038, 278.7891986399261 +5.975828734271417, 271.383867634092 +6.971194097701775, 265.692780180841 +8.132352734462563, 258.3391991378944 +9.48692004136333, 253.79240406567155 +11.067111180485346, 247.90157653062977 +12.970922127990187, 241.5318118152786 +15.060949011985775, 236.1208479753841 +17.618863818498795, 229.54628085055066 +20.496060936083154, 223.2314835842758 +23.909992300202614, 217.3019029730845 +27.892565970532186, 211.4086969647845 +32.53849799081271, 206.147098218244 +37.958280805597894, 199.9828369269898 +44.28081106028432, 193.7807781680185 +51.65645510129202, 186.37803645873726 +60.26062508202531, 179.36080012164197 +70.29795072379233, 172.80561961785477 +82.0071459470937, 165.2548659942107 +95.6666861145325, 158.0340430777453 +111.60143090506897, 149.06535054405288 +130.19035032893146, 139.90937104204863 +151.8755376280755, 130.51589873409813 +177.1727234125981, 121.49765534647908 +195.4228471568372, 116.70016866901982 From eac81b347de6160c6e15a9aee2f77a2651662347 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 14:21:15 -0800 Subject: [PATCH 015/125] Rename inner_slop_half.csv to inner_slope_half.csv --- popsycle/data/{inner_slop_half.csv => inner_slope_half.csv} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename popsycle/data/{inner_slop_half.csv => inner_slope_half.csv} (100%) diff --git a/popsycle/data/inner_slop_half.csv b/popsycle/data/inner_slope_half.csv similarity index 100% rename from popsycle/data/inner_slop_half.csv rename to popsycle/data/inner_slope_half.csv From 352ac83c7000d8f817d1aab6b072c2ea5db5b12a Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 16:24:51 -0800 Subject: [PATCH 016/125] Delete inner_slope_half.csv --- popsycle/data/inner_slope_half.csv | 62 ------------------------------ 1 file changed, 62 deletions(-) delete mode 100644 popsycle/data/inner_slope_half.csv diff --git a/popsycle/data/inner_slope_half.csv b/popsycle/data/inner_slope_half.csv deleted file mode 100644 index b79e6068..00000000 --- a/popsycle/data/inner_slope_half.csv +++ /dev/null @@ -1,62 +0,0 @@ -x,y -0.010792642776929309, 111.71180517031452 -0.012590321940393834, 118.88165941281024 -0.014687431970009386, 126.54067684534101 -0.01713384764066691, 134.0005466632754 -0.01993184060733583, 142.11168012051704 -0.023448011680987736, 150.78286434254127 -0.027455988964315874, 159.3571491494456 -0.03213404568533922, 168.4661816124074 -0.03764430047370784, 177.73468233692108 -0.043859894720648336, 186.54417763189218 -0.05108586704256386, 196.01349797292718 -0.05934511962521002, 204.67635482054385 -0.07083192093472244, 215.1824749096273 -0.08253367166054373, 223.90379647052103 -0.09684442649610563, 232.571184061413 -0.11350401149180162, 242.54850927105662 -0.13427738741960332, 251.89354878059038 -0.1586056689369005, 261.0497880274381 -0.186179188876288, 269.5935014641335 -0.21617856025114549, 276.95879153868964 -0.2526914812832889, 283.38932791498735 -0.2962593087128887, 290.36094394136614 -0.35792137580603484, 297.77197612125576 -0.4309060654759373, 304.89151975943804 -0.5059594624841087, 309.244695083941 -0.5778483816986392, 312.37804867591933 -0.6816933312568133, 314.7048135621079 -0.927699047785235, 320.24095532420716 -1.0671697449159139, 319.8132280123578 -1.4727675574625494, 314.96530138192946 -1.7718422284532012, 311.9846498190986 -2.1197411537756987, 307.83085241102907 -2.4598627413134806, 301.78571980162417 -2.8847016095688685, 295.5628794127563 -3.369909155131431, 289.9952727173605 -4.037236963747592, 283.1343596020251 -4.820932000343341, 279.00220082094125 -5.650248971118326, 275.5083826876096 -6.703101332022676, 271.60158332331133 -7.770476112914143, 265.4265828672343 -9.096562802640236, 259.22858778705637 -10.611733993556404, 252.9216664462353 -12.3792800416127, 246.90957929498413 -14.44123782613885, 240.62655413150787 -16.846646109472406, 235.17599223163336 -19.652711807301024, 229.3228663966612 -22.766181747138315, 223.97434631744974 -26.744873675700347, 218.17494681231219 -31.068822348129252, 212.83026716581725 -36.396415667350134, 206.61996451113927 -43.05764653615527, 199.86831942509124 -50.22955388134484, 193.3372953786207 -58.59605171407531, 186.69858418697999 -68.54786206569099, 179.3094384837924 -79.24703527090092, 171.75301799473309 -93.02409457669958, 163.93495203048826 -108.51867546639197, 155.9656915244779 -126.5941149824405, 147.0301382654459 -147.68029446830633, 138.0518741736095 -172.27869855775518, 128.43932707311797 -192.7048730125345, 121.55334325539347 From 0f82f8a44f725d0b9a128c7d5ba4542599ded072 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 16:25:02 -0800 Subject: [PATCH 017/125] Delete inner_slope_one.csv --- popsycle/data/inner_slope_one.csv | 64 ------------------------------- 1 file changed, 64 deletions(-) delete mode 100644 popsycle/data/inner_slope_one.csv diff --git a/popsycle/data/inner_slope_one.csv b/popsycle/data/inner_slope_one.csv deleted file mode 100644 index f082fbec..00000000 --- a/popsycle/data/inner_slope_one.csv +++ /dev/null @@ -1,64 +0,0 @@ -x,y -0.010792642776929309, 51.189721234746486 -0.012590321940393834, 56.2637730438137 -0.014687431970009386, 61.73460135767326 -0.01713384764066691, 67.58236521833585 -0.019987751131241467, 73.81473571195039 -0.023317015749352955, 80.43733893511352 -0.02720082013657757, 87.40343466110791 -0.031731531344141896, 94.80975218695623 -0.03701690156357718, 102.66708492081379 -0.04318263075634799, 110.8576402505859 -0.05037535612850712, 119.0179284411037 -0.05876613954792163, 127.63259980070472 -0.0685545358439959, 136.4012214489796 -0.07997333874472501, 145.5219851562832 -0.0932941173219056, 155.0748727428563 -0.10883367461568658, 164.21690036223077 -0.1269615820414861, 173.59930211335876 -0.14810896876722457, 183.3076420995835 -0.17277877509531006, 193.00534947186358 -0.20155771370168304, 202.28701613851908 -0.235130223201544, 211.89363246804209 -0.2742947458941446, 221.19495574296548 -0.3199827168140894, 230.37612812090563 -0.3732807156985778, 239.11517900904093 -0.4354563087024429, 247.7596190021874 -0.5079881944447303, 255.5428827333543 -0.5926013713388398, 263.2688804002836 -0.6913081625775506, 269.98842786464206 -0.8064560744546262, 275.92953461668526 -0.9407836262193813, 282.1629525216277 -1.0974854792445174, 284.43477324360026 -1.2802884144496132, 287.7119979375535 -1.4935399649225864, 289.6964297280955 -1.7423118116552716, 290.02849651765854 -2.03252040141478, 292.53116033258596 -2.3710676553598873, 288.7025088087155 -2.7660051148222395, 288.5371871710757 -3.2267254280694635, 285.9048791942064 -3.764185732107431, 282.1629525216277 -4.340214672149436, 278.4168360406884 -6.424289627691652, 275.5083826876096 -7.582335742589992, 269.21628633919846 -8.942171302597005, 263.2521253122571 -10.318607100674031, 258.0434148632248 -12.037328396676813, 252.19833425421467 -14.042328922474109, 246.06245599323356 -16.381292847455335, 240.62655413150787 -19.109846866257, 236.1208479753841 -22.29288314740797, 229.84889395002654 -26.00610263923727, 224.89996081862702 -30.337815436904652, 218.4250315448684 -35.39104102800242, 213.35512334606335 -41.285958366076336, 207.68788330277627 -48.16276404123605, 200.44156345555936 -56.185006522648195, 193.66981219227398 -65.54347576993862, 187.55607596607996 -76.46074071866175, 180.18459004092205 -89.19644255313948, 172.31141178905165 -104.05346965457406, 163.6534877832804 -121.38516108089176, 156.32345022691644 -141.603709895957, 146.35792791879018 -165.18996620135306, 135.46686778654006 -187.38180510452284, 127.19470448022776 From b3bf9f06d9a5979ba8ff1802ac028fad80dd10a1 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 16:25:09 -0800 Subject: [PATCH 018/125] Delete inner_slope_quarter.csv --- popsycle/data/inner_slope_quarter.csv | 66 --------------------------- 1 file changed, 66 deletions(-) delete mode 100644 popsycle/data/inner_slope_quarter.csv diff --git a/popsycle/data/inner_slope_quarter.csv b/popsycle/data/inner_slope_quarter.csv deleted file mode 100644 index 43364660..00000000 --- a/popsycle/data/inner_slope_quarter.csv +++ /dev/null @@ -1,66 +0,0 @@ -x,y -0.010792642776929309, 204.26643023097807 -0.012590321940393834, 210.92487127516114 -0.014687431970009386, 218.17494681231219 -0.01713384764066691, 225.4158429807519 -0.019987751131241467, 232.89705353032554 -0.023317015749352955, 240.21341604790845 -0.02720082013657757, 247.6177427638289 -0.031731531344141896, 255.39654951292243 -0.03701690156357718, 262.5159561489671 -0.04318263075634799, 269.98842786464206 -0.05037535612850712, 277.3556810971669 -0.05876613954792163, 284.43477324360026 -0.0685545358439959, 291.3605743829068 -0.07997333874472501, 298.11329956878745 -0.0932941173219056, 304.6732953099007 -0.10883367461568658, 311.0211336366553 -0.1269615820414861, 316.5932048851325 -0.14810896876722457, 322.0805612681069 -0.17277877509531006, 326.913144325661 -0.20155771370168304, 331.43832299459757 -0.235130223201544, 335.44920768526305 -0.2742947458941446, 338.73163728292803 -0.3199827168140894, 342.046186032515 -0.3732807156985778, 343.8140575474295 -0.4354563087024429, 346.3837933152392 -0.5079881944447303, 346.3837933152392 -0.5926013713388398, 346.3837933152392 -0.6913081625775506, 346.3837933152392 -0.8064560744546262, 343.61717701711524 -0.9407836262193813, 341.654561829232 -1.0974854792445174, 338.15005999198166 -1.2802884144496132, 334.10688186636213 -1.4935399649225864, 329.73408677356304 -1.7423118116552716, 324.4878564913184 -2.03252040141478, 318.41185653421763 -2.3710676553598873, 312.4496290180272 -2.7660051148222395, 306.07263557365945 -3.2267254280694635, 299.1396191737312 -3.764185732107431, 292.02890586026 -4.391168242126657, 285.9048791942064 -5.122584246093038, 278.7891986399261 -5.975828734271417, 271.383867634092 -6.971194097701775, 265.692780180841 -8.132352734462563, 258.3391991378944 -9.48692004136333, 253.79240406567155 -11.067111180485346, 247.90157653062977 -12.970922127990187, 241.5318118152786 -15.060949011985775, 236.1208479753841 -17.618863818498795, 229.54628085055066 -20.496060936083154, 223.2314835842758 -23.909992300202614, 217.3019029730845 -27.892565970532186, 211.4086969647845 -32.53849799081271, 206.147098218244 -37.958280805597894, 199.9828369269898 -44.28081106028432, 193.7807781680185 -51.65645510129202, 186.37803645873726 -60.26062508202531, 179.36080012164197 -70.29795072379233, 172.80561961785477 -82.0071459470937, 165.2548659942107 -95.6666861145325, 158.0340430777453 -111.60143090506897, 149.06535054405288 -130.19035032893146, 139.90937104204863 -151.8755376280755, 130.51589873409813 -177.1727234125981, 121.49765534647908 -195.4228471568372, 116.70016866901982 From 9a3192829be08f4441ab7525b85aaa9feda27915 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 16:25:32 -0800 Subject: [PATCH 019/125] Add files via upload updated .csv files --- .../data/radial_velocity_profile_middle.csv | 62 +++++++++++++++++ .../data/radial_velocity_profile_shallow.csv | 66 +++++++++++++++++++ .../data/radial_velocity_profile_steep.csv | 64 ++++++++++++++++++ 3 files changed, 192 insertions(+) create mode 100644 popsycle/data/radial_velocity_profile_middle.csv create mode 100644 popsycle/data/radial_velocity_profile_shallow.csv create mode 100644 popsycle/data/radial_velocity_profile_steep.csv diff --git a/popsycle/data/radial_velocity_profile_middle.csv b/popsycle/data/radial_velocity_profile_middle.csv new file mode 100644 index 00000000..2319354c --- /dev/null +++ b/popsycle/data/radial_velocity_profile_middle.csv @@ -0,0 +1,62 @@ +r,v +0.010792642776929309, 111.71180517031452 +0.012590321940393834, 118.88165941281024 +0.014687431970009386, 126.54067684534101 +0.01713384764066691, 134.0005466632754 +0.01993184060733583, 142.11168012051704 +0.023448011680987736, 150.78286434254127 +0.027455988964315874, 159.3571491494456 +0.03213404568533922, 168.4661816124074 +0.03764430047370784, 177.73468233692108 +0.043859894720648336, 186.54417763189218 +0.05108586704256386, 196.01349797292718 +0.05934511962521002, 204.67635482054385 +0.07083192093472244, 215.1824749096273 +0.08253367166054373, 223.90379647052103 +0.09684442649610563, 232.571184061413 +0.11350401149180162, 242.54850927105662 +0.13427738741960332, 251.89354878059038 +0.1586056689369005, 261.0497880274381 +0.186179188876288, 269.5935014641335 +0.21617856025114549, 276.95879153868964 +0.2526914812832889, 283.38932791498735 +0.2962593087128887, 290.36094394136614 +0.35792137580603484, 297.77197612125576 +0.4309060654759373, 304.89151975943804 +0.5059594624841087, 309.244695083941 +0.5778483816986392, 312.37804867591933 +0.6816933312568133, 314.7048135621079 +0.927699047785235, 320.24095532420716 +1.0671697449159139, 319.8132280123578 +1.4727675574625494, 314.96530138192946 +1.7718422284532012, 311.9846498190986 +2.1197411537756987, 307.83085241102907 +2.4598627413134806, 301.78571980162417 +2.8847016095688685, 295.5628794127563 +3.369909155131431, 289.9952727173605 +4.037236963747592, 283.1343596020251 +4.820932000343341, 279.00220082094125 +5.650248971118326, 275.5083826876096 +6.703101332022676, 271.60158332331133 +7.770476112914143, 265.4265828672343 +9.096562802640236, 259.22858778705637 +10.611733993556404, 252.9216664462353 +12.3792800416127, 246.90957929498413 +14.44123782613885, 240.62655413150787 +16.846646109472406, 235.17599223163336 +19.652711807301024, 229.3228663966612 +22.766181747138315, 223.97434631744974 +26.744873675700347, 218.17494681231219 +31.068822348129252, 212.83026716581725 +36.396415667350134, 206.61996451113927 +43.05764653615527, 199.86831942509124 +50.22955388134484, 193.3372953786207 +58.59605171407531, 186.69858418697999 +68.54786206569099, 179.3094384837924 +79.24703527090092, 171.75301799473309 +93.02409457669958, 163.93495203048826 +108.51867546639197, 155.9656915244779 +126.5941149824405, 147.0301382654459 +147.68029446830633, 138.0518741736095 +172.27869855775518, 128.43932707311797 +192.7048730125345, 121.55334325539347 diff --git a/popsycle/data/radial_velocity_profile_shallow.csv b/popsycle/data/radial_velocity_profile_shallow.csv new file mode 100644 index 00000000..8e412d1d --- /dev/null +++ b/popsycle/data/radial_velocity_profile_shallow.csv @@ -0,0 +1,66 @@ +r,v +0.010792642776929309, 204.26643023097807 +0.012590321940393834, 210.92487127516114 +0.014687431970009386, 218.17494681231219 +0.01713384764066691, 225.4158429807519 +0.019987751131241467, 232.89705353032554 +0.023317015749352955, 240.21341604790845 +0.02720082013657757, 247.6177427638289 +0.031731531344141896, 255.39654951292243 +0.03701690156357718, 262.5159561489671 +0.04318263075634799, 269.98842786464206 +0.05037535612850712, 277.3556810971669 +0.05876613954792163, 284.43477324360026 +0.0685545358439959, 291.3605743829068 +0.07997333874472501, 298.11329956878745 +0.0932941173219056, 304.6732953099007 +0.10883367461568658, 311.0211336366553 +0.1269615820414861, 316.5932048851325 +0.14810896876722457, 322.0805612681069 +0.17277877509531006, 326.913144325661 +0.20155771370168304, 331.43832299459757 +0.235130223201544, 335.44920768526305 +0.2742947458941446, 338.73163728292803 +0.3199827168140894, 342.046186032515 +0.3732807156985778, 343.8140575474295 +0.4354563087024429, 346.3837933152392 +0.5079881944447303, 346.3837933152392 +0.5926013713388398, 346.3837933152392 +0.6913081625775506, 346.3837933152392 +0.8064560744546262, 343.61717701711524 +0.9407836262193813, 341.654561829232 +1.0974854792445174, 338.15005999198166 +1.2802884144496132, 334.10688186636213 +1.4935399649225864, 329.73408677356304 +1.7423118116552716, 324.4878564913184 +2.03252040141478, 318.41185653421763 +2.3710676553598873, 312.4496290180272 +2.7660051148222395, 306.07263557365945 +3.2267254280694635, 299.1396191737312 +3.764185732107431, 292.02890586026 +4.391168242126657, 285.9048791942064 +5.122584246093038, 278.7891986399261 +5.975828734271417, 271.383867634092 +6.971194097701775, 265.692780180841 +8.132352734462563, 258.3391991378944 +9.48692004136333, 253.79240406567155 +11.067111180485346, 247.90157653062977 +12.970922127990187, 241.5318118152786 +15.060949011985775, 236.1208479753841 +17.618863818498795, 229.54628085055066 +20.496060936083154, 223.2314835842758 +23.909992300202614, 217.3019029730845 +27.892565970532186, 211.4086969647845 +32.53849799081271, 206.147098218244 +37.958280805597894, 199.9828369269898 +44.28081106028432, 193.7807781680185 +51.65645510129202, 186.37803645873726 +60.26062508202531, 179.36080012164197 +70.29795072379233, 172.80561961785477 +82.0071459470937, 165.2548659942107 +95.6666861145325, 158.0340430777453 +111.60143090506897, 149.06535054405288 +130.19035032893146, 139.90937104204863 +151.8755376280755, 130.51589873409813 +177.1727234125981, 121.49765534647908 +195.4228471568372, 116.70016866901982 diff --git a/popsycle/data/radial_velocity_profile_steep.csv b/popsycle/data/radial_velocity_profile_steep.csv new file mode 100644 index 00000000..dd5109fd --- /dev/null +++ b/popsycle/data/radial_velocity_profile_steep.csv @@ -0,0 +1,64 @@ +r,v +0.010792642776929309, 51.189721234746486 +0.012590321940393834, 56.2637730438137 +0.014687431970009386, 61.73460135767326 +0.01713384764066691, 67.58236521833585 +0.019987751131241467, 73.81473571195039 +0.023317015749352955, 80.43733893511352 +0.02720082013657757, 87.40343466110791 +0.031731531344141896, 94.80975218695623 +0.03701690156357718, 102.66708492081379 +0.04318263075634799, 110.8576402505859 +0.05037535612850712, 119.0179284411037 +0.05876613954792163, 127.63259980070472 +0.0685545358439959, 136.4012214489796 +0.07997333874472501, 145.5219851562832 +0.0932941173219056, 155.0748727428563 +0.10883367461568658, 164.21690036223077 +0.1269615820414861, 173.59930211335876 +0.14810896876722457, 183.3076420995835 +0.17277877509531006, 193.00534947186358 +0.20155771370168304, 202.28701613851908 +0.235130223201544, 211.89363246804209 +0.2742947458941446, 221.19495574296548 +0.3199827168140894, 230.37612812090563 +0.3732807156985778, 239.11517900904093 +0.4354563087024429, 247.7596190021874 +0.5079881944447303, 255.5428827333543 +0.5926013713388398, 263.2688804002836 +0.6913081625775506, 269.98842786464206 +0.8064560744546262, 275.92953461668526 +0.9407836262193813, 282.1629525216277 +1.0974854792445174, 284.43477324360026 +1.2802884144496132, 287.7119979375535 +1.4935399649225864, 289.6964297280955 +1.7423118116552716, 290.02849651765854 +2.03252040141478, 292.53116033258596 +2.3710676553598873, 288.7025088087155 +2.7660051148222395, 288.5371871710757 +3.2267254280694635, 285.9048791942064 +3.764185732107431, 282.1629525216277 +4.340214672149436, 278.4168360406884 +6.424289627691652, 275.5083826876096 +7.582335742589992, 269.21628633919846 +8.942171302597005, 263.2521253122571 +10.318607100674031, 258.0434148632248 +12.037328396676813, 252.19833425421467 +14.042328922474109, 246.06245599323356 +16.381292847455335, 240.62655413150787 +19.109846866257, 236.1208479753841 +22.29288314740797, 229.84889395002654 +26.00610263923727, 224.89996081862702 +30.337815436904652, 218.4250315448684 +35.39104102800242, 213.35512334606335 +41.285958366076336, 207.68788330277627 +48.16276404123605, 200.44156345555936 +56.185006522648195, 193.66981219227398 +65.54347576993862, 187.55607596607996 +76.46074071866175, 180.18459004092205 +89.19644255313948, 172.31141178905165 +104.05346965457406, 163.6534877832804 +121.38516108089176, 156.32345022691644 +141.603709895957, 146.35792791879018 +165.18996620135306, 135.46686778654006 +187.38180510452284, 127.19470448022776 From 445304cea6ca57a8567f9eeee678b4cfe39f02b7 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 16:26:28 -0800 Subject: [PATCH 020/125] Add files via upload added inner slope and v_esc --- popsycle/data/pbh_config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/popsycle/data/pbh_config.yaml b/popsycle/data/pbh_config.yaml index 6f813f9c..9587c457 100644 --- a/popsycle/data/pbh_config.yaml +++ b/popsycle/data/pbh_config.yaml @@ -5,3 +5,5 @@ pbh_mass: 40 #Msun r_max: 8.3 #kpc, How far from the galactic center that you want to go out c: 10 #concentration index r_vir: 200 #kpc +inner_slope: .5 +v_esc: 550 From 2393aadb8ea24fc5cbda3cfbe27035edd2595d99 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 21 Feb 2020 16:28:36 -0800 Subject: [PATCH 021/125] gave default values to arguments in add_pbh, minor fixes --- popsycle/run.py | 4 ++-- popsycle/synthetic.py | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index aee8e0e9..3642f568 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -207,8 +207,8 @@ def run(): r_max=pbh_config['r_max'], c=pbh_config['c'], r_vir=pbh_config['r_vir'], - inner_slope=args.inner_slope, - v_esc=args.v_esc, + inner_slope=pbh_config['inner_slope'], + v_esc=pbh_config['v_esc'], overwrite=args.overwrite, seed=args.seed) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 4fd4f29c..8a54ee41 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3546,7 +3546,7 @@ def angdist(ra1, dec1, ra2, dec2): return distance -def add_pbh(hdf5_file, ebf_file, output_root2, fdm, pbh_mass, r_max, c, r_vir, inner_slope = .5, v_esc = 550, overwrite = False, seed = None): +def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c=10, r_vir=200, inner_slope = .5, v_esc = 550, overwrite = False, seed = None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, and saves them in a new HDF5 file with the PBHs added. @@ -3798,20 +3798,19 @@ def qnfw(p, con=5, logp=False): pbh_r_galacto = cart[0] if inner_slope == 1: - vel_data = pd.read_csv('data/inner_slope_one.csv') + vel_data = pd.read_csv('data/radial_velocity_profile_steep.csv') elif inner_slope == .25: - vel_data = pd.read_csv('data/inner_slope_quarter.csv') + vel_data = pd.read_csv('data/radial_velocity_profile_shallow.csv') else: - vel_data = pd.read_csv('data/inner_slope_half.csv') + vel_data = pd.read_csv('data/radial_velocity_profile_middle.csv') - pbh_vrms = np.interp(pbh_r_galacto, vel_data['x'], vel_data['y']) + pbh_vrms = np.interp(pbh_r_galacto, vel_data['r'], vel_data['v']) v_vals = np.arange(0, v_esc) #Goes from v to v_esc a = (1/2)*pbh_vrms*((np.pi/2)**(1/2)) rand_cdf = [] for a_val in a: - pdf = ((2/np.pi)**(1/2))*((v_vals**2*np.exp(-v_vals**2/(2*a_val**2)))/a_val**3) cdf = scipy.special.erf(v_vals/(a_val*2**(1/2)))-(((2/np.pi)**(1/2))*((v_vals*np.exp(-v_vals**2/2*a_val**2))/a_val)) rand_cdf.append(np.random.uniform(0, np.amax(cdf), 1)) interpreted_rms_velocities = np.interp(rand_cdf, cdf, v_vals) From d2b69051c4fbae88e6d6181f1530f1442fdf540e Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 28 Feb 2020 13:46:28 -0800 Subject: [PATCH 022/125] minor fixes From 919d0d3513a41328fdbba1108379ceaaa97c9feb Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 28 Feb 2020 13:47:33 -0800 Subject: [PATCH 023/125] added inner_slope and v_esc --- popsycle/data/pbh_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/data/pbh_config.yaml b/popsycle/data/pbh_config.yaml index 9587c457..14c5eac4 100644 --- a/popsycle/data/pbh_config.yaml +++ b/popsycle/data/pbh_config.yaml @@ -3,7 +3,7 @@ fdm: 1 #Fraction of dark matter that consists of PBHs pbh_mass: 40 #Msun r_max: 8.3 #kpc, How far from the galactic center that you want to go out -c: 10 #concentration index +c: 12.94 #concentration index r_vir: 200 #kpc inner_slope: .5 v_esc: 550 From 6e15bc43b2a56f38d356a2883730398076249fa9 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 28 Feb 2020 13:48:09 -0800 Subject: [PATCH 024/125] minor fixes From ebc955fc7b98a597a320403bf67778970529a444 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 28 Feb 2020 13:49:55 -0800 Subject: [PATCH 025/125] fixed hdf5 writing, various changes as requested on pull request --- popsycle/synthetic.py | 179 ++++++++++++++++++++++++++---------------- 1 file changed, 112 insertions(+), 67 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 8a54ee41..1610f0d8 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1732,7 +1732,8 @@ def _calc_blends(bigpatch, c, event_lbt, blend_rad): # Query ball against the existing (cached) tree. # NOTE: results is an array of lists. - results = kdtree_cache.query_ball_point(flatxyz1.T.copy(order='C'), r_kdt) + #results = kdtree_cache.query_ball_point(flatxyz1.T.copy(order='C'), r_kdt) + results = kdtree_cache.query_ball_point(flatxyz1.T, r_kdt) # Figure out the number of blends for each lens. blend_lens_obj_id = [] @@ -3546,7 +3547,7 @@ def angdist(ra1, dec1, ra2, dec2): return distance -def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c=10, r_vir=200, inner_slope = .5, v_esc = 550, overwrite = False, seed = None): +def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c=12.94, r_vir=200, inner_slope = .5, v_esc = 550, overwrite = False, seed = None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, and saves them in a new HDF5 file with the PBHs added. @@ -3582,7 +3583,7 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c= c : float Concentration index. - Defaults to 10 (concentration index of the Milky Way, multiple papers show it to be anywhere from 10-12) + Defaults to 12.94 (the value given in McMillan 2016, used in the paper we derive the velocities from) r_vir : float The virial radius. @@ -3677,19 +3678,34 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c= #Read in the hdf5 file that doesn't have PBHs. Product of perform_pop_syn. no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') key_list = list(no_pbh_hdf5_file) + #Delete lat_bin_edges and long_bin_edges from key_list. key_list = [key for key in key_list if 'bin_edges' not in key] + + #Get data from lat_bin_edges and long_bin_edges lat_bin = pd.DataFrame(np.array(no_pbh_hdf5_file['lat_bin_edges'])) long_bin = pd.DataFrame(np.array(no_pbh_hdf5_file['long_bin_edges'])) + bin_edges_number = len(long_bin) + + #Getting the maximum ID from all of the stars and compact objects. + #Later used to set the IDs of the PBHs. + max_id_no_pbh = [] + for key in key_list: + max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key][20])) + max_id = np.amax(max_id_no_pbh) + no_pbh_hdf5_file.close() + #Read in ebf file t = ebf.read_ind(ebf_file, '/log', 0) # Convert log to useful dictionary. ebf_log = make_ebf_log(t) + #Obtain survey area and center latitude and longitude b = float(ebf_log['latitude']) l = float(ebf_log['longitude']) surveyArea = float(ebf_log['surveyArea']) + #Calculate the size of the field of view we are running field_of_view_diameter = 2*((surveyArea/np.pi)**(1/2)) #NFW Profile calculations to determine mass of dark matter within given distance of galactic center @@ -3698,9 +3714,8 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c= mass_within_r_max = (4*np.pi*rho_knot*(r_s**3)*(np.log((r_s + r_max)/r_s) - (r_max/(r_s+r_max))))*(((10**3)**3)/((10**6)**3))*fdm #Msun #Determine the number of PBHs within that distance - #num_pbh_within_r_max = (mass_within_r_max/pbh_mass) - #num_pbh_within_r_max = round(num_pbh_within_r_max) - num_pbh_within_r_max = 5e7 + num_pbh_within_r_max = (mass_within_r_max/pbh_mass) + num_pbh_within_r_max = round(num_pbh_within_r_max) """ Defining needed functions from the python package "NFWdist". @@ -3758,45 +3773,52 @@ def qnfw(p, con=5, logp=False): #Calculating radius values for all PBHs within r_max of the galactic center, using NFWDist functions r_values = (qnfw(np.random.rand(int(num_pbh_within_r_max)) * pnfw(r_max/r_vir,con=c, logp=False), con=c)*r_vir) - #Sample PBH latitude and longitude to have full spherical coordinates. + #Sample PBH latitude and longitudes to get full spherical coordinates. sin_lats = np.random.uniform(-1, 1, int(num_pbh_within_r_max)) lats=np.arcsin(sin_lats) - longs = np.random.uniform(0, 2*np.pi, int(num_pbh_within_r_max)) - + longs = np.random.uniform(0, np.pi*2, int(num_pbh_within_r_max)) + #Converting spherical galactocentric coordinates to cartesian galactocentric coordinates cart = astropy.coordinates.spherical_to_cartesian(r_values, lats, longs) - #Defining galactocentric coordinates - c = coord.Galactocentric(x=cart[0] * units.kpc, y=cart[1] * units.kpc, z=cart[2] * units.kpc) + #Defining galactocentric coordinate frame + galacto = coord.Galactocentric(x=cart[0] * units.kpc, y=cart[1] * units.kpc, z=cart[2] * units.kpc) #Transforming from galactocentric to galactic coordinates - #outputs l, b, dist in degrees. - c = c.transform_to(coord.Galactic(representation_type='cartesian')) - print(c.l.deg) - - #Set minimum and maximum l, b, and r values for PBH mask. - #Determining the center of the field of view circle. + #outputs l, b, and distance in degrees. + galactic = galacto.transform_to(coord.Galactic(representation_type='cartesian')) + + latitude=galactic.b.deg + longitude=galactic.l.deg + #Adjusting longitude values to match the coordinate format that we need + longitude=np.where(longitude>180, longitude-360, longitude) + + #Set minimum and maximum l and b for PBH mask. l_min = np.min(long_bin).values l_max = np.max(long_bin).values b_min = np.min(lat_bin).values b_max = np.max(lat_bin).values - dists = angdist(l, b, c.l.deg, c.b.deg) + #Calculating distances between each latitude, longitude pair, and the center of the field of view. + dists = angdist(l, b, longitude, latitude) - #Masking the full PBH data for our field of view - mask = (c.distance.kpc <= 2*r_max) & (dists < field_of_view_diameter) - data_in_field = c[mask] + #Masking the full PBH data to obtain just the PBHs in our field of view. + mask = (galactic.distance.kpc <= 2*r_max) & (dists < field_of_view_diameter) + data_in_field = galactic[mask] - #Obtain the radius values for all of the PBHs in our field of view. + #Obtain the radius, l, and b values for all of the PBHs in our field of view. r_in_field = data_in_field.distance.kpc l_in_field = data_in_field.l.deg b_in_field = data_in_field.b.deg - c_pbh = coord.Galactic(l=l_in_field * units.deg, b=b_in_field * units.deg, distance=r_in_field * units.kpc) - c_pbh = c_pbh.transform_to(coord.Galactocentric(representation_type='spherical')) - cart_pbh = astropy.coordinates.cartesian_to_spherical(c_pbh.x, c_pbh.y, c_pbh.z) - pbh_r_galacto = cart[0] - + #Converting the PBH positions from the field of view back to galactocentric for determining velocities. + galactic_pbh = coord.Galactic(l=l_in_field * units.deg, b=b_in_field * units.deg, distance=r_in_field * units.kpc) + galactic_pbh = galactic_pbh.transform_to(coord.Galactocentric(representation_type='spherical')) + cart_pbh = astropy.coordinates.cartesian_to_spherical(galactic_pbh.x, galactic_pbh.y, galactic_pbh.z) + pbh_r_galacto = cart_pbh[0] + + #Inner slope of the MW halo + #From Lacroix et al 2018, Figure 11 (top left panel) if inner_slope == 1: vel_data = pd.read_csv('data/radial_velocity_profile_steep.csv') elif inner_slope == .25: @@ -3804,50 +3826,51 @@ def qnfw(p, con=5, logp=False): else: vel_data = pd.read_csv('data/radial_velocity_profile_middle.csv') + #Interpolating v values from the above data, given the PBH r values. pbh_vrms = np.interp(pbh_r_galacto, vel_data['r'], vel_data['v']) v_vals = np.arange(0, v_esc) #Goes from v to v_esc a = (1/2)*pbh_vrms*((np.pi/2)**(1/2)) - rand_cdf = [] - + #Calculating the v_rms velocities for the PBHs by randomly sampling from the CDF. + rand_cdf = np.array([]) + for a_val in a: cdf = scipy.special.erf(v_vals/(a_val*2**(1/2)))-(((2/np.pi)**(1/2))*((v_vals*np.exp(-v_vals**2/2*a_val**2))/a_val)) - rand_cdf.append(np.random.uniform(0, np.amax(cdf), 1)) + rand_cdf = np.append(rand_cdf, np.random.uniform(0, np.amax(cdf))) interpreted_rms_velocities = np.interp(rand_cdf, cdf, v_vals) - - #Sampling random latitude and longitude values for velocity to complete the spherical velocities + + #Sampling random latitude and longitude values for velocity to complete the spherical velocities. sin_lat_vel = np.random.uniform(-1, 1, len(data_in_field)) lat_vel = np.arcsin(sin_lat_vel) long_vel = np.random.uniform(0, 2*np.pi, len(data_in_field)) - #Transforming velocities to cartesian to get vx, vy, vz values + #Transforming velocities to cartesian to get vx, vy, and vz. cart_vel = astropy.coordinates.spherical_to_cartesian(interpreted_rms_velocities, lat_vel, long_vel) - vx= cart_vel[0] - vy= cart_vel[1] - vz= cart_vel[2] - #Getting rest of data for PBHs for the combined .h5 file + vx = cart_vel[0] + vy = cart_vel[1] + vz = cart_vel[2] + + #Getting the rest of the PBH data for the combined .h5 file mass = np.full(len(data_in_field), pbh_mass) zams_mass = mass - age = np.full(len(data_in_field), 14*(10**9)) + age = np.full(len(data_in_field), np.nan) pop_id = np.full(len(data_in_field), 10) rem_id = np.full(len(data_in_field), 104) b_rad = np.radians(b_in_field) l_rad = np.radians(l_in_field) - cart_helio = astropy.coordinates.spherical_to_cartesian(r_in_field, b_rad, l_rad) - px = cart_helio[0] py = cart_helio[1] pz = cart_helio[2] vr, mu_b, mu_lcosb = calc_sph_motion(vx, vy, vz, r_in_field, b_in_field, l_in_field) + obj_id = np.arange((max_id+1), (max_id+len(data_in_field)+1)) - exbv = np.full(len(data_in_field), 0) + exbv = np.full(len(data_in_field), np.nan) ubv_k = np.full(len(data_in_field), np.nan) ubv_j = np.full(len(data_in_field), np.nan) - obj_id = np.arange(1, (len(data_in_field)+1)) ubv_i = np.full(len(data_in_field), np.nan) ubv_u = np.full(len(data_in_field), np.nan) ubv_r = np.full(len(data_in_field), np.nan) @@ -3855,36 +3878,46 @@ def qnfw(p, con=5, logp=False): ubv_h = np.full(len(data_in_field), np.nan) ubv_v = np.full(len(data_in_field), np.nan) + #Making a dataframe of all PBH data from PBHs in the field of view. pbh_data = pd.DataFrame({'zams_mass':zams_mass, 'rem_id':rem_id, 'mass':mass, 'px':px, 'py':py, 'pz':pz, 'vx':vx, 'vy':vy, 'vz':vz, 'rad':r_in_field, 'glat':b_in_field, 'glon':l_in_field, 'vr':vr, 'mu_b':mu_b, 'mu_lcosb':mu_lcosb, 'age':age, 'popid':pop_id, 'ubv_k':ubv_k, 'ubv_i':ubv_i, 'exbv':exbv, 'obj_id':obj_id, 'ubv_j':ubv_j, 'ubv_u':ubv_u, 'ubv_r':ubv_r, 'ubv_b':ubv_b, 'ubv_h':ubv_h, 'ubv_v':ubv_v}) - - #Getting information out of .h5 file before PBHs + + #Opening the file with no PBHs and creating a new file for the PBHs added. no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') - - for key in key_list: - keys = pd.DataFrame(np.array(no_pbh_hdf5_file['{}'.format(key)])) - vars()[key+'_max_l'] = np.amax(keys.T[11]) - vars()[key+'_min_l'] = np.amin(keys.T[11]) - vars()[key+'_max_b'] = np.amax(keys.T[10]) - vars()[key+'_min_b'] = np.amin(keys.T[10]) - vars()[key+'_mask'] = (pbh_data['glon'] >= vars()[key+'_min_l']) & (pbh_data['glon'] <= vars()[key+'_max_l']) & (pbh_data['glat'] >= vars()[key+'_min_b']) & (pbh_data['glat'] <= vars()[key+'_max_b']) - vars()['pbh_'+key] = (pbh_data[vars()[key+'_mask']]).T - vars()['pbh_'+key].reset_index(drop=True, inplace=True) - vars()['full_'+key]=pd.concat([keys, vars()['pbh_'+key]], axis=1) + pbh_hdf5_file = h5py.File(output_root2 + '.h5', 'w') + + #Calculate the maximum and minimum l and b values for each dataset in the no PBH file, + #so that we can determine which datasets to correctly add the PBHs. + lat_bin_values = lat_bin.values + long_bin_values = long_bin.values + lat_long_list = [] + for idx in range(len(lat_bin_values)-1): + max_l = lat_bin_values[idx+1] + min_l = lat_bin_values[idx] + for idx2 in range(len(long_bin_values)-1): + max_b = long_bin_values[idx2+1] + min_b = long_bin_values[idx2] + lat_long_list.append((min_l[0], max_l[0], min_b[0], max_b[0])) + + #Appending the PBH data to the no PBH data and writing to the new .h5 file. + for idx, key in enumerate(key_list): + data = pd.DataFrame(no_pbh_hdf5_file[key][:]) + min_l, max_l, min_b, max_b = lat_long_list[idx] + mask = (pbh_data['glon'] >= min_b) & (pbh_data['glon'] <= max_b) & (pbh_data['glat'] >= min_l) & (pbh_data['glat'] <= max_l) + pbh_key = pbh_data[mask].T + pbh_key.reset_index(drop=True, inplace=True) + full_key = pd.concat([data, pbh_key], axis=1) + d_=pbh_hdf5_file.create_dataset(key, (full_key.shape[0], full_key.shape[1]), data=full_key) + d_lat = pbh_hdf5_file.create_dataset('lat_bin_edges', (len(lat_bin), 1), data=lat_bin) + d_long = pbh_hdf5_file.create_dataset('long_bin_edges', (len(lat_bin), 1), data=long_bin) no_pbh_hdf5_file.close() + pbh_hdf5_file.close() - #Creating final .h5 file with PBHs injected - f = h5py.File(output_root2 + '.h5', 'w') - - for key in key_list: - vars()['d_'+key]=f.create_dataset(key, (vars()['full_'+key].shape[0], vars()['full_'+key].shape[1]), data=vars()['full_'+key]) - - d_lat = f.create_dataset('lat_bin_edges', (len(lat_bin), 1), data=lat_bin) - d_long = f.create_dataset('long_bin_edges', (len(lat_bin), 1), data=long_bin) - f.close() + t1 = time.time() + print('Total runtime: {0:f} s'.format(t1 - t0)) return -def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir): +def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, inner_slope, v_esc): """ Save PBH configuration parameters into a yaml file @@ -3906,12 +3939,22 @@ def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir): c : float Concentration index. - Defaults to 10 (concentration index of the Milky Way, multiple papers show it to be anywhere from 10-12) + Defaults to 12.94 (the value given in McMillan 2016, used in the paper we derive the velocities from) r_vir : float The virial radius. Defaults to 200 kpc (The virial radius of the Milky Way) + inner_slope: float + The inner slope of the MW halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. + Inner_slope goes into the determination of the velocities and each value returns a slightly different distribution. + The default value is .5 because it is in the middle of the options. More investigation is needed. + + v_esc: int + The escape velocity of the Milky Way. + v_esc is used in calculating the velocities. + Default is 550 because most papers cite values of 515-575, with a lot being around 550. + Output ------ None @@ -3921,7 +3964,9 @@ def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir): 'pbh_mass': pbh_mass, 'r_max': r_max, 'c': c, - 'r_vir': r_vir} + 'r_vir': r_vir, + 'inner_slope': inner_slope, + 'v_esc': v_esc} generate_config_file(config_filename, config) From ac506b1e936cf1a19ca1ee467c1aaab5db2d768b Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Fri, 28 Feb 2020 16:18:04 -0800 Subject: [PATCH 026/125] Deleted default values in generate_pbh_config_file --- popsycle/synthetic.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 1610f0d8..109ed1fe 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3927,33 +3927,26 @@ def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, in fdm : float Fraction of dark matter. The fraction of dark matter that you want to consist of PBHs. - Defaults to 1. pbh_mass : int The single mass that all PBHs will have. - Defaults to 40 Msun (from LIGO detections thought to be primordial) r_max : float The maximum radius from the galactic center that you want to find PBHs at. - Defaults to 8.3 kpc (Where Earth is located) c : float Concentration index. - Defaults to 12.94 (the value given in McMillan 2016, used in the paper we derive the velocities from) r_vir : float The virial radius. - Defaults to 200 kpc (The virial radius of the Milky Way) inner_slope: float The inner slope of the MW halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. Inner_slope goes into the determination of the velocities and each value returns a slightly different distribution. - The default value is .5 because it is in the middle of the options. More investigation is needed. v_esc: int The escape velocity of the Milky Way. v_esc is used in calculating the velocities. - Default is 550 because most papers cite values of 515-575, with a lot being around 550. Output ------ From e43227fb222e06023424e157802201c2e1df9981 Mon Sep 17 00:00:00 2001 From: Kerianne28 Date: Wed, 4 Mar 2020 10:47:19 -0800 Subject: [PATCH 027/125] updating files --- popsycle/synthetic.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 47fc3509..fb43b33c 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1398,14 +1398,14 @@ def calc_events(hdf5_file, output_root2, ########## # Should I use starmap_async? results = pool.starmap(_calc_event_time_loop, inputs) - + pool.close() pool.join() # Remove all the None values # (occurs for patches with less than 10 objects) results = [i for i in results if i is not None] - + results_ev = [] results_bl = [] @@ -1536,6 +1536,7 @@ def _calc_event_time_loop(llbb, hdf5_file, obs_time, n_obs, radius_cut, #################### # Initialize events_llbb and blends_llbb. + events_llbb = None blends_llbb = None @@ -1559,8 +1560,9 @@ def _calc_event_time_loop(llbb, hdf5_file, obs_time, n_obs, radius_cut, return time_array = np.linspace(-1 * obs_time / 2.0, obs_time / 2.0, n_obs) - + for i in np.arange(len(time_array)): + # Find potential lenses and sources that fall within radius cut. lens_id, sorc_id, r_t, sep, event_id1, c = _calc_event_cands_radius(bigpatch, time_array[i], @@ -1575,7 +1577,7 @@ def _calc_event_time_loop(llbb, hdf5_file, obs_time, n_obs, radius_cut, # to hope that we can detect them. Trim on a Theta_E criteria. event_lbt = _calc_event_cands_thetaE(bigpatch, theta_E, u, theta_frac, lens_id, sorc_id, time_array[i]) - + if event_lbt is not None: # Concatenate the current event table # (at this l, b, time) with the rest. @@ -1585,6 +1587,7 @@ def _calc_event_time_loop(llbb, hdf5_file, obs_time, n_obs, radius_cut, events_llbb = event_lbt # Keep only unique events within our different time stamps + events_llbb = unique_events(events_llbb) ######### @@ -1594,7 +1597,6 @@ def _calc_event_time_loop(llbb, hdf5_file, obs_time, n_obs, radius_cut, # or the source, in the table. ########## blends_lbt = _calc_blends(bigpatch, c, event_lbt, blend_rad) - if blends_lbt is not None: # Concatenate the current blend table (at this l, b, time) # with the rest. @@ -1951,11 +1953,13 @@ def unique_events(event_table): # Pull the unique ID numbers for the lens and source and put them into a # table of 2 x N_lenses. + lens_uid = event_table[col_idx['obj_id'], :] sorc_uid = event_table[col_idx['obj_id'] + len(col_idx), :] events_uid = np.swapaxes(np.vstack((lens_uid, sorc_uid)), 0, 1) # Determine if we have unique events (and how many duplicates there are). + unique_returns = np.unique(events_uid, return_index=True, return_inverse=True, return_counts=True, axis=0) @@ -1967,12 +1971,14 @@ def unique_events(event_table): new_event_table = event_table[:, unique_indices] # Check for duplicate events and keep the one with the closest u. + dpdx = np.where(unique_counts > 1)[0] for ii in range(len(dpdx)): + # Fetch the duplicates for this event. dup_idx = np.where(unique_inverse == dpdx[ii])[0] dup_events = event_table[:, dup_idx] - min_idx = np.argmin(dup_events[len(col_idx) + len(col_idx) + 2 - 1]) + min_idx = np.argmin(dup_events[len(col_idx) + 2 - 1]) new_event_table[:, dpdx[ii]] = event_table[:, dup_idx[min_idx]] return new_event_table @@ -4250,8 +4256,9 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c= mass_within_r_max = (4*np.pi*rho_knot*(r_s**3)*(np.log((r_s + r_max)/r_s) - (r_max/(r_s+r_max))))*(((10**3)**3)/((10**6)**3))*fdm #Msun #Determine the number of PBHs within that distance - num_pbh_within_r_max = (mass_within_r_max/pbh_mass) - num_pbh_within_r_max = round(num_pbh_within_r_max) + #num_pbh_within_r_max = (mass_within_r_max/pbh_mass) + #num_pbh_within_r_max = round(num_pbh_within_r_max) + num_pbh_within_r_max = 8e7 """ Defining needed functions from the python package "NFWdist". @@ -4413,9 +4420,15 @@ def qnfw(p, con=5, logp=False): ubv_b = np.full(len(data_in_field), np.nan) ubv_h = np.full(len(data_in_field), np.nan) ubv_v = np.full(len(data_in_field), np.nan) + teff = np.full(len(data_in_field), np.nan) + grav = np.full(len(data_in_field), np.nan) + mbol = np.full(len(data_in_field), np.nan) + feh = np.full(len(data_in_field), np.nan) + ztf_g = np.full(len(data_in_field), np.nan) + ztf_r = np.full(len(data_in_field), np.nan) #Making a dataframe of all PBH data from PBHs in the field of view. - pbh_data = pd.DataFrame({'zams_mass':zams_mass, 'rem_id':rem_id, 'mass':mass, 'px':px, 'py':py, 'pz':pz, 'vx':vx, 'vy':vy, 'vz':vz, 'rad':r_in_field, 'glat':b_in_field, 'glon':l_in_field, 'vr':vr, 'mu_b':mu_b, 'mu_lcosb':mu_lcosb, 'age':age, 'popid':pop_id, 'ubv_k':ubv_k, 'ubv_i':ubv_i, 'exbv':exbv, 'obj_id':obj_id, 'ubv_j':ubv_j, 'ubv_u':ubv_u, 'ubv_r':ubv_r, 'ubv_b':ubv_b, 'ubv_h':ubv_h, 'ubv_v':ubv_v}) + pbh_data = pd.DataFrame({'zams_mass':zams_mass, 'rem_id':rem_id, 'mass':mass, 'px':px, 'py':py, 'pz':pz, 'vx':vx, 'vy':vy, 'vz':vz, 'rad':r_in_field, 'glat':b_in_field, 'glon':l_in_field, 'vr':vr, 'mu_b':mu_b, 'mu_lcosb':mu_lcosb, 'age':age, 'popid':pop_id, 'ubv_k':ubv_k, 'ubv_i':ubv_i, 'exbv':exbv, 'obj_id':obj_id, 'ubv_j':ubv_j, 'ubv_u':ubv_u, 'ubv_r':ubv_r, 'ubv_b':ubv_b, 'ubv_h':ubv_h, 'ubv_v':ubv_v, 'teff':teff, 'grav':grav, 'mbol':mbol, 'feh':feh, 'ztf_g':ztf_g, 'ztf_r':ztf_r}) #Opening the file with no PBHs and creating a new file for the PBHs added. no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') From 562191b0867019820519984da552f20e84bea0e7 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 4 Mar 2020 11:03:22 -0800 Subject: [PATCH 028/125] migrate changes from synthetic.py to run.py --- popsycle/run.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index c060552f..ded1df4e 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -279,6 +279,52 @@ def generate_popsycle_config_file(radius_cut, obs_time, generate_config_file(config_filename, config) +def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, inner_slope, v_esc): + """ + Save PBH configuration parameters into a yaml file + + Parameters + ---------- + + fdm : float + Fraction of dark matter. + The fraction of dark matter that you want to consist of PBHs. + + pbh_mass : int + The single mass that all PBHs will have. + + r_max : float + The maximum radius from the galactic center that you want to find PBHs at. + + c : float + Concentration index. + + r_vir : float + The virial radius. + + inner_slope: float + The inner slope of the MW halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. + Inner_slope goes into the determination of the velocities and each value returns a slightly different distribution. + + v_esc: int + The escape velocity of the Milky Way. + v_esc is used in calculating the velocities. + + Output + ------ + None + """ + + config = {'fdm': fdm, + 'pbh_mass': pbh_mass, + 'r_max': r_max, + 'c': c, + 'r_vir': r_vir, + 'inner_slope': inner_slope, + 'v_esc': v_esc} + generate_config_file(config_filename, config) + + def generate_config_file(config_filename, config): """ Save configuration parameters from a dictionary into a yaml file @@ -307,7 +353,8 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, walltime, seed=None, overwrite=False, submitFlag=True, skip_galaxia=False, skip_perform_pop_syn=False, - skip_calc_events=False, skip_refine_events=False): + skip_calc_events=False, skip_refine_events=False, + pbh_config_filename=None): """ Generates the slurm script that executes the PopSyCLE pipeline @@ -350,6 +397,11 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, Optional Parameters ------------------- + pbh_config_filename : str + Name of pbh_config.yaml file containing the PBH parameters + that will be passed along to the run_on_slurm.py command in the + slurm script. + seed : int If set to non-None, all random sampling will be seeded with the specified seed, forcing identical output for PyPopStar and PopSyCLE. @@ -496,6 +548,9 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, if skip_refine_events: optional_cmds += '--skip-refine-events ' + if pbh_config_filename: + optional_cmds += '--pbh-config-filename={0}'.format(pbh_config_filename) + # Populate the mpi_template specified inputs job_script = slurm_template.format(**locals()) From cd128705e3223f07aa875e13651da7c6a88f17ad Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 4 Mar 2020 11:13:01 -0800 Subject: [PATCH 029/125] change run.py to output files with _pbh if add_pbh is run --- popsycle/run.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index ded1df4e..721afe02 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -14,7 +14,7 @@ from popsycle import utils -def _return_filename_dict(output_root): +def _return_filename_dict(output_root, add_pbh_flag=False): """ Return the filenames of the files output by the pipeline @@ -36,9 +36,16 @@ def _return_filename_dict(output_root): # Write out all of the filenames using the output_root ebf_filename = '%s.ebf' % output_root hdf5_filename = '%s.h5' % output_root - events_filename = '%s_events.fits' % output_root - blends_filename = '%s_blends.fits' % output_root - noevents_filename = '%s_NOEVENTS.txt' % output_root + + # Append _pbh to filenames if add_pbh will be run + if add_pbh_flag: + events_filename = '%s_pbh_events.fits' % output_root + blends_filename = '%s_pbh_blends.fits' % output_root + noevents_filename = '%s_pbh_NOEVENTS.txt' % output_root + else: + events_filename = '%s_events.fits' % output_root + blends_filename = '%s_blends.fits' % output_root + noevents_filename = '%s_NOEVENTS.txt' % output_root # Add the filenames to a dictionary filename_dict = { @@ -671,7 +678,13 @@ def run(): sys.exit(1) # Return the dictionary containing PopSyCLE output filenames - filename_dict = _return_filename_dict(args.output_root) + # Append '_pbh' to filenames if add_pbh will be run + if args.pbh_config_filename is not None: + add_pbh_flag = True + else: + add_pbh_flag = False + filename_dict = _return_filename_dict(args.output_root, + add_pbh_flag=add_pbh_flag) # Prepare additional_photometric_systems additional_photometric_systems = None @@ -711,6 +724,12 @@ def run(): overwrite=args.overwrite, seed=args.seed) + # Append '_pbh' to output_root if add_pbh will be run + if args.pbh_config_filename is not None: + output_root = '%s_pbh' % args.output_root + else: + output_root = args.output_root + # only do stuff if optional config file for pbhs was provided if args.pbh_config_filename is not None: # Check for pbh config file. Exit if not present. @@ -725,14 +744,14 @@ def run(): pbh_config = synthetic.load_config(args.pbh_config_filename) # Check if .h5 file exists from perform popsyn, use as input for following function - if not os.path.exists({0}+'.h5'.format(args.output_root)): + if not os.path.exists(filename_dict['hdf5_filename']): print("""Error: H5 file was not created properly by synthetic.perform_pop_syn""") sys.exit(1) synthetic.add_pbh(hdf5_file=filename_dict['hdf5_filename'], ebf_file=filename_dict['ebf_filename'], - output_root2=args.output_root, + output_root2=output_root, fdm=pbh_config['fdm'], pbh_mass=pbh_config['pbh_mass'], r_max=pbh_config['r_max'], @@ -755,7 +774,7 @@ def run(): # Run calc_events print('-- Executing calc_events') synthetic.calc_events(hdf5_file=filename_dict['hdf5_filename'], - output_root2=args.output_root, + output_root2=output_root, radius_cut=popsycle_config['radius_cut'], obs_time=popsycle_config['obs_time'], n_obs=popsycle_config['n_obs'], @@ -775,7 +794,7 @@ def run(): if not args.skip_refine_events: # Remove refine_events output if already exists and overwrite=True filename = '{0:s}_refined_events_{1:s}_{2:s}.' \ - 'fits'.format(args.output_root, + 'fits'.format(output_root, popsycle_config['filter_name'], popsycle_config['red_law']) if _check_for_output(filename, args.overwrite): @@ -783,7 +802,7 @@ def run(): # Run refine_events print('-- Executing refine_events') - synthetic.refine_events(input_root=args.output_root, + synthetic.refine_events(input_root=output_root, filter_name=popsycle_config['filter_name'], photometric_system=popsycle_config['photometric_system'], red_law=popsycle_config['red_law'], From ff69b49ea4fb16205b2757ad9133343c018d3574 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 4 Mar 2020 11:15:55 -0800 Subject: [PATCH 030/125] add safety check for input and output hdf5 files --- popsycle/synthetic.py | 55 ++++++------------------------------------- 1 file changed, 7 insertions(+), 48 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index da13b285..fd1c5fba 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3254,7 +3254,6 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c= 'or pick a new name.') # Error handling/complaining if input types are not right. - if ebf_file[-4:] != '.ebf': raise Exception('ebf_file must be an ebf file.') @@ -3285,6 +3284,13 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c= if type(seed) != int: raise Exception('seed must be an integer.') + # Check to make sure that the output hdf5 file + # will not overwrite the input hdf5 file + output_hdf5_file = '%s.h5' % output_root2 + if hdf5_file == output_hdf5_file: + raise Exception('Output %s cannot be equal to ' + 'input %s' % (output_hdf5_file, hdf5_file)) + ########## # Start of code ######### @@ -3547,50 +3553,3 @@ def qnfw(p, con=5, logp=False): print('Total runtime: {0:f} s'.format(t1 - t0)) return - -def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, inner_slope, v_esc): - """ - Save PBH configuration parameters into a yaml file - - Parameters - ---------- - - fdm : float - Fraction of dark matter. - The fraction of dark matter that you want to consist of PBHs. - - pbh_mass : int - The single mass that all PBHs will have. - - r_max : float - The maximum radius from the galactic center that you want to find PBHs at. - - c : float - Concentration index. - - r_vir : float - The virial radius. - - inner_slope: float - The inner slope of the MW halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. - Inner_slope goes into the determination of the velocities and each value returns a slightly different distribution. - - v_esc: int - The escape velocity of the Milky Way. - v_esc is used in calculating the velocities. - - Output - ------ - None - """ - - config = {'fdm': fdm, - 'pbh_mass': pbh_mass, - 'r_max': r_max, - 'c': c, - 'r_vir': r_vir, - 'inner_slope': inner_slope, - 'v_esc': v_esc} - generate_config_file(config_filename, config) - - From d680da4c647ef8ecabbe011de77940b1653c12c5 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 4 Mar 2020 11:16:19 -0800 Subject: [PATCH 031/125] add safety check for input and output hdf5 files --- popsycle/synthetic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index fd1c5fba..c29ed2b6 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3288,8 +3288,8 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c= # will not overwrite the input hdf5 file output_hdf5_file = '%s.h5' % output_root2 if hdf5_file == output_hdf5_file: - raise Exception('Output %s cannot be equal to ' - 'input %s' % (output_hdf5_file, hdf5_file)) + raise Exception('Output hdf5 file %s cannot be equal to ' + 'input hdf5 file %s' % (output_hdf5_file, hdf5_file)) ########## # Start of code From 67caa8c75236dcbe1e9fdc43782d0d216a82b926 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 4 Mar 2020 11:23:15 -0800 Subject: [PATCH 032/125] commented out broken code --- popsycle/synthetic.py | 47 +++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index c29ed2b6..fbb135d9 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3320,7 +3320,8 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c= #Later used to set the IDs of the PBHs. max_id_no_pbh = [] for key in key_list: - max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key][20])) + # max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key][20])) + max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key]['obj_id'])) max_id = np.amax(max_id_no_pbh) no_pbh_hdf5_file.close() @@ -3516,7 +3517,17 @@ def qnfw(p, con=5, logp=False): ztf_r = np.full(len(data_in_field), np.nan) #Making a dataframe of all PBH data from PBHs in the field of view. - pbh_data = pd.DataFrame({'zams_mass':zams_mass, 'rem_id':rem_id, 'mass':mass, 'px':px, 'py':py, 'pz':pz, 'vx':vx, 'vy':vy, 'vz':vz, 'rad':r_in_field, 'glat':b_in_field, 'glon':l_in_field, 'vr':vr, 'mu_b':mu_b, 'mu_lcosb':mu_lcosb, 'age':age, 'popid':pop_id, 'ubv_k':ubv_k, 'ubv_i':ubv_i, 'exbv':exbv, 'obj_id':obj_id, 'ubv_j':ubv_j, 'ubv_u':ubv_u, 'ubv_r':ubv_r, 'ubv_b':ubv_b, 'ubv_h':ubv_h, 'ubv_v':ubv_v, 'teff':teff, 'grav':grav, 'mbol':mbol, 'feh':feh, 'ztf_g':ztf_g, 'ztf_r':ztf_r}) + pbh_data = pd.DataFrame({'zams_mass': zams_mass, 'rem_id': rem_id, + 'mass': mass, 'px': px, 'py': py, 'pz': pz, + 'vx': vx, 'vy': vy, 'vz': vz, 'rad': r_in_field, + 'glat': b_in_field, 'glon': l_in_field, 'vr': vr, + 'mu_b': mu_b, 'mu_lcosb': mu_lcosb, 'age': age, + 'popid': pop_id, 'ubv_k': ubv_k, 'ubv_i': ubv_i, + 'exbv': exbv, 'obj_id': obj_id, 'ubv_j': ubv_j, + 'ubv_u': ubv_u, 'ubv_r': ubv_r, 'ubv_b': ubv_b, + 'ubv_h': ubv_h, 'ubv_v': ubv_v, 'teff': teff, + 'grav': grav, 'mbol': mbol, 'feh': feh, + 'ztf_g': ztf_g, 'ztf_r': ztf_r}) #Opening the file with no PBHs and creating a new file for the PBHs added. no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') @@ -3535,19 +3546,25 @@ def qnfw(p, con=5, logp=False): min_b = long_bin_values[idx2] lat_long_list.append((min_l[0], max_l[0], min_b[0], max_b[0])) - #Appending the PBH data to the no PBH data and writing to the new .h5 file. - for idx, key in enumerate(key_list): - data = pd.DataFrame(no_pbh_hdf5_file[key][:]) - min_l, max_l, min_b, max_b = lat_long_list[idx] - mask = (pbh_data['glon'] >= min_b) & (pbh_data['glon'] <= max_b) & (pbh_data['glat'] >= min_l) & (pbh_data['glat'] <= max_l) - pbh_key = pbh_data[mask].T - pbh_key.reset_index(drop=True, inplace=True) - full_key = pd.concat([data, pbh_key], axis=1) - d_=pbh_hdf5_file.create_dataset(key, (full_key.shape[0], full_key.shape[1]), data=full_key) - d_lat = pbh_hdf5_file.create_dataset('lat_bin_edges', (len(lat_bin), 1), data=lat_bin) - d_long = pbh_hdf5_file.create_dataset('long_bin_edges', (len(lat_bin), 1), data=long_bin) - no_pbh_hdf5_file.close() - pbh_hdf5_file.close() + """ + WARNING: THIS CODE IS NOW BROKEN AND NEEDS TO BE REWRITTEN + THE NEW DATA FORMAT FOR HDF5 FILES IS COMPOUND DATATYPE + AND THE DATA ONCE READ IS A NUMPY RECARRAY + """ + + # #Appending the PBH data to the no PBH data and writing to the new .h5 file. + # for idx, key in enumerate(key_list): + # data = pd.DataFrame(no_pbh_hdf5_file[key][:]) + # min_l, max_l, min_b, max_b = lat_long_list[idx] + # mask = (pbh_data['glon'] >= min_b) & (pbh_data['glon'] <= max_b) & (pbh_data['glat'] >= min_l) & (pbh_data['glat'] <= max_l) + # pbh_key = pbh_data[mask].T + # pbh_key.reset_index(drop=True, inplace=True) + # full_key = pd.concat([data, pbh_key], axis=1) + # d_= pbh_hdf5_file.create_dataset(key, (full_key.shape[0], full_key.shape[1]), data=full_key) + # d_lat = pbh_hdf5_file.create_dataset('lat_bin_edges', (len(lat_bin), 1), data=lat_bin) + # d_long = pbh_hdf5_file.create_dataset('long_bin_edges', (len(lat_bin), 1), data=long_bin) + # no_pbh_hdf5_file.close() + # pbh_hdf5_file.close() t1 = time.time() print('Total runtime: {0:f} s'.format(t1 - t0)) From d5a2393cbe3e47cee3e7ef2e1044150d21594677 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 4 Mar 2020 11:39:02 -0800 Subject: [PATCH 033/125] reformat --- popsycle/synthetic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index fbb135d9..4ed1e636 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3171,7 +3171,9 @@ def angdist(ra1, dec1, ra2, dec2): return distance -def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c=12.94, r_vir=200, inner_slope = .5, v_esc = 550, overwrite = False, seed = None): +def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, + r_max=8.3, c=12.94, r_vir=200, inner_slope = .5, v_esc = 550, + overwrite = False, seed = None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, and saves them in a new HDF5 file with the PBHs added. From 6d00c9ab3c6e420486aa08c58a186a496ff53c20 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 4 Mar 2020 11:39:15 -0800 Subject: [PATCH 034/125] reformat --- popsycle/synthetic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 4ed1e636..2e9c0e9c 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3172,8 +3172,8 @@ def angdist(ra1, dec1, ra2, dec2): def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, - r_max=8.3, c=12.94, r_vir=200, inner_slope = .5, v_esc = 550, - overwrite = False, seed = None): + r_max=8.3, c=12.94, r_vir=200, inner_slope=.5, v_esc=550, + overwrite=False, seed=None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, and saves them in a new HDF5 file with the PBHs added. From 6955ed3cad52cf41afaa20bcb97bb14eb253659d Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 6 Mar 2020 07:36:06 -0800 Subject: [PATCH 035/125] correct documentation --- docs/PopSyCLE_example.ipynb | 125 +++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 3 deletions(-) diff --git a/docs/PopSyCLE_example.ipynb b/docs/PopSyCLE_example.ipynb index 5896e6ca..6fa3bae8 100755 --- a/docs/PopSyCLE_example.ipynb +++ b/docs/PopSyCLE_example.ipynb @@ -95,7 +95,10 @@ }, { "cell_type": "markdown", - "source": " \nThe results should look like this:\n", + "source": [ + " \n", + "The results should look like this:\n" + ], "metadata": { "pycharm": { "metadata": false @@ -107,7 +110,122 @@ "metadata": { "pycharm": {} }, - "source": "\nCODEDATAPATH=/u/casey_lam/scratch/GalaxiaData/\n\nReading Parameter file- galaxia_params.example.txt\n___\n\noutputFile example \noutputDir ./ \nphotoSys UBV \nmagcolorNames V,B-V \nappMagLimits[0] -1000 \nappMagLimits[1] 1000 \nabsMagLimits[0] -1000 \nabsMagLimits[1] 1000 \ncolorLimits[0] -1000 \ncolorLimits[1] 1000 \ngeometryOption 1 \nlongitude 1.25 \nlatitude -2.65 \nsurveyArea 0.001 \nfSample 1 \npopID -1 \nwarpFlareOn 1 \nseed 12 \nr_max 20 \nstarType 0 \nphotoError 0 \n___\nReading tabulated values from file- /u/casey_lam/scratch/GalaxiaData/Model/vcirc.dat\nUsing geometry: Patch at l ,b : (1.25 -2.65) d_theta=0.0178412\nReading Isochrones from dir- /u/casey_lam/scratch/GalaxiaData/Isochrones/padova/UBV\nIsochrone Grid Size: (Age bins=182,Feh bins=34,Alpha bins=1)\nTime Isocrhone Reading 2.73553 \nGenerating populations................\n___\nThin Disk,ID=0:\nReading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_0_E1.ebf\nTime Tree generation/reading = 1.34589 \nCompleted % <0..10..20..30..40..50..60..70..80..90..>\nStars spawned = 15 \nTime Spawning= 0.519746 \n___\nThin Disk,ID=1:\nReading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_1_E1.ebf\nTime Tree generation/reading = 0.894184 \nCompleted % <0..9..19..29..39..49..59..69..79..89..99..>\nStars spawned = 785 \nTime Spawning= 3.01123 \n___\nThin Disk,ID=2:\nReading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_2_E1.ebf\nTime Tree generation/reading = 1.21969 \nCompleted % <0..9..19..29..39..49..59..69..79..89..99..>\nStars spawned = 2304 \nTime Spawning= 2.31239 \n___\nThin Disk,ID=3:\nReading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_3_E1.ebf\nTime Tree generation/reading = 1.12221 \nCompleted % <0..9..19..29..39..49..59..69..79..89..99..>\nStars spawned = 3299 \nTime Spawning= 2.63636 \n___\nThin Disk,ID=4:\nReading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_4_E1.ebf\nTime Tree generation/reading = 1.12869 \nCompleted % <0..9..19..29..39..49..59..69..79..89..99..>\nStars spawned = 9040 \nTime Spawning= 5.16435 \n___\nThin Disk,ID=5:\nReading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_5_E1.ebf\nTime Tree generation/reading = 0.924357 \nCompleted % <0..9..19..29..39..49..59..69..79..89..99..>\nStars spawned = 9983 \nTime Spawning= 5.99943 \n___\nThin Disk,ID=6:\nReading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_6_E1.ebf\nTime Tree generation/reading = 1.03473 \nCompleted % <0..9..19..29..39..49..59..69..79..89..99..>\nStars spawned = 11927 \nTime Spawning= 5.80737 \n___\nThickDisk:\nReading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_7_E0.ebf\nTime Tree generation/reading = 1.00016 \nCompleted % <0..9..19..29..39..49..59..69..79..89..99..>\nStars spawned = 6561 \nTime Spawning= 1.83702 \n___\nSpheroid:\nReading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_8_E0.ebf\nTime Tree generation/reading = 0.320754 \nCompleted % <0..9..19..29..39..49..59..69..79..89..99..>\nStars spawned = 589 \nTime Spawning= 0.693446 \n___\nBulge:\nReading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_9_E0.ebf\nTime Tree generation/reading = 1.0575 \nCompleted % <0..9..19..29..39..49..59..69..79..89..99..>\nStars spawned = 620676 \nTime Spawning= 28.1113 \n___\nTotal stars written 665179 \nFile written- .//example.ebf\nCalulating magnitudes................\nReading Isochrones from dir- /u/casey_lam/scratch/GalaxiaData/Isochrones/padova/UBV\nIsochrone Grid Size: (Age bins=182,Feh bins=34,Alpha bins=1)\nTime Isocrhone Reading 2.67512 \nCalulating Extinction................\nTime for extinction calculation 0.876306 \nTotal Time= 73.89 \n" + "source": [ + "\n", + "CODEDATAPATH=/u/casey_lam/scratch/GalaxiaData/\n", + "\n", + "Reading Parameter file- galaxia_params.example.txt\n", + "___\n", + "\n", + "outputFile example \n", + "outputDir ./ \n", + "photoSys UBV \n", + "magcolorNames V,B-V \n", + "appMagLimits[0] -1000 \n", + "appMagLimits[1] 1000 \n", + "absMagLimits[0] -1000 \n", + "absMagLimits[1] 1000 \n", + "colorLimits[0] -1000 \n", + "colorLimits[1] 1000 \n", + "geometryOption 1 \n", + "longitude 1.25 \n", + "latitude -2.65 \n", + "surveyArea 0.001 \n", + "fSample 1 \n", + "popID -1 \n", + "warpFlareOn 1 \n", + "seed 12 \n", + "r_max 20 \n", + "starType 0 \n", + "photoError 0 \n", + "___\n", + "Reading tabulated values from file- /u/casey_lam/scratch/GalaxiaData/Model/vcirc.dat\n", + "Using geometry: Patch at l ,b : (1.25 -2.65) d_theta=0.0178412\n", + "Reading Isochrones from dir- /u/casey_lam/scratch/GalaxiaData/Isochrones/padova/UBV\n", + "Isochrone Grid Size: (Age bins=182,Feh bins=34,Alpha bins=1)\n", + "Time Isocrhone Reading 2.73553 \n", + "Generating populations................\n", + "___\n", + "Thin Disk,ID=0:\n", + "Reading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_0_E1.ebf\n", + "Time Tree generation/reading = 1.34589 \n", + "Completed % <0..10..20..30..40..50..60..70..80..90..>\n", + "Stars spawned = 15 \n", + "Time Spawning= 0.519746 \n", + "___\n", + "Thin Disk,ID=1:\n", + "Reading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_1_E1.ebf\n", + "Time Tree generation/reading = 0.894184 \n", + "Completed % <0..9..19..29..39..49..59..69..79..89..99..>\n", + "Stars spawned = 785 \n", + "Time Spawning= 3.01123 \n", + "___\n", + "Thin Disk,ID=2:\n", + "Reading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_2_E1.ebf\n", + "Time Tree generation/reading = 1.21969 \n", + "Completed % <0..9..19..29..39..49..59..69..79..89..99..>\n", + "Stars spawned = 2304 \n", + "Time Spawning= 2.31239 \n", + "___\n", + "Thin Disk,ID=3:\n", + "Reading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_3_E1.ebf\n", + "Time Tree generation/reading = 1.12221 \n", + "Completed % <0..9..19..29..39..49..59..69..79..89..99..>\n", + "Stars spawned = 3299 \n", + "Time Spawning= 2.63636 \n", + "___\n", + "Thin Disk,ID=4:\n", + "Reading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_4_E1.ebf\n", + "Time Tree generation/reading = 1.12869 \n", + "Completed % <0..9..19..29..39..49..59..69..79..89..99..>\n", + "Stars spawned = 9040 \n", + "Time Spawning= 5.16435 \n", + "___\n", + "Thin Disk,ID=5:\n", + "Reading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_5_E1.ebf\n", + "Time Tree generation/reading = 0.924357 \n", + "Completed % <0..9..19..29..39..49..59..69..79..89..99..>\n", + "Stars spawned = 9983 \n", + "Time Spawning= 5.99943 \n", + "___\n", + "Thin Disk,ID=6:\n", + "Reading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_6_E1.ebf\n", + "Time Tree generation/reading = 1.03473 \n", + "Completed % <0..9..19..29..39..49..59..69..79..89..99..>\n", + "Stars spawned = 11927 \n", + "Time Spawning= 5.80737 \n", + "___\n", + "ThickDisk:\n", + "Reading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_7_E0.ebf\n", + "Time Tree generation/reading = 1.00016 \n", + "Completed % <0..9..19..29..39..49..59..69..79..89..99..>\n", + "Stars spawned = 6561 \n", + "Time Spawning= 1.83702 \n", + "___\n", + "Spheroid:\n", + "Reading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_8_E0.ebf\n", + "Time Tree generation/reading = 0.320754 \n", + "Completed % <0..9..19..29..39..49..59..69..79..89..99..>\n", + "Stars spawned = 589 \n", + "Time Spawning= 0.693446 \n", + "___\n", + "Bulge:\n", + "Reading tree from file- /u/casey_lam/scratch/GalaxiaData/BHTree-2.2/bhtree_with_wf/bhtree_9_E0.ebf\n", + "Time Tree generation/reading = 1.0575 \n", + "Completed % <0..9..19..29..39..49..59..69..79..89..99..>\n", + "Stars spawned = 620676 \n", + "Time Spawning= 28.1113 \n", + "___\n", + "Total stars written 665179 \n", + "File written- .//example.ebf\n", + "Calulating magnitudes................\n", + "Reading Isochrones from dir- /u/casey_lam/scratch/GalaxiaData/Isochrones/padova/UBV\n", + "Isochrone Grid Size: (Age bins=182,Feh bins=34,Alpha bins=1)\n", + "Time Isocrhone Reading 2.67512 \n", + "Calulating Extinction................\n", + "Time for extinction calculation 0.876306 \n", + "Total Time= 73.89 \n" + ] }, { "cell_type": "markdown", @@ -320,7 +438,8 @@ ], "source": [ "synthetic.refine_events(input_root = 'example', \n", - " filter_name = 'i', \n", + " filter_name = 'I',\n", + " photometric_system = 'ubv',\n", " red_law = 'Damineli16', \n", " overwrite = False, \n", " output_file = 'default')" From e2ec7fa86e5a55b49c11c88d70396572d7cc1694 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 6 Mar 2020 09:14:53 -0800 Subject: [PATCH 036/125] working version of updated hdf5 file format --- popsycle/synthetic.py | 214 ++++++++++++++++++++++++------------------ 1 file changed, 123 insertions(+), 91 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 2e9c0e9c..5f39d451 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -21,7 +21,7 @@ import astropy.coordinates as coord from astropy.coordinates.representation import UnitSphericalRepresentation from astropy.coordinates import SkyCoord # High-level coordinates -from astropy.coordinates import Angle, Latitude, Longitude # Angles +from astropy.coordinates import Angle # Angles from astropy.table import Table from astropy.table import vstack from popstar.imf import imf @@ -41,6 +41,7 @@ from multiprocessing import Pool import inspect import numpy.lib.recfunctions as rfn +import shutil from popsycle import utils @@ -3314,8 +3315,8 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, key_list = [key for key in key_list if 'bin_edges' not in key] #Get data from lat_bin_edges and long_bin_edges - lat_bin = pd.DataFrame(np.array(no_pbh_hdf5_file['lat_bin_edges'])) - long_bin = pd.DataFrame(np.array(no_pbh_hdf5_file['long_bin_edges'])) + lat_bin = no_pbh_hdf5_file['lat_bin_edges'][:] + long_bin = no_pbh_hdf5_file['long_bin_edges'][:] bin_edges_number = len(long_bin) #Getting the maximum ID from all of the stars and compact objects. @@ -3326,6 +3327,8 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key]['obj_id'])) max_id = np.amax(max_id_no_pbh) + hdf5_dset_names = no_pbh_hdf5_file[key_list[0]][:].dtype.names + no_pbh_hdf5_file.close() #Read in ebf file @@ -3339,7 +3342,7 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, surveyArea = float(ebf_log['surveyArea']) #Calculate the size of the field of view we are running - field_of_view_diameter = 2*((surveyArea/np.pi)**(1/2)) + field_of_view_radius = (surveyArea/np.pi)**(1/2) #NFW Profile calculations to determine mass of dark matter within given distance of galactic center rho_crit = ((3*(h**2))/(8*np.pi*g))*(10**6) #Msun/Mpc^3 @@ -3347,9 +3350,10 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, mass_within_r_max = (4*np.pi*rho_knot*(r_s**3)*(np.log((r_s + r_max)/r_s) - (r_max/(r_s+r_max))))*(((10**3)**3)/((10**6)**3))*fdm #Msun #Determine the number of PBHs within that distance - #num_pbh_within_r_max = (mass_within_r_max/pbh_mass) - #num_pbh_within_r_max = round(num_pbh_within_r_max) - num_pbh_within_r_max = 8e7 + num_pbh_within_r_max = (mass_within_r_max/pbh_mass) + num_pbh_within_r_max = int(num_pbh_within_r_max) + # num_pbh_within_r_max = int(1e8) + print('%i PBHs generated' % num_pbh_within_r_max) """ Defining needed functions from the python package "NFWdist". @@ -3409,7 +3413,7 @@ def qnfw(p, con=5, logp=False): #Sample PBH latitude and longitudes to get full spherical coordinates. sin_lats = np.random.uniform(-1, 1, int(num_pbh_within_r_max)) - lats=np.arcsin(sin_lats) + lats = np.arcsin(sin_lats) longs = np.random.uniform(0, np.pi*2, int(num_pbh_within_r_max)) #Converting spherical galactocentric coordinates to cartesian galactocentric coordinates @@ -3422,23 +3426,28 @@ def qnfw(p, con=5, logp=False): #outputs l, b, and distance in degrees. galactic = galacto.transform_to(coord.Galactic(representation_type='cartesian')) - latitude=galactic.b.deg - longitude=galactic.l.deg + latitude = galactic.b.deg + longitude = galactic.l.deg #Adjusting longitude values to match the coordinate format that we need - longitude=np.where(longitude>180, longitude-360, longitude) - - #Set minimum and maximum l and b for PBH mask. - l_min = np.min(long_bin).values - l_max = np.max(long_bin).values - b_min = np.min(lat_bin).values - b_max = np.max(lat_bin).values + longitude = np.where(longitude>180, longitude-360, longitude) #Calculating distances between each latitude, longitude pair, and the center of the field of view. - dists = angdist(l, b, longitude, latitude) + # dists = angdist(l, b, longitude, latitude) + delta_l = np.cos(np.radians(b)) * (longitude - l) + delta_b = latitude - b + dists = np.hypot(delta_l, delta_b) #Masking the full PBH data to obtain just the PBHs in our field of view. - mask = (galactic.distance.kpc <= 2*r_max) & (dists < field_of_view_diameter) + mask = (galactic.distance.kpc <= 2*r_max) & (dists < field_of_view_radius) data_in_field = galactic[mask] + N_PBHs_in_field = len(data_in_field) + print('%i PBHs in the field' % N_PBHs_in_field) + + if N_PBHs_in_field == 0: + print('-- No PBHs in the field') + print('-- Copying %s to %s' % (hdf5_file, output_hdf5_file)) + shutil.copy(hdf5_file, output_hdf5_file) + return #Obtain the radius, l, and b values for all of the PBHs in our field of view. r_in_field = data_in_field.distance.kpc @@ -3453,12 +3462,13 @@ def qnfw(p, con=5, logp=False): #Inner slope of the MW halo #From Lacroix et al 2018, Figure 11 (top left panel) + data_dir = '%s/data' % os.path.dirname(inspect.getfile(add_pbh)) if inner_slope == 1: - vel_data = pd.read_csv('data/radial_velocity_profile_steep.csv') + vel_data = pd.read_csv('%s/radial_velocity_profile_steep.csv' % data_dir) elif inner_slope == .25: - vel_data = pd.read_csv('data/radial_velocity_profile_shallow.csv') + vel_data = pd.read_csv('%s/radial_velocity_profile_shallow.csv' % data_dir) else: - vel_data = pd.read_csv('data/radial_velocity_profile_middle.csv') + vel_data = pd.read_csv('%s/radial_velocity_profile_middle.csv' % data_dir) #Interpolating v values from the above data, given the PBH r values. pbh_vrms = np.interp(pbh_r_galacto, vel_data['r'], vel_data['v']) @@ -3481,92 +3491,114 @@ def qnfw(p, con=5, logp=False): #Transforming velocities to cartesian to get vx, vy, and vz. cart_vel = astropy.coordinates.spherical_to_cartesian(interpreted_rms_velocities, lat_vel, long_vel) - vx = cart_vel[0] - vy = cart_vel[1] - vz = cart_vel[2] + #Load up a numpy array + comp_dtype = _generate_comp_dtype(hdf5_dset_names) + pbh_data = np.empty(len(data_in_field), dtype=comp_dtype) + + pbh_data['rad'] = r_in_field + pbh_data['glon'] = l_in_field + pbh_data['glat'] = b_in_field + + pbh_data['vx'] = cart_vel[0] + pbh_data['vy'] = cart_vel[1] + pbh_data['vz'] = cart_vel[2] #Getting the rest of the PBH data for the combined .h5 file - mass = np.full(len(data_in_field), pbh_mass) - zams_mass = mass - age = np.full(len(data_in_field), np.nan) - pop_id = np.full(len(data_in_field), 10) - rem_id = np.full(len(data_in_field), 104) + pbh_data['mass'] = np.full(len(data_in_field), pbh_mass) + pbh_data['zams_mass'] = np.full(len(data_in_field), pbh_mass) + pbh_data['age'] = np.full(len(data_in_field), np.nan) + pbh_data['popid'] = np.full(len(data_in_field), 10) + pbh_data['rem_id'] = np.full(len(data_in_field), 104) b_rad = np.radians(b_in_field) l_rad = np.radians(l_in_field) cart_helio = astropy.coordinates.spherical_to_cartesian(r_in_field, b_rad, l_rad) - px = cart_helio[0] - py = cart_helio[1] - pz = cart_helio[2] - - vr, mu_b, mu_lcosb = calc_sph_motion(vx, vy, vz, r_in_field, b_in_field, l_in_field) - obj_id = np.arange((max_id+1), (max_id+len(data_in_field)+1)) - - exbv = np.full(len(data_in_field), np.nan) - ubv_k = np.full(len(data_in_field), np.nan) - ubv_j = np.full(len(data_in_field), np.nan) - ubv_i = np.full(len(data_in_field), np.nan) - ubv_u = np.full(len(data_in_field), np.nan) - ubv_r = np.full(len(data_in_field), np.nan) - ubv_b = np.full(len(data_in_field), np.nan) - ubv_h = np.full(len(data_in_field), np.nan) - ubv_v = np.full(len(data_in_field), np.nan) - teff = np.full(len(data_in_field), np.nan) - grav = np.full(len(data_in_field), np.nan) - mbol = np.full(len(data_in_field), np.nan) - feh = np.full(len(data_in_field), np.nan) - ztf_g = np.full(len(data_in_field), np.nan) - ztf_r = np.full(len(data_in_field), np.nan) - - #Making a dataframe of all PBH data from PBHs in the field of view. - pbh_data = pd.DataFrame({'zams_mass': zams_mass, 'rem_id': rem_id, - 'mass': mass, 'px': px, 'py': py, 'pz': pz, - 'vx': vx, 'vy': vy, 'vz': vz, 'rad': r_in_field, - 'glat': b_in_field, 'glon': l_in_field, 'vr': vr, - 'mu_b': mu_b, 'mu_lcosb': mu_lcosb, 'age': age, - 'popid': pop_id, 'ubv_k': ubv_k, 'ubv_i': ubv_i, - 'exbv': exbv, 'obj_id': obj_id, 'ubv_j': ubv_j, - 'ubv_u': ubv_u, 'ubv_r': ubv_r, 'ubv_b': ubv_b, - 'ubv_h': ubv_h, 'ubv_v': ubv_v, 'teff': teff, - 'grav': grav, 'mbol': mbol, 'feh': feh, - 'ztf_g': ztf_g, 'ztf_r': ztf_r}) - - #Opening the file with no PBHs and creating a new file for the PBHs added. - no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') - pbh_hdf5_file = h5py.File(output_root2 + '.h5', 'w') + pbh_data['px'] = cart_helio[0] + pbh_data['py'] = cart_helio[1] + pbh_data['pz'] = cart_helio[2] + + vr, mu_b, mu_lcosb = calc_sph_motion(pbh_data['vx'], + pbh_data['vy'], + pbh_data['vz'], + r_in_field, b_in_field, l_in_field) + pbh_data['vr'] = vr + pbh_data['mu_b'] = mu_b + pbh_data['mu_lcosb'] = mu_lcosb + pbh_data['obj_id'] = np.arange((max_id+1), (max_id+len(data_in_field)+1)) + + pbh_data['exbv'] = np.full(len(data_in_field), np.nan) + pbh_data['ubv_K'] = np.full(len(data_in_field), np.nan) + pbh_data['ubv_J'] = np.full(len(data_in_field), np.nan) + pbh_data['ubv_I'] = np.full(len(data_in_field), np.nan) + pbh_data['ubv_U'] = np.full(len(data_in_field), np.nan) + pbh_data['ubv_R'] = np.full(len(data_in_field), np.nan) + pbh_data['ubv_B'] = np.full(len(data_in_field), np.nan) + pbh_data['ubv_H'] = np.full(len(data_in_field), np.nan) + pbh_data['ubv_V'] = np.full(len(data_in_field), np.nan) + pbh_data['teff'] = np.full(len(data_in_field), np.nan) + pbh_data['grav'] = np.full(len(data_in_field), np.nan) + pbh_data['mbol'] = np.full(len(data_in_field), np.nan) + pbh_data['feh'] = np.full(len(data_in_field), np.nan) + if any(['ztf' in n for n in hdf5_dset_names]): + pbh_data['ztf_g'] = np.full(len(data_in_field), np.nan) + pbh_data['ztf_r'] = np.full(len(data_in_field), np.nan) #Calculate the maximum and minimum l and b values for each dataset in the no PBH file, #so that we can determine which datasets to correctly add the PBHs. - lat_bin_values = lat_bin.values - long_bin_values = long_bin.values lat_long_list = [] - for idx in range(len(lat_bin_values)-1): - max_l = lat_bin_values[idx+1] - min_l = lat_bin_values[idx] - for idx2 in range(len(long_bin_values)-1): - max_b = long_bin_values[idx2+1] - min_b = long_bin_values[idx2] - lat_long_list.append((min_l[0], max_l[0], min_b[0], max_b[0])) + for idx in range(len(long_bin) - 1): + max_l = long_bin[idx + 1] + min_l = long_bin[idx] + for idx2 in range(len(lat_bin)-1): + max_b = lat_bin[idx2 + 1] + min_b = lat_bin[idx2] + lat_long_list.append((min_l, max_l, min_b, max_b)) """ WARNING: THIS CODE IS NOW BROKEN AND NEEDS TO BE REWRITTEN THE NEW DATA FORMAT FOR HDF5 FILES IS COMPOUND DATATYPE AND THE DATA ONCE READ IS A NUMPY RECARRAY """ + #Opening the file with no PBHs and creating a new file for the PBHs added. + no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') + pbh_hdf5_file = h5py.File(output_hdf5_file, 'w') + + #Appending the PBH data to the no PBH data and writing to the new .h5 file. + N_objs_no_pbh = 0 + N_objs_pbh = 0 + for idx, key in enumerate(key_list): + key_data = no_pbh_hdf5_file[key][:] + N_objs_no_pbh += key_data.shape[0] + + min_l, max_l, min_b, max_b = lat_long_list[idx] + mask = (pbh_data['glon'] >= min_l) & \ + (pbh_data['glon'] <= max_l) & \ + (pbh_data['glat'] >= min_b) & \ + (pbh_data['glat'] <= max_b) + + if np.sum(mask) == 0: + combined_data = key_data + else: + pbh_data_in_key = pbh_data[mask] + combined_data = np.hstack((key_data, pbh_data_in_key)) + N_objs_pbh += combined_data.shape[0] + _ = pbh_hdf5_file.create_dataset(key, + shape=(combined_data.shape[0],), + dtype=comp_dtype) + _ = pbh_hdf5_file.create_dataset('lat_bin_edges', (len(lat_bin), 1), data=lat_bin) + _ = pbh_hdf5_file.create_dataset('long_bin_edges', (len(lat_bin), 1), data=long_bin) + no_pbh_hdf5_file.close() + pbh_hdf5_file.close() + + print('Checking totals') + print('-- %i original objects' % N_objs_no_pbh) + print('-- %i PBHs in the field' % N_PBHs_in_field) + print('-- %i new total objects' % N_objs_pbh) - # #Appending the PBH data to the no PBH data and writing to the new .h5 file. - # for idx, key in enumerate(key_list): - # data = pd.DataFrame(no_pbh_hdf5_file[key][:]) - # min_l, max_l, min_b, max_b = lat_long_list[idx] - # mask = (pbh_data['glon'] >= min_b) & (pbh_data['glon'] <= max_b) & (pbh_data['glat'] >= min_l) & (pbh_data['glat'] <= max_l) - # pbh_key = pbh_data[mask].T - # pbh_key.reset_index(drop=True, inplace=True) - # full_key = pd.concat([data, pbh_key], axis=1) - # d_= pbh_hdf5_file.create_dataset(key, (full_key.shape[0], full_key.shape[1]), data=full_key) - # d_lat = pbh_hdf5_file.create_dataset('lat_bin_edges', (len(lat_bin), 1), data=lat_bin) - # d_long = pbh_hdf5_file.create_dataset('long_bin_edges', (len(lat_bin), 1), data=long_bin) - # no_pbh_hdf5_file.close() - # pbh_hdf5_file.close() + if N_objs_pbh == N_objs_no_pbh + N_PBHs_in_field: + print('-- Totals match!') + else: + print('** MISSING PBHs!! **') t1 = time.time() print('Total runtime: {0:f} s'.format(t1 - t0)) From d9dee2fb0f66a306cb11a4b77754268c97affd62 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 6 Mar 2020 09:56:07 -0800 Subject: [PATCH 037/125] pass data through to pbh_hdf5_file --- popsycle/synthetic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 5f39d451..1689cdfe 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3352,7 +3352,7 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, #Determine the number of PBHs within that distance num_pbh_within_r_max = (mass_within_r_max/pbh_mass) num_pbh_within_r_max = int(num_pbh_within_r_max) - # num_pbh_within_r_max = int(1e8) + num_pbh_within_r_max = int(1e7) print('%i PBHs generated' % num_pbh_within_r_max) """ @@ -3584,7 +3584,8 @@ def qnfw(p, con=5, logp=False): N_objs_pbh += combined_data.shape[0] _ = pbh_hdf5_file.create_dataset(key, shape=(combined_data.shape[0],), - dtype=comp_dtype) + dtype=comp_dtype, + data=combined_data) _ = pbh_hdf5_file.create_dataset('lat_bin_edges', (len(lat_bin), 1), data=lat_bin) _ = pbh_hdf5_file.create_dataset('long_bin_edges', (len(lat_bin), 1), data=long_bin) no_pbh_hdf5_file.close() From 2b126e83b11a7b3f476d60ccce894b2f1e39cb65 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Thu, 12 Mar 2020 17:24:43 -0700 Subject: [PATCH 038/125] WIP commit: updated with new method for determining positions, working through a error with appending PBHs --- popsycle/data/pbh_config.yaml | 2 + popsycle/run.py | 13 +- popsycle/synthetic.py | 286 +++++++++++++++++----------------- 3 files changed, 155 insertions(+), 146 deletions(-) diff --git a/popsycle/data/pbh_config.yaml b/popsycle/data/pbh_config.yaml index 14c5eac4..abc00d18 100644 --- a/popsycle/data/pbh_config.yaml +++ b/popsycle/data/pbh_config.yaml @@ -7,3 +7,5 @@ c: 12.94 #concentration index r_vir: 200 #kpc inner_slope: .5 v_esc: 550 +rho_0 : 0.0106 +n_lin : 1000 diff --git a/popsycle/run.py b/popsycle/run.py index 721afe02..64beb15f 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -286,7 +286,7 @@ def generate_popsycle_config_file(radius_cut, obs_time, generate_config_file(config_filename, config) -def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, inner_slope, v_esc): +def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, inner_slope, v_esc, rho_0, n_lin): """ Save PBH configuration parameters into a yaml file @@ -317,6 +317,13 @@ def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, in The escape velocity of the Milky Way. v_esc is used in calculating the velocities. + rho_0: float + The initial density that will be used in the NFW profile equations. + + n_lin: int + The number of times you want the density determined along the line of sight when calculating PBH positions + Defaults to 1000. Will need to make large if you are closer to the galactic center. + Output ------ None @@ -328,7 +335,9 @@ def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, in 'c': c, 'r_vir': r_vir, 'inner_slope': inner_slope, - 'v_esc': v_esc} + 'v_esc': v_esc, + 'rho_0':rho_0, + 'n_lin':n_lin} generate_config_file(config_filename, config) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 1689cdfe..d55ee35f 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -29,7 +29,7 @@ import scipy from scipy.interpolate import interp1d from scipy.spatial import cKDTree -from scipy import special +from scipy import special, integrate, interpolate import time import datetime from popsycle import ebf @@ -3171,10 +3171,26 @@ def angdist(ra1, dec1, ra2, dec2): distance = arccos(cosdist) / d2r return distance +def rho_dmhalo(r, rho_0, r_s, gamma): + """ + Density profile of the dark matter halo. + We are using the parametrization from McMillan (2017) Equation 5, + with defaults based on the mean values in Table 2. + r: galactocentric radius [units: kpc] + rho_0: characteristic density in [units: m_sun / pc**3] + r_s: scale radius in [units: kpc] + gamma: gamma=1 for NFW, gamma > 1 cuspy, gamma < 1 cored + + returns: density at r [units: m_sun / pc**3] + """ + x = r / r_s + rho = rho_0 / (x**gamma * (1 + x)**(3 - gamma)) + return rho + def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=8.3, c=12.94, r_vir=200, inner_slope=.5, v_esc=550, - overwrite=False, seed=None): + rho_0=0.0106, n_lin=1000, overwrite=False, seed=None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, and saves them in a new HDF5 file with the PBHs added. @@ -3216,6 +3232,15 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, The virial radius. Defaults to 200 kpc (The virial radius of the Milky Way) + rho_0: float + The initial density that will be used in the NFW profile equations. + Defaults to .0106 from McMillan 2016 + + n_lin: int + The number of times you want the density determined along the line of sight when calculating PBH positions + Defaults to 1000. Will need to make large if you are closer to the galactic center. + + Optional Parameters ------------------- inner_slope: float @@ -3338,109 +3363,90 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, #Obtain survey area and center latitude and longitude b = float(ebf_log['latitude']) + b_radian = b * np.pi / 180 l = float(ebf_log['longitude']) + l_radian = l * np.pi /180 surveyArea = float(ebf_log['surveyArea']) #Calculate the size of the field of view we are running field_of_view_radius = (surveyArea/np.pi)**(1/2) - #NFW Profile calculations to determine mass of dark matter within given distance of galactic center - rho_crit = ((3*(h**2))/(8*np.pi*g))*(10**6) #Msun/Mpc^3 - rho_knot = (200/3)*(rho_crit)*((c**3)/(np.log(c+1)-(c/(1+c)))) # Msun/Mpc^3 - mass_within_r_max = (4*np.pi*rho_knot*(r_s**3)*(np.log((r_s + r_max)/r_s) - (r_max/(r_s+r_max))))*(((10**3)**3)/((10**6)**3))*fdm #Msun - - #Determine the number of PBHs within that distance - num_pbh_within_r_max = (mass_within_r_max/pbh_mass) - num_pbh_within_r_max = int(num_pbh_within_r_max) - num_pbh_within_r_max = int(1e7) - print('%i PBHs generated' % num_pbh_within_r_max) - - """ - Defining needed functions from the python package "NFWdist". - Used to calculate PBH radii from galactic center assuming NFW profile. - https://github.com/CullanHowlett/NFWdist - - x, q: array_like - Vector of quantiles. This is scaled such that x=R/Rvir for NFW. This means the PDF is only defined between 0 and 1. - p: array_like - Vector of probabilities - - pnfw: distribution function - qnfw: Quantile function (CDF inversion) - """ - - def pnfwunorm(q, con=5): - if hasattr(con, '__len__'): - y = np.outer(q,con) - else: - y = q*con - return np.log(1.0 + y)-y/(1.0 + y) - - def pnfw(q, con=5, logp=False): - p = pnfwunorm(q, con=con)/pnfwunorm(1, con=con) - if hasattr(q, '__len__'): - p[q>1] = 1 - p[q<=0] = 0 - else: - if (q > 1): - p = 1 - elif (q <= 0): - p = 0 - if(logp): - return np.log(p) - else: - return p - - def qnfw(p, con=5, logp=False): - if (logp): - p = np.exp(p) - if hasattr(p, '__len__'): - p[p>1] = 1 - p[p<=0] = 0 - else: - if (p > 1): - p = 1 - elif (p <= 0): - p = 0 - if hasattr(con, '__len__'): - p = np.outer(p,pnfwunorm(1, con=con)) - else: - p *= pnfwunorm(1, con=con) - return (-(1.0/np.real(special.lambertw(-np.exp(-p-1))))-1)/con - - #Calculating radius values for all PBHs within r_max of the galactic center, using NFWDist functions - r_values = (qnfw(np.random.rand(int(num_pbh_within_r_max)) * pnfw(r_max/r_vir,con=c, logp=False), con=c)*r_vir) - - #Sample PBH latitude and longitudes to get full spherical coordinates. - sin_lats = np.random.uniform(-1, 1, int(num_pbh_within_r_max)) - lats = np.arcsin(sin_lats) - longs = np.random.uniform(0, np.pi*2, int(num_pbh_within_r_max)) - - #Converting spherical galactocentric coordinates to cartesian galactocentric coordinates - cart = astropy.coordinates.spherical_to_cartesian(r_values, lats, longs) - - #Defining galactocentric coordinate frame - galacto = coord.Galactocentric(x=cart[0] * units.kpc, y=cart[1] * units.kpc, z=cart[2] * units.kpc) - - #Transforming from galactocentric to galactic coordinates + # Generate an array of heliocentric radii + # These radii will just be used to numerically integrate the density + n_lin = 1000 + if np.logical_and(np.logical_and(np.abs(l_radian)<0.5 * np.pi / 180, + np.abs(b_radian)<0.5 * np.pi / 180), + n_lin<100000): + print('Warning: for fields very near the center of the Milky Way it is reocmmended that the number of elements used to estimate the density be n_lin>100000') + r_h_linspace = np.linspace(0, 2*r_max, num=n_lin) + + # Represent the line of sight line in galactic coordinates + galactic_lin = coord.Galactic(l=l_radian * units.rad, + b=b_radian * units.rad, + distance=r_h_linspace * units.kpc) + + # convert the line of sight to galactocentric coordinates #outputs l, b, and distance in degrees. - galactic = galacto.transform_to(coord.Galactic(representation_type='cartesian')) - - latitude = galactic.b.deg - longitude = galactic.l.deg - #Adjusting longitude values to match the coordinate format that we need - longitude = np.where(longitude>180, longitude-360, longitude) - - #Calculating distances between each latitude, longitude pair, and the center of the field of view. - # dists = angdist(l, b, longitude, latitude) - delta_l = np.cos(np.radians(b)) * (longitude - l) - delta_b = latitude - b - dists = np.hypot(delta_l, delta_b) - - #Masking the full PBH data to obtain just the PBHs in our field of view. - mask = (galactic.distance.kpc <= 2*r_max) & (dists < field_of_view_radius) - data_in_field = galactic[mask] - N_PBHs_in_field = len(data_in_field) + galactocen_lin = galactic_lin.transform_to(coord.Galactocentric(representation_type='spherical')) + + # Determine the dark matter density at all galactocentric radii along the line of sight. + rho_lin = rho_dmhalo(galactocen_lin.spherical.distance.value, + rho_0=rho_0, r_s=r_s, gamma=inner_slope) + + # Estimate the total mass within the line-of-sight cylinder [units: M_sun kpc**-2] + # Projected density along line of light + # (multiply by projected area to get total mass) + rho_marg_r = np.trapz(rho_lin, dx=(2*r_max) / n_lin) * 1000**3 + print("Projected density along line-of-sight = {0:0.2e} [M_sun kpc**-2]".format(rho_marg_r)) + # LOS cylinder radius in kpc, assuming small angle approximation [units: kpc] + r_proj_los_cyl = field_of_view_radius * np.pi / 180 * (2 * r_max) + # Projected area of the LOS cylinder [units: kpc**2] + area_proj_los_cyl = np.pi * r_proj_los_cyl**2 + # Mass within the total cylinder + mass_los_cyl = rho_marg_r * area_proj_los_cyl + print("Mass within line-of-sight cylinder = {0:0.2e} [M_sun]".format(mass_los_cyl)) + + # Total number of black holes to randomly draw + n_pbh = int(np.round(fdm * mass_los_cyl / pbh_mass)) + print('Number of PBH to populate LOS cylinder with = {0}'.format(n_pbh)) + + # Estimate the discrete CDF based on the discrete PDF + rho_marg_r_cum = integrate.cumtrapz(y=rho_lin, + x=galactic_lin.distance.kpc, + dx=(2*r_max) / n_lin) + cdf_los = rho_marg_r_cum / rho_marg_r_cum[-1] + # Since cumtrapz does not include zero for the first element insert it + cdf_los = np.insert(cdf_los,0,0) + + # Create a function to interpolate the CDF so that we can randomly sample from it + f_cdf_d = interpolate.interp1d(cdf_los, galactic_lin.distance.kpc) + + # Randomly sample galactic coordinates for the PBHs based on CDF + d_galac = f_cdf_d(np.random.uniform(size=n_pbh)) + + # Randomly assign a l & b galactic coordinate to each PBH, within the LOS cone + # sample the angle from 0 to 2pi + theta = np.random.uniform(size=n_pbh) * 2 * np.pi + # sample radius correcting for annular area to make uniform + r_cyl = r_proj_los_cyl * np.sqrt(np.random.uniform(size=n_pbh)) # kpc + y_cyl = r_cyl * np.sin(theta) # kpc + x_cyl = r_cyl * np.cos(theta) # kpc + + # Verify correct size + print(np.size(cdf_los)) + print(np.size(galactic_lin.distance.kpc)) + + # Mask out sampled PBH outside the observation cone + mask_obs_cone = r_cyl <= r_proj_los_cyl * d_galac / (2 * r_max) + + # Assuming small angle approximation + b_galac = r_cyl * np.sin(theta) / d_galac + b_radian # rad + l_galac = r_cyl * np.cos(theta) / np.cos(b_radian) / d_galac + l_radian # rad + + latitude = b_galac * (180/np.pi) #degrees + longitude = l_galac * (180/np.pi) #degrees + + N_PBHs_in_field = len(d_galac) print('%i PBHs in the field' % N_PBHs_in_field) if N_PBHs_in_field == 0: @@ -3449,15 +3455,10 @@ def qnfw(p, con=5, logp=False): shutil.copy(hdf5_file, output_hdf5_file) return - #Obtain the radius, l, and b values for all of the PBHs in our field of view. - r_in_field = data_in_field.distance.kpc - l_in_field = data_in_field.l.deg - b_in_field = data_in_field.b.deg - #Converting the PBH positions from the field of view back to galactocentric for determining velocities. - galactic_pbh = coord.Galactic(l=l_in_field * units.deg, b=b_in_field * units.deg, distance=r_in_field * units.kpc) - galactic_pbh = galactic_pbh.transform_to(coord.Galactocentric(representation_type='spherical')) - cart_pbh = astropy.coordinates.cartesian_to_spherical(galactic_pbh.x, galactic_pbh.y, galactic_pbh.z) + galactic_pbh = coord.Galactic(l=longitude * units.deg, b=latitude * units.deg, distance=d_galac * units.kpc) + galacto_pbh = galactic_pbh.transform_to(coord.Galactocentric(representation_type='spherical')) + cart_pbh = astropy.coordinates.cartesian_to_spherical(galacto_pbh.x, galacto_pbh.y, galacto_pbh.z) pbh_r_galacto = cart_pbh[0] #Inner slope of the MW halo @@ -3484,35 +3485,36 @@ def qnfw(p, con=5, logp=False): interpreted_rms_velocities = np.interp(rand_cdf, cdf, v_vals) #Sampling random latitude and longitude values for velocity to complete the spherical velocities. - sin_lat_vel = np.random.uniform(-1, 1, len(data_in_field)) + sin_lat_vel = np.random.uniform(-1, 1, len(d_galac)) lat_vel = np.arcsin(sin_lat_vel) - long_vel = np.random.uniform(0, 2*np.pi, len(data_in_field)) + long_vel = np.random.uniform(0, 2*np.pi, len(d_galac)) #Transforming velocities to cartesian to get vx, vy, and vz. cart_vel = astropy.coordinates.spherical_to_cartesian(interpreted_rms_velocities, lat_vel, long_vel) #Load up a numpy array comp_dtype = _generate_comp_dtype(hdf5_dset_names) - pbh_data = np.empty(len(data_in_field), dtype=comp_dtype) + pbh_data = np.empty(len(d_galac), dtype=comp_dtype) + + #Getting longitude into the right format for PopSyCLE + longitude = np.where(longitude>180, longitude-360, longitude) - pbh_data['rad'] = r_in_field - pbh_data['glon'] = l_in_field - pbh_data['glat'] = b_in_field + pbh_data['rad'] = d_galac + pbh_data['glon'] = longitude + pbh_data['glat'] = latitude pbh_data['vx'] = cart_vel[0] pbh_data['vy'] = cart_vel[1] pbh_data['vz'] = cart_vel[2] #Getting the rest of the PBH data for the combined .h5 file - pbh_data['mass'] = np.full(len(data_in_field), pbh_mass) - pbh_data['zams_mass'] = np.full(len(data_in_field), pbh_mass) - pbh_data['age'] = np.full(len(data_in_field), np.nan) - pbh_data['popid'] = np.full(len(data_in_field), 10) - pbh_data['rem_id'] = np.full(len(data_in_field), 104) - - b_rad = np.radians(b_in_field) - l_rad = np.radians(l_in_field) - cart_helio = astropy.coordinates.spherical_to_cartesian(r_in_field, b_rad, l_rad) + pbh_data['mass'] = np.full(len(d_galac), pbh_mass) + pbh_data['zams_mass'] = np.full(len(d_galac), pbh_mass) + pbh_data['age'] = np.full(len(d_galac), np.nan) + pbh_data['popid'] = np.full(len(d_galac), 10) + pbh_data['rem_id'] = np.full(len(d_galac), 104) + + cart_helio = astropy.coordinates.spherical_to_cartesian(d_galac, b_galac, l_galac) pbh_data['px'] = cart_helio[0] pbh_data['py'] = cart_helio[1] pbh_data['pz'] = cart_helio[2] @@ -3520,28 +3522,28 @@ def qnfw(p, con=5, logp=False): vr, mu_b, mu_lcosb = calc_sph_motion(pbh_data['vx'], pbh_data['vy'], pbh_data['vz'], - r_in_field, b_in_field, l_in_field) + d_galac, b_galac, l_galac) pbh_data['vr'] = vr pbh_data['mu_b'] = mu_b pbh_data['mu_lcosb'] = mu_lcosb - pbh_data['obj_id'] = np.arange((max_id+1), (max_id+len(data_in_field)+1)) - - pbh_data['exbv'] = np.full(len(data_in_field), np.nan) - pbh_data['ubv_K'] = np.full(len(data_in_field), np.nan) - pbh_data['ubv_J'] = np.full(len(data_in_field), np.nan) - pbh_data['ubv_I'] = np.full(len(data_in_field), np.nan) - pbh_data['ubv_U'] = np.full(len(data_in_field), np.nan) - pbh_data['ubv_R'] = np.full(len(data_in_field), np.nan) - pbh_data['ubv_B'] = np.full(len(data_in_field), np.nan) - pbh_data['ubv_H'] = np.full(len(data_in_field), np.nan) - pbh_data['ubv_V'] = np.full(len(data_in_field), np.nan) - pbh_data['teff'] = np.full(len(data_in_field), np.nan) - pbh_data['grav'] = np.full(len(data_in_field), np.nan) - pbh_data['mbol'] = np.full(len(data_in_field), np.nan) - pbh_data['feh'] = np.full(len(data_in_field), np.nan) + pbh_data['obj_id'] = np.arange((max_id+1), (max_id+len(d_galac)+1)) + + pbh_data['exbv'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_K'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_J'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_I'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_U'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_R'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_B'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_H'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_V'] = np.full(len(d_galac), np.nan) + pbh_data['teff'] = np.full(len(d_galac), np.nan) + pbh_data['grav'] = np.full(len(d_galac), np.nan) + pbh_data['mbol'] = np.full(len(d_galac), np.nan) + pbh_data['feh'] = np.full(len(d_galac), np.nan) if any(['ztf' in n for n in hdf5_dset_names]): - pbh_data['ztf_g'] = np.full(len(data_in_field), np.nan) - pbh_data['ztf_r'] = np.full(len(data_in_field), np.nan) + pbh_data['ztf_g'] = np.full(len(d_galac), np.nan) + pbh_data['ztf_r'] = np.full(len(d_galac), np.nan) #Calculate the maximum and minimum l and b values for each dataset in the no PBH file, #so that we can determine which datasets to correctly add the PBHs. @@ -3554,11 +3556,7 @@ def qnfw(p, con=5, logp=False): min_b = lat_bin[idx2] lat_long_list.append((min_l, max_l, min_b, max_b)) - """ - WARNING: THIS CODE IS NOW BROKEN AND NEEDS TO BE REWRITTEN - THE NEW DATA FORMAT FOR HDF5 FILES IS COMPOUND DATATYPE - AND THE DATA ONCE READ IS A NUMPY RECARRAY - """ + #Opening the file with no PBHs and creating a new file for the PBHs added. no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') pbh_hdf5_file = h5py.File(output_hdf5_file, 'w') From 2825cc1c4e0c1b64fe0d92117ebc33d12d2aece8 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Fri, 13 Mar 2020 15:42:16 -0700 Subject: [PATCH 039/125] removed r_vir and c. Added r_s. Changed dafault values. --- popsycle/data/pbh_config.yaml | 9 ++- popsycle/run.py | 28 ++++----- popsycle/synthetic.py | 115 +++++++++++++++------------------- 3 files changed, 66 insertions(+), 86 deletions(-) diff --git a/popsycle/data/pbh_config.yaml b/popsycle/data/pbh_config.yaml index abc00d18..b5249cd8 100644 --- a/popsycle/data/pbh_config.yaml +++ b/popsycle/data/pbh_config.yaml @@ -3,9 +3,8 @@ fdm: 1 #Fraction of dark matter that consists of PBHs pbh_mass: 40 #Msun r_max: 8.3 #kpc, How far from the galactic center that you want to go out -c: 12.94 #concentration index -r_vir: 200 #kpc -inner_slope: .5 -v_esc: 550 -rho_0 : 0.0106 +r_s: 18.6 #kpc +gamma: 1 +v_esc: 550 #km/s +rho_0 : 0.0093 #Msun/pc^3 n_lin : 1000 diff --git a/popsycle/run.py b/popsycle/run.py index 64beb15f..cfa1eef0 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -286,7 +286,7 @@ def generate_popsycle_config_file(radius_cut, obs_time, generate_config_file(config_filename, config) -def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, inner_slope, v_esc, rho_0, n_lin): +def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, r_s, gamma, v_esc, rho_0, n_lin): """ Save PBH configuration parameters into a yaml file @@ -298,27 +298,24 @@ def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, in The fraction of dark matter that you want to consist of PBHs. pbh_mass : int - The single mass that all PBHs will have. + The single mass that all PBHs will have (in units of Msun). r_max : float - The maximum radius from the galactic center that you want to find PBHs at. + The maximum radius (in kpc) from the galactic center that you want to find PBHs at. - c : float - Concentration index. + r_s: float + The scale radius of the Milky Way (in units of kpc). r_s = r_vir / c (virial radius / concentration index) - r_vir : float - The virial radius. - - inner_slope: float - The inner slope of the MW halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. - Inner_slope goes into the determination of the velocities and each value returns a slightly different distribution. + gamma: float + The inner slope of the MW dark matter halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. + Gamma goes into the determination of the velocities and each value returns a slightly different distribution. v_esc: int - The escape velocity of the Milky Way. + The escape velocity of the Milky Way in km/s. v_esc is used in calculating the velocities. rho_0: float - The initial density that will be used in the NFW profile equations. + The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). n_lin: int The number of times you want the density determined along the line of sight when calculating PBH positions @@ -332,9 +329,8 @@ def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, c, r_vir, in config = {'fdm': fdm, 'pbh_mass': pbh_mass, 'r_max': r_max, - 'c': c, - 'r_vir': r_vir, - 'inner_slope': inner_slope, + 'r_s': r_s, + 'gamma': gamma, 'v_esc': v_esc, 'rho_0':rho_0, 'n_lin':n_lin} diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index d55ee35f..ea9e9314 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3153,32 +3153,14 @@ def calc_f(lambda_eff): return f - -def angdist(ra1, dec1, ra2, dec2): - ''' - Takes two spherical coordinates and determines the angluar distance between - them. - Input: - ra1,dec1 [degrees] angular coordinates of the first point - ra2,dec2 [degrees] angular coordinates of the second point - Output: - distance [degrees] angular distance between the two points - ''' - from numpy import pi, sin, cos, arccos - d2r = pi / 180.0 - cosdist = (cos(np.absolute(ra1 - ra2) * d2r) * cos(np.absolute(dec1 * d2r)) * cos(np.absolute(dec2 * d2r)) + - sin(np.absolute(dec1 * d2r)) * sin(np.absolute(dec2 * d2r))) - distance = arccos(cosdist) / d2r - return distance - -def rho_dmhalo(r, rho_0, r_s, gamma): +def rho_dmhalo(r, rho_0=.0093, r_s=18.6, gamma=1): """ Density profile of the dark matter halo. We are using the parametrization from McMillan (2017) Equation 5, with defaults based on the mean values in Table 2. r: galactocentric radius [units: kpc] rho_0: characteristic density in [units: m_sun / pc**3] - r_s: scale radius in [units: kpc] + r_s: scale radius in [units: kpc]. gamma: gamma=1 for NFW, gamma > 1 cuspy, gamma < 1 cored returns: density at r [units: m_sun / pc**3] @@ -3189,8 +3171,8 @@ def rho_dmhalo(r, rho_0, r_s, gamma): def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, - r_max=8.3, c=12.94, r_vir=200, inner_slope=.5, v_esc=550, - rho_0=0.0106, n_lin=1000, overwrite=False, seed=None): + r_max=8.3, r_s=18.6, gamma=1, v_esc=550, + rho_0=0.0093, n_lin=1000, overwrite=False, seed=None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, and saves them in a new HDF5 file with the PBHs added. @@ -3217,42 +3199,37 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, Defaults to 1. pbh_mass : int - The single mass that all PBHs will have. + The single mass that all PBHs will have (in units of Msun). Defaults to 40 Msun (from LIGO detections thought to be primordial) r_max : float The maximum radius from the galactic center that you want to find PBHs at. Defaults to 8.3 kpc (Where Earth is located) - c : float - Concentration index. - Defaults to 12.94 (the value given in McMillan 2016, used in the paper we derive the velocities from) - - r_vir : float - The virial radius. - Defaults to 200 kpc (The virial radius of the Milky Way) + r_s: float + The scale radius of the Milky Way (in kpc). r_s = r_vir / c (virial radius / concentration index) + Defaults to 18.6 kpc. The median value given in McMillan 2016 rho_0: float - The initial density that will be used in the NFW profile equations. - Defaults to .0106 from McMillan 2016 + The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). + Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2016. n_lin: int The number of times you want the density determined along the line of sight when calculating PBH positions Defaults to 1000. Will need to make large if you are closer to the galactic center. - - Optional Parameters - ------------------- - inner_slope: float - The inner slope of the MW halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. - Inner_slope goes into the determination of the velocities and each value returns a slightly different distribution. - The default value is .5 because it is in the middle of the options. More investigation is needed. + gamma: float + The inner slope of the MW dark matter halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. + Gamma goes into the determination of the velocities and each value returns a slightly different distribution. + The default value is 1, corresponding to an NFW profile. v_esc: int - The escape velocity of the Milky Way. + The escape velocity of the Milky Way (in km/s). v_esc is used in calculating the velocities. - Default is 550 because most papers cite values of 515-575, with a lot being around 550. + Default is 550 km/s. Most papers cite values of 515-575. + Optional Parameters + ------------------- overwrite : bool If set to True, overwrites output files. If set to False, exists the function if output files are already on disk. @@ -3300,13 +3277,24 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, if type(r_max) != int: raise Exception('r_max must be a float or an integer.') - if type(c) != int: - if type(c) != float: - raise Exception('c must be an integer or a float.') + if type(r_s) != float: + if type(r_s) != int: + raise Exception('r_s must be a float or an integer.') - if type(r_vir) != float: - if type(r_vir) != int: - raise Exception('r_vir must be a float or an integer.') + if type(rho_0) != float: + if type(rho_0) != int: + raise Exception('rho_0 must be a float or an integer.') + + if type(n_lin) != int: + raise Exception('n_lin must be an integer.') + + if type(gamma) != float: + if type(gamma) != int: + raise Exception('gamma must be a float or an integer.') + + if type(v_esc) != int: + if type(v_esc) != float: + raise Exception('v_esc must be an interger or a float.') if seed is not None: if type(seed) != int: @@ -3328,11 +3316,6 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, t0 = time.time() - #Define parameters for NFW profile calculations - r_s = r_vir/c #kpc, scale radius - g = 4.3*(10**-3) #(pc*km^2)/(Msun*s^2) - h = 70 #km/(s*Mpc) - #Read in the hdf5 file that doesn't have PBHs. Product of perform_pop_syn. no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') key_list = list(no_pbh_hdf5_file) @@ -3362,11 +3345,11 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, ebf_log = make_ebf_log(t) #Obtain survey area and center latitude and longitude - b = float(ebf_log['latitude']) - b_radian = b * np.pi / 180 - l = float(ebf_log['longitude']) - l_radian = l * np.pi /180 - surveyArea = float(ebf_log['surveyArea']) + b = float(ebf_log['latitude']) #deg + b_radian = b * np.pi / 180 #rad + l = float(ebf_log['longitude']) #deg + l_radian = l * np.pi /180 #rad + surveyArea = float(ebf_log['surveyArea']) #deg^2 #Calculate the size of the field of view we are running field_of_view_radius = (surveyArea/np.pi)**(1/2) @@ -3391,7 +3374,7 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, # Determine the dark matter density at all galactocentric radii along the line of sight. rho_lin = rho_dmhalo(galactocen_lin.spherical.distance.value, - rho_0=rho_0, r_s=r_s, gamma=inner_slope) + rho_0=rho_0, r_s=r_s, gamma=gamma) # Estimate the total mass within the line-of-sight cylinder [units: M_sun kpc**-2] # Projected density along line of light @@ -3408,7 +3391,6 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, # Total number of black holes to randomly draw n_pbh = int(np.round(fdm * mass_los_cyl / pbh_mass)) - print('Number of PBH to populate LOS cylinder with = {0}'.format(n_pbh)) # Estimate the discrete CDF based on the discrete PDF rho_marg_r_cum = integrate.cumtrapz(y=rho_lin, @@ -3432,10 +3414,6 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, y_cyl = r_cyl * np.sin(theta) # kpc x_cyl = r_cyl * np.cos(theta) # kpc - # Verify correct size - print(np.size(cdf_los)) - print(np.size(galactic_lin.distance.kpc)) - # Mask out sampled PBH outside the observation cone mask_obs_cone = r_cyl <= r_proj_los_cyl * d_galac / (2 * r_max) @@ -3464,9 +3442,9 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, #Inner slope of the MW halo #From Lacroix et al 2018, Figure 11 (top left panel) data_dir = '%s/data' % os.path.dirname(inspect.getfile(add_pbh)) - if inner_slope == 1: + if gamma == 1: vel_data = pd.read_csv('%s/radial_velocity_profile_steep.csv' % data_dir) - elif inner_slope == .25: + elif gamma == .25: vel_data = pd.read_csv('%s/radial_velocity_profile_shallow.csv' % data_dir) else: vel_data = pd.read_csv('%s/radial_velocity_profile_middle.csv' % data_dir) @@ -3550,10 +3528,14 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, lat_long_list = [] for idx in range(len(long_bin) - 1): max_l = long_bin[idx + 1] + print(max_l) min_l = long_bin[idx] + print(min_l) for idx2 in range(len(lat_bin)-1): max_b = lat_bin[idx2 + 1] + print(max_b) min_b = lat_bin[idx2] + print(min_b) lat_long_list.append((min_l, max_l, min_b, max_b)) @@ -3578,8 +3560,11 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, combined_data = key_data else: pbh_data_in_key = pbh_data[mask] + print(len(pbh_data_in_key)) combined_data = np.hstack((key_data, pbh_data_in_key)) + print(len(combined_data)) N_objs_pbh += combined_data.shape[0] + print(N_objs_pbh) _ = pbh_hdf5_file.create_dataset(key, shape=(combined_data.shape[0],), dtype=comp_dtype, From a53a72f46ca584f790ff3a4389986b28044af0dd Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 13 Mar 2020 15:56:26 -0700 Subject: [PATCH 040/125] read_ind by groups of continuous indices --- popsycle/ebf.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/popsycle/ebf.py b/popsycle/ebf.py index e16c2922..ccd98376 100644 --- a/popsycle/ebf.py +++ b/popsycle/ebf.py @@ -259,6 +259,9 @@ import sys import time import os +from itertools import groupby +from operator import itemgetter + __version__ = "0.0.14" @@ -2753,9 +2756,17 @@ def read_ind(self,ind): if numpy.max(ind) Date: Fri, 13 Mar 2020 15:57:34 -0700 Subject: [PATCH 041/125] rename some variables --- popsycle/ebf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/popsycle/ebf.py b/popsycle/ebf.py index ccd98376..5736ec04 100644 --- a/popsycle/ebf.py +++ b/popsycle/ebf.py @@ -2756,9 +2756,9 @@ def read_ind(self,ind): if numpy.max(ind) Date: Fri, 13 Mar 2020 16:59:38 -0700 Subject: [PATCH 042/125] Fixed issue with binning PBHs and getting correct totals --- popsycle/run.py | 2 +- popsycle/synthetic.py | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index cfa1eef0..dce50f09 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -307,7 +307,7 @@ def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, r_s, gamma, The scale radius of the Milky Way (in units of kpc). r_s = r_vir / c (virial radius / concentration index) gamma: float - The inner slope of the MW dark matter halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. + The inner slope of the MW dark matter halo as described in LaCroix 2018. Gamma goes into the determination of the velocities and each value returns a slightly different distribution. v_esc: int diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index ea9e9314..bde10de8 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3208,18 +3208,18 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_s: float The scale radius of the Milky Way (in kpc). r_s = r_vir / c (virial radius / concentration index) - Defaults to 18.6 kpc. The median value given in McMillan 2016 + Defaults to 18.6 kpc. The median value given in McMillan 2017. rho_0: float The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). - Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2016. + Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2017. n_lin: int The number of times you want the density determined along the line of sight when calculating PBH positions Defaults to 1000. Will need to make large if you are closer to the galactic center. gamma: float - The inner slope of the MW dark matter halo as described in https://iopscience.iop.org/article/10.1088/1475-7516/2018/09/040/pdf. + The inner slope of the MW dark matter halo as described in LaCroix 2018. Gamma goes into the determination of the velocities and each value returns a slightly different distribution. The default value is 1, corresponding to an NFW profile. @@ -3416,11 +3416,16 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, # Mask out sampled PBH outside the observation cone mask_obs_cone = r_cyl <= r_proj_los_cyl * d_galac / (2 * r_max) - + print('Number of PBH before and after light cone masking: {0} and {1}, respectively'.format(n_pbh, np.sum(mask_obs_cone))) + # Assuming small angle approximation b_galac = r_cyl * np.sin(theta) / d_galac + b_radian # rad l_galac = r_cyl * np.cos(theta) / np.cos(b_radian) / d_galac + l_radian # rad + d_galac = d_galac[mask_obs_cone] + b_galac = b_galac[mask_obs_cone] + l_galac = l_galac[mask_obs_cone] + latitude = b_galac * (180/np.pi) #degrees longitude = l_galac * (180/np.pi) #degrees @@ -3528,14 +3533,10 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, lat_long_list = [] for idx in range(len(long_bin) - 1): max_l = long_bin[idx + 1] - print(max_l) min_l = long_bin[idx] - print(min_l) for idx2 in range(len(lat_bin)-1): max_b = lat_bin[idx2 + 1] - print(max_b) min_b = lat_bin[idx2] - print(min_b) lat_long_list.append((min_l, max_l, min_b, max_b)) @@ -3560,11 +3561,8 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, combined_data = key_data else: pbh_data_in_key = pbh_data[mask] - print(len(pbh_data_in_key)) combined_data = np.hstack((key_data, pbh_data_in_key)) - print(len(combined_data)) N_objs_pbh += combined_data.shape[0] - print(N_objs_pbh) _ = pbh_hdf5_file.create_dataset(key, shape=(combined_data.shape[0],), dtype=comp_dtype, From d10024aca78a9cfd2110dccf08fe47676dbb4d87 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Sun, 15 Mar 2020 14:17:38 -0700 Subject: [PATCH 043/125] attempt to move kdtree for comp objects outside of random loop --- popsycle/synthetic.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 57bbe714..4b1c64ee 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -475,6 +475,20 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, num_stars_in_bin = 2e6 num_bins = int(math.ceil(len_adx / num_stars_in_bin)) + # Create the KDTree used for calculating extinction from the first + # sample of stars + if len_adx > 0: + num_kdtree_samples = int(min(len_adx, 1e5)) + kdt_idx = np.random.choice(np.arange(len_adx), + size=num_kdtree_samples, + replace=False) + bin_idx = popid_idx[age_idx[kdt_idx]] + star_px = ebf.read_ind(ebf_file, '/px', bin_idx) + star_py = ebf.read_ind(ebf_file, '/py', bin_idx) + star_pz = ebf.read_ind(ebf_file, '/pz', bin_idx) + star_xyz = np.array([star_px, star_py, star_pz]).T + kdt_star_p = cKDTree(star_xyz) + ########## # Loop through bins of 2 million stars at a time. ########## @@ -577,6 +591,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, comp_dict, next_id = _make_comp_dict(iso_dir, age_of_bin, mass_in_bin, stars_in_bin, next_id, + kdt_star_p, BH_kick_speed_mean=BH_kick_speed_mean, NS_kick_speed_mean=NS_kick_speed_mean, additional_photometric_systems=additional_photometric_systems, @@ -598,6 +613,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, ########## del star_dict gc.collect() + del kdt_star_p t1 = time.time() print('Total run time is {0:f} s'.format(t1 - t0)) @@ -806,7 +822,8 @@ def current_initial_ratio(logage, ratio_file, iso_dir, seed=None): return _Mclust_v_age_func(logage) -def _make_comp_dict(iso_dir, log_age, currentClusterMass, star_dict, next_id, +def _make_comp_dict(iso_dir, log_age, currentClusterMass, + star_dict, next_id, kdt_star_p, BH_kick_speed_mean=50, NS_kick_speed_mean=400, additional_photometric_systems=None, seed=None): @@ -830,6 +847,9 @@ def _make_comp_dict(iso_dir, log_age, currentClusterMass, star_dict, next_id, next_id : The next unique ID number (int) that will be assigned to the new compact objects created. + kdt_star_p + #FIXME# + Optional Parameters ------------------- BH_kick_speed_mean : float @@ -1062,15 +1082,10 @@ def _make_comp_dict(iso_dir, log_age, currentClusterMass, star_dict, next_id, lum_WD_idx = np.argwhere(~np.isnan(comp_table['m_ubv_I'])) if len(lum_WD_idx) > 0: - star_xyz = np.array([star_dict['px'], - star_dict['py'], - star_dict['pz']]).T comp_xyz = np.array([comp_dict['px'][lum_WD_idx], comp_dict['py'][lum_WD_idx], comp_dict['pz'][lum_WD_idx]]).T - - kdt = cKDTree(star_xyz) - dist, indices = kdt.query(comp_xyz) + dist, indices = kdt_star_p.query(comp_xyz) comp_dict['exbv'][lum_WD_idx] = star_dict['exbv'][indices.T] From 5f45c2e9794ae93bfab94f110b2a1624010959f1 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Sun, 15 Mar 2020 14:32:12 -0700 Subject: [PATCH 044/125] del variables --- popsycle/synthetic.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 4b1c64ee..84ca2870 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -477,6 +477,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, # Create the KDTree used for calculating extinction from the first # sample of stars + kdt_star_p = None if len_adx > 0: num_kdtree_samples = int(min(len_adx, 1e5)) kdt_idx = np.random.choice(np.arange(len_adx), @@ -488,6 +489,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, star_pz = ebf.read_ind(ebf_file, '/pz', bin_idx) star_xyz = np.array([star_px, star_py, star_pz]).T kdt_star_p = cKDTree(star_xyz) + del bin_idx, star_px, star_py, star_pz ########## # Loop through bins of 2 million stars at a time. From d4d9eef33e629e2011341e5e1d3ece59c683a77f Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 16 Mar 2020 11:44:29 -0700 Subject: [PATCH 045/125] up num_kdtree_samples to 2e6 --- popsycle/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 84ca2870..763f3636 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -479,7 +479,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, # sample of stars kdt_star_p = None if len_adx > 0: - num_kdtree_samples = int(min(len_adx, 1e5)) + num_kdtree_samples = int(min(len_adx, 2e6)) kdt_idx = np.random.choice(np.arange(len_adx), size=num_kdtree_samples, replace=False) From 1b53e2381bcb4113c6e484b796a729aec7e5b9d1 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 16 Mar 2020 16:51:49 -0700 Subject: [PATCH 046/125] assign comp exbv by star_exbv indices --- popsycle/synthetic.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 763f3636..9b6f121a 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -489,6 +489,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, star_pz = ebf.read_ind(ebf_file, '/pz', bin_idx) star_xyz = np.array([star_px, star_py, star_pz]).T kdt_star_p = cKDTree(star_xyz) + kdt_star_exbv = ebf.read_ind(ebf_file, '/exbv_schlegel', bin_idx) del bin_idx, star_px, star_py, star_pz ########## @@ -593,7 +594,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, comp_dict, next_id = _make_comp_dict(iso_dir, age_of_bin, mass_in_bin, stars_in_bin, next_id, - kdt_star_p, + kdt_star_p, kdt_star_exbv, BH_kick_speed_mean=BH_kick_speed_mean, NS_kick_speed_mean=NS_kick_speed_mean, additional_photometric_systems=additional_photometric_systems, @@ -615,7 +616,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, ########## del star_dict gc.collect() - del kdt_star_p + del kdt_star_p, kdt_star_exbv t1 = time.time() print('Total run time is {0:f} s'.format(t1 - t0)) @@ -825,7 +826,8 @@ def current_initial_ratio(logage, ratio_file, iso_dir, seed=None): def _make_comp_dict(iso_dir, log_age, currentClusterMass, - star_dict, next_id, kdt_star_p, + star_dict, next_id, + kdt_star_p, kdt_star_exbv, BH_kick_speed_mean=50, NS_kick_speed_mean=400, additional_photometric_systems=None, seed=None): @@ -852,6 +854,9 @@ def _make_comp_dict(iso_dir, log_age, currentClusterMass, kdt_star_p #FIXME# + kdt_star_exbv + #FIXME# + Optional Parameters ------------------- BH_kick_speed_mean : float @@ -1088,8 +1093,7 @@ def _make_comp_dict(iso_dir, log_age, currentClusterMass, comp_dict['py'][lum_WD_idx], comp_dict['pz'][lum_WD_idx]]).T dist, indices = kdt_star_p.query(comp_xyz) - - comp_dict['exbv'][lum_WD_idx] = star_dict['exbv'][indices.T] + comp_dict['exbv'][lum_WD_idx] = kdt_star_exbv[indices.T] comp_dict['ubv_I'][lum_WD_idx] = comp_table['m_ubv_I'][lum_WD_idx].data comp_dict['ubv_K'][lum_WD_idx] = comp_table['m_ukirt_K'][lum_WD_idx].data From 0e45735db9e69f5650d2da3e4441cd392d7c1c68 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 16 Mar 2020 17:18:40 -0700 Subject: [PATCH 047/125] set kdt_star_exbv to None as default --- popsycle/synthetic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 9b6f121a..2faccfff 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -478,6 +478,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, # Create the KDTree used for calculating extinction from the first # sample of stars kdt_star_p = None + kdt_star_exbv = None if len_adx > 0: num_kdtree_samples = int(min(len_adx, 2e6)) kdt_idx = np.random.choice(np.arange(len_adx), From 34bb7531a68ebf2a818c3a4cb3294612bcfc33fb Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Tue, 17 Mar 2020 15:28:11 -0700 Subject: [PATCH 048/125] few small fixes --- popsycle/synthetic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index bde10de8..9e0a6553 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -43,6 +43,7 @@ import numpy.lib.recfunctions as rfn import shutil from popsycle import utils +import matplotlib.pyplot as plt ########## From d59a44b6147629bf7c7fc662e285f79aa556c794 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Tue, 17 Mar 2020 15:37:28 -0700 Subject: [PATCH 049/125] corrected parameters for add_pbh --- popsycle/run.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index dce50f09..a38603d6 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -760,10 +760,11 @@ def run(): fdm=pbh_config['fdm'], pbh_mass=pbh_config['pbh_mass'], r_max=pbh_config['r_max'], - c=pbh_config['c'], - r_vir=pbh_config['r_vir'], - inner_slope=pbh_config['inner_slope'], + r_s=pbh_config['r_s'], + gamma=pbh_config['gamma'], v_esc=pbh_config['v_esc'], + rho_0=pbh_config['rho_0'], + n_lin=pbh_config['n_lin'], overwrite=args.overwrite, seed=args.seed) From 5d0b0a485e7f3d6b80d6b98811cb7a50b2b5c52e Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Tue, 17 Mar 2020 15:51:25 -0700 Subject: [PATCH 050/125] updated r_max value/documentation --- popsycle/data/pbh_config.yaml | 14 +++++++------- popsycle/run.py | 2 +- popsycle/synthetic.py | 16 ++++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/popsycle/data/pbh_config.yaml b/popsycle/data/pbh_config.yaml index b5249cd8..b4a8fb09 100644 --- a/popsycle/data/pbh_config.yaml +++ b/popsycle/data/pbh_config.yaml @@ -1,10 +1,10 @@ # Example configuration parameters for PBHs -fdm: 1 #Fraction of dark matter that consists of PBHs -pbh_mass: 40 #Msun -r_max: 8.3 #kpc, How far from the galactic center that you want to go out -r_s: 18.6 #kpc -gamma: 1 -v_esc: 550 #km/s -rho_0 : 0.0093 #Msun/pc^3 +fdm: 1 +pbh_mass: 40 +r_max: 16.6 +r_s: 18.6 +gamma: 1 +v_esc: 550 +rho_0 : 0.0093 n_lin : 1000 diff --git a/popsycle/run.py b/popsycle/run.py index a38603d6..0664fd92 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -301,7 +301,7 @@ def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, r_s, gamma, The single mass that all PBHs will have (in units of Msun). r_max : float - The maximum radius (in kpc) from the galactic center that you want to find PBHs at. + The maximum radius (in kpc) from the Earth that you want to find PBHs. r_s: float The scale radius of the Milky Way (in units of kpc). r_s = r_vir / c (virial radius / concentration index) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 9e0a6553..a08e5f3c 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3172,7 +3172,7 @@ def rho_dmhalo(r, rho_0=.0093, r_s=18.6, gamma=1): def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, - r_max=8.3, r_s=18.6, gamma=1, v_esc=550, + r_max=16.6, r_s=18.6, gamma=1, v_esc=550, rho_0=0.0093, n_lin=1000, overwrite=False, seed=None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, @@ -3204,8 +3204,8 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, Defaults to 40 Msun (from LIGO detections thought to be primordial) r_max : float - The maximum radius from the galactic center that you want to find PBHs at. - Defaults to 8.3 kpc (Where Earth is located) + The maximum radius from the Earth that you want to find PBHs. + Defaults to 16.6 kpc. (2 * distance to galactic center) r_s: float The scale radius of the Milky Way (in kpc). r_s = r_vir / c (virial radius / concentration index) @@ -3362,7 +3362,7 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, np.abs(b_radian)<0.5 * np.pi / 180), n_lin<100000): print('Warning: for fields very near the center of the Milky Way it is reocmmended that the number of elements used to estimate the density be n_lin>100000') - r_h_linspace = np.linspace(0, 2*r_max, num=n_lin) + r_h_linspace = np.linspace(0, r_max, num=n_lin) # Represent the line of sight line in galactic coordinates galactic_lin = coord.Galactic(l=l_radian * units.rad, @@ -3380,10 +3380,10 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, # Estimate the total mass within the line-of-sight cylinder [units: M_sun kpc**-2] # Projected density along line of light # (multiply by projected area to get total mass) - rho_marg_r = np.trapz(rho_lin, dx=(2*r_max) / n_lin) * 1000**3 + rho_marg_r = np.trapz(rho_lin, dx=(r_max) / n_lin) * 1000**3 print("Projected density along line-of-sight = {0:0.2e} [M_sun kpc**-2]".format(rho_marg_r)) # LOS cylinder radius in kpc, assuming small angle approximation [units: kpc] - r_proj_los_cyl = field_of_view_radius * np.pi / 180 * (2 * r_max) + r_proj_los_cyl = field_of_view_radius * np.pi / 180 * (r_max) # Projected area of the LOS cylinder [units: kpc**2] area_proj_los_cyl = np.pi * r_proj_los_cyl**2 # Mass within the total cylinder @@ -3396,7 +3396,7 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, # Estimate the discrete CDF based on the discrete PDF rho_marg_r_cum = integrate.cumtrapz(y=rho_lin, x=galactic_lin.distance.kpc, - dx=(2*r_max) / n_lin) + dx=(r_max) / n_lin) cdf_los = rho_marg_r_cum / rho_marg_r_cum[-1] # Since cumtrapz does not include zero for the first element insert it cdf_los = np.insert(cdf_los,0,0) @@ -3416,7 +3416,7 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, x_cyl = r_cyl * np.cos(theta) # kpc # Mask out sampled PBH outside the observation cone - mask_obs_cone = r_cyl <= r_proj_los_cyl * d_galac / (2 * r_max) + mask_obs_cone = r_cyl <= r_proj_los_cyl * d_galac / (r_max) print('Number of PBH before and after light cone masking: {0} and {1}, respectively'.format(n_pbh, np.sum(mask_obs_cone))) # Assuming small angle approximation From 7ec435bf1b890e72d34141b69a23bdf72ab8b5fc Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Tue, 17 Mar 2020 16:40:24 -0700 Subject: [PATCH 051/125] correct load_config_file --- popsycle/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index 445548b9..d1478f0a 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -753,7 +753,7 @@ def run(): Exiting...""".format(args.pbh_config_filename)) sys.exit(1) - pbh_config = synthetic.load_config(args.pbh_config_filename) + pbh_config = load_config_file(args.pbh_config_filename) # Check if .h5 file exists from perform popsyn, use as input for following function if not os.path.exists(filename_dict['hdf5_filename']): From f891741819642967a03648d0b8c6f109e32a049f Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Tue, 17 Mar 2020 19:14:23 -0700 Subject: [PATCH 052/125] add comments on new arguments --- popsycle/synthetic.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index e34618c8..d08fae44 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -978,14 +978,16 @@ def _make_comp_dict(iso_dir, log_age, currentClusterMass, star_dict : dictionary The number of entries for each key is the number of stars. - next_id : The next unique ID number (int) that will be assigned to - the new compact objects created. + next_id : int + The next unique ID number (int) that will be assigned to + the new compact objects created. - kdt_star_p - #FIXME# + kdt_star_p : scipy cKDTree + KDTree constructed from the positions of randomly selected stars + that all share the same popid and log_age. - kdt_star_exbv - #FIXME# + kdt_star_exbv : numpy + Array of galactic extinctions for the stars in kdt_star_p Optional Parameters ------------------- From 607817ad7eac3f49cc7228b487ddf22d026f2287 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Tue, 17 Mar 2020 19:18:19 -0700 Subject: [PATCH 053/125] fix typo --- popsycle/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index d08fae44..b75eb271 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -984,7 +984,7 @@ def _make_comp_dict(iso_dir, log_age, currentClusterMass, kdt_star_p : scipy cKDTree KDTree constructed from the positions of randomly selected stars - that all share the same popid and log_age. + that all share the same popid and similar log_age. kdt_star_exbv : numpy Array of galactic extinctions for the stars in kdt_star_p From 871d623896101033a313fd7b5417faf7e493ae2b Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 18 Mar 2020 11:00:54 -0700 Subject: [PATCH 054/125] add _check_add_pbh --- popsycle/run.py | 40 +++++++++- popsycle/synthetic.py | 165 +++++++++++++++++++++++++++++++----------- 2 files changed, 160 insertions(+), 45 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index 6bc1b0ec..0cd74378 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -17,6 +17,7 @@ from popsycle.synthetic import _check_perform_pop_syn from popsycle.synthetic import _check_calc_events from popsycle.synthetic import _check_refine_events +from popsycle.synthetic import _check_add_pbh def _return_filename_dict(output_root, add_pbh_flag=False): @@ -279,7 +280,10 @@ def generate_popsycle_config_file(radius_cut=2, obs_time=1000, generate_config_file(config_filename, config) -def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, r_s, gamma, v_esc, rho_0, n_lin): +def generate_pbh_config_file(fdm=1, pbh_mass=40, + r_max=16.6, r_s=18.6, gamma=1, + v_esc=550, rho_0=0.0093, n_lin=1000, + config_filename='pbh_config.yaml'): """ Save PBH configuration parameters into a yaml file @@ -325,8 +329,8 @@ def generate_pbh_config_file(config_filename, fdm, pbh_mass, r_max, r_s, gamma, 'r_s': r_s, 'gamma': gamma, 'v_esc': v_esc, - 'rho_0':rho_0, - 'n_lin':n_lin} + 'rho_0': rho_0, + 'n_lin': n_lin} generate_config_file(config_filename, config) @@ -576,6 +580,21 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, additional_photometric_systems=[popsycle_config['photometric_system']], overwrite=overwrite, seed=seed) + if pbh_config_filename is not None: + pbh_config = load_config_file(pbh_config_filename) + _check_add_pbh(hdf5_file='test.h5', + ebf_file='test.ebf', + output_root2=output_root, + fdm=pbh_config['fdm'], + pbh_mass=pbh_config['pbh_mass'], + r_max=pbh_config['r_max'], + r_s=pbh_config['r_s'], + gamma=pbh_config['gamma'], + v_esc=pbh_config['v_esc'], + rho_0=pbh_config['rho_0'], + n_lin=pbh_config['n_lin'], + overwrite=overwrite, + seed=seed) if not skip_calc_events: _check_calc_events(hdf5_file='test.h5', output_root2=output_root, @@ -857,6 +876,21 @@ def run(): additional_photometric_systems=additional_photometric_systems, overwrite=args.overwrite, seed=args.seed) + if add_pbh_flag: + pbh_config = load_config_file(args.pbh_config_filename) + _check_add_pbh(hdf5_file=filename_dict['hdf5_filename'], + ebf_file=filename_dict['ebf_filename'], + output_root2=args.output_root, + fdm=pbh_config['fdm'], + pbh_mass=pbh_config['pbh_mass'], + r_max=pbh_config['r_max'], + r_s=pbh_config['r_s'], + gamma=pbh_config['gamma'], + v_esc=pbh_config['v_esc'], + rho_0=pbh_config['rho_0'], + n_lin=pbh_config['n_lin'], + overwrite=args.overwrite, + seed=args.seed) if not args.skip_calc_events: _check_calc_events(hdf5_file=filename_dict['hdf5_filename'], output_root2=args.output_root, diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 22ea62e8..b82fbe15 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3409,6 +3409,122 @@ def rho_dmhalo(r, rho_0=.0093, r_s=18.6, gamma=1): return rho +def _check_add_pbh(hdf5_file, ebf_file, output_root2, + fdm, pbh_mass, + r_max, r_s, gamma, v_esc, + rho_0, n_lin, overwrite, seed): + """ + Checks that the inputs of add_pbj are valid + + Parameters + ---------- + hdf5_file : str or hdf5 file + str : name of the hdf5 file from the output of perform_pop_syn + + ebf_file : str or ebf file + str : name of the ebf file from Galaxia + ebf file : actually the ebf file from Galaxia + + output_root2 : str + The thing you want the output files to be named + Examples: + 'myout' + '/some/path/to/myout' + '../back/to/some/path/myout' + + fdm : float + Fraction of dark matter. + The fraction of dark matter that you want to consist of PBHs. + Defaults to 1. + + pbh_mass : int + The single mass that all PBHs will have (in units of Msun). + Defaults to 40 Msun (from LIGO detections thought to be primordial) + + r_max : float + The maximum radius from the Earth that you want to find PBHs. + Defaults to 16.6 kpc. (2 * distance to galactic center) + + r_s: float + The scale radius of the Milky Way (in kpc). r_s = r_vir / c (virial radius / concentration index) + Defaults to 18.6 kpc. The median value given in McMillan 2017. + + rho_0: float + The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). + Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2017. + + n_lin: int + The number of times you want the density determined along the line of sight when calculating PBH positions + Defaults to 1000. Will need to make large if you are closer to the galactic center. + + gamma: float + The inner slope of the MW dark matter halo as described in LaCroix 2018. + Gamma goes into the determination of the velocities and each value returns a slightly different distribution. + The default value is 1, corresponding to an NFW profile. + + v_esc: int + The escape velocity of the Milky Way (in km/s). + v_esc is used in calculating the velocities. + Default is 550 km/s. Most papers cite values of 515-575. + + overwrite : bool + If set to True, overwrites output files. If set to False, exists the + function if output files are already on disk. + Default is False. + + seed : int + If set to non-None, all random sampling will be seeded with the + specified seed, forcing identical output for PyPopStar and PopSyCLE. + Default None. + """ + if hdf5_file[-3:] != '.h5': + raise Exception('hdf5_file (%s) must be an ebf file.' % str(hdf5_file)) + + if ebf_file[-4:] != '.ebf': + raise Exception('ebf_file (%s) must be an ebf file.' % str(ebf_file)) + + if type(output_root2) != str: + raise Exception('output_root2 (%s) must be a string.' % str(output_root)) + + if type(fdm) != int: + if type(fdm) != float: + raise Exception('fdm (%s) must be an integer or a float.' % str(fdm)) + + if type(pbh_mass) != int: + if type(pbh_mass) != float: + raise Exception('pbh_mass (%s) must be an integer or a float.' % str(pbh_mass)) + + if type(r_max) != int: + if type(r_max) != float: + raise Exception('r_max (%s) must be an integer or a float.' % str(r_max)) + + if type(r_s) != int: + if type(r_s) != float: + raise Exception('r_s (%s) must be an integer or a float.' % str(r_s)) + + if gamma not in [.25, 1]: + raise Exception('gamma (%s) must be either .25 or 1' % str(gamma)) + + if type(v_esc) != int: + if type(v_esc) != float: + raise Exception('v_esc (%s) must be an integer or a float.' % str(v_esc)) + + if type(rho_0) != int: + if type(rho_0) != float: + raise Exception('rho_0 (%s) must be an integer or a float.' % str(rho_0)) + + if type(n_lin) != int: + raise Exception('n_lin (%s) must be an integerr.' % str(n_lin)) + + if type(overwrite) != bool: + raise Exception('overwrite (%s) must be a boolean.' % str(overwrite)) + + if seed is not None: + if type(seed) != int: + raise Exception('seed (%s) must be None or an integer.' % str(seed)) + + + def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, r_max=16.6, r_s=18.6, gamma=1, v_esc=550, rho_0=0.0093, n_lin=1000, overwrite=False, seed=None): @@ -3489,6 +3605,12 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, # whether input types are correct. ########## + _check_add_pbh(hdf5_file=hdf5_file, ebf_file=ebf_file, + output_root2=output_root2, fdm=fdm, pbh_mass=pbh_mass, + r_max=r_max, r_s=r_s, gamma=gamma, v_esc=v_esc, + rho_0=rho_0, n_lin=n_lin, + overwrite=overwrite, seed=seed) + if not overwrite: # Check if HDF5 file exists already. If it does, throw an error message # to complain and exit. @@ -3497,48 +3619,6 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, 'That .h5 file name is taken! Either delete the .h5 file, ' 'or pick a new name.') - # Error handling/complaining if input types are not right. - if ebf_file[-4:] != '.ebf': - raise Exception('ebf_file must be an ebf file.') - - if type(output_root2) != str: - raise Exception('output_root must be a string.') - - if type(fdm) != float: - if type(fdm) != int: - raise Exception('fdm must be a float or an integer.') - - if type(pbh_mass) != int: - if type(pbh_mass) != float: - raise Exception('pbh_mass must be an integer or a float.') - - if type(r_max) != float: - if type(r_max) != int: - raise Exception('r_max must be a float or an integer.') - - if type(r_s) != float: - if type(r_s) != int: - raise Exception('r_s must be a float or an integer.') - - if type(rho_0) != float: - if type(rho_0) != int: - raise Exception('rho_0 must be a float or an integer.') - - if type(n_lin) != int: - raise Exception('n_lin must be an integer.') - - if type(gamma) != float: - if type(gamma) != int: - raise Exception('gamma must be a float or an integer.') - - if type(v_esc) != int: - if type(v_esc) != float: - raise Exception('v_esc must be an interger or a float.') - - if seed is not None: - if type(seed) != int: - raise Exception('seed must be an integer.') - # Check to make sure that the output hdf5 file # will not overwrite the input hdf5 file output_hdf5_file = '%s.h5' % output_root2 @@ -3546,6 +3626,7 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, raise Exception('Output hdf5 file %s cannot be equal to ' 'input hdf5 file %s' % (output_hdf5_file, hdf5_file)) + ########## # Start of code ######### From 4163ac94dc721d3395cf5ac92420d83970e85572 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 18 Mar 2020 11:04:34 -0700 Subject: [PATCH 055/125] catch gamma not equal to .25, .5 or 1 --- popsycle/synthetic.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index b82fbe15..d73deccd 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3502,8 +3502,8 @@ def _check_add_pbh(hdf5_file, ebf_file, output_root2, if type(r_s) != float: raise Exception('r_s (%s) must be an integer or a float.' % str(r_s)) - if gamma not in [.25, 1]: - raise Exception('gamma (%s) must be either .25 or 1' % str(gamma)) + if gamma not in [.25, .5, 1]: + raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) if type(v_esc) != int: if type(v_esc) != float: @@ -3771,8 +3771,10 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, vel_data = pd.read_csv('%s/radial_velocity_profile_steep.csv' % data_dir) elif gamma == .25: vel_data = pd.read_csv('%s/radial_velocity_profile_shallow.csv' % data_dir) - else: + elif gamma == .5: vel_data = pd.read_csv('%s/radial_velocity_profile_middle.csv' % data_dir) + else: + raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) #Interpolating v values from the above data, given the PBH r values. pbh_vrms = np.interp(pbh_r_galacto, vel_data['r'], vel_data['v']) From e740ae0968ec4c9010f94c93d24bc9821273b588 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 18 Mar 2020 11:18:16 -0700 Subject: [PATCH 056/125] move functions to their own block --- popsycle/synthetic.py | 3831 +++++++++++++++++++++-------------------- 1 file changed, 1932 insertions(+), 1899 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index d73deccd..e336898d 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1361,2550 +1361,2583 @@ def _bin_lb_hdf5(lat_bin_edges, long_bin_edges, obj_arr, output_root): ############################################################################ -########### Candidate event calculation and associated functions ########### +########### Primordial black hole injection and associated functions ####### ############################################################################ -def _check_calc_events(hdf5_file, output_root2, - radius_cut, obs_time, n_obs, theta_frac, - blend_rad, n_proc, overwrite): + +def rho_dmhalo(r, rho_0=.0093, r_s=18.6, gamma=1): """ - Checks that the inputs of calc_events are valid + Density profile of the dark matter halo. + We are using the parametrization from McMillan (2017) Equation 5, + with defaults based on the mean values in Table 2. + r: galactocentric radius [units: kpc] + rho_0: characteristic density in [units: m_sun / pc**3] + r_s: scale radius in [units: kpc]. + gamma: gamma=1 for NFW, gamma > 1 cuspy, gamma < 1 cored + + returns: density at r [units: m_sun / pc**3] + """ + x = r / r_s + rho = rho_0 / (x ** gamma * (1 + x) ** (3 - gamma)) + return rho + + +def _check_add_pbh(hdf5_file, ebf_file, output_root2, + fdm, pbh_mass, + r_max, r_s, gamma, v_esc, + rho_0, n_lin, overwrite, seed): + """ + Checks that the inputs of add_pbj are valid Parameters ---------- - hdf5_file : str - Name of the HDF5 file. + hdf5_file : str or hdf5 file + str : name of the hdf5 file from the output of perform_pop_syn + + ebf_file : str or ebf file + str : name of the ebf file from Galaxia + ebf file : actually the ebf file from Galaxia output_root2 : str - The name for the h5 file + The thing you want the output files to be named + Examples: + 'myout' + '/some/path/to/myout' + '../back/to/some/path/myout' - radius_cut : float - Initial radius cut, in ARCSECONDS. + fdm : float + Fraction of dark matter. + The fraction of dark matter that you want to consist of PBHs. + Defaults to 1. - obs_time : float - Survey duration, in DAYS. + pbh_mass : int + The single mass that all PBHs will have (in units of Msun). + Defaults to 40 Msun (from LIGO detections thought to be primordial) - n_obs : int - Number of observations. + r_max : float + The maximum radius from the Earth that you want to find PBHs. + Defaults to 16.6 kpc. (2 * distance to galactic center) - theta_frac : float - Another cut, in multiples of Einstein radii. + r_s: float + The scale radius of the Milky Way (in kpc). r_s = r_vir / c (virial radius / concentration index) + Defaults to 18.6 kpc. The median value given in McMillan 2017. - blend_rad : float - Stars within this distance of the lens are said to be blended. - Units are in ARCSECONDS. + rho_0: float + The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). + Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2017. - n_proc : int - Number of processors to use. Should not exceed the number of cores. - Default is one processor (no parallelization). + n_lin: int + The number of times you want the density determined along the line of sight when calculating PBH positions + Defaults to 1000. Will need to make large if you are closer to the galactic center. + + gamma: float + The inner slope of the MW dark matter halo as described in LaCroix 2018. + Gamma goes into the determination of the velocities and each value returns a slightly different distribution. + The default value is 1, corresponding to an NFW profile. + + v_esc: int + The escape velocity of the Milky Way (in km/s). + v_esc is used in calculating the velocities. + Default is 550 km/s. Most papers cite values of 515-575. overwrite : bool - If set to True, overwrites output files. If set to False, exits the + If set to True, overwrites output files. If set to False, exists the function if output files are already on disk. Default is False. + + seed : int + If set to non-None, all random sampling will be seeded with the + specified seed, forcing identical output for PyPopStar and PopSyCLE. + Default None. """ if hdf5_file[-3:] != '.h5': - raise Exception('hdf5_file (%s) must be an h5 file.' % str(hdf5_file)) + raise Exception('hdf5_file (%s) must be an ebf file.' % str(hdf5_file)) + + if ebf_file[-4:] != '.ebf': + raise Exception('ebf_file (%s) must be an ebf file.' % str(ebf_file)) if type(output_root2) != str: - raise Exception('output_root2 (%s) must be a string.' % str(output_root2)) + raise Exception( + 'output_root2 (%s) must be a string.' % str(output_root)) - if type(radius_cut) != int: - if type(radius_cut) != float: - raise Exception('radius_cut (%s) must be an integer or a float.' % str(radius_cut)) + if type(fdm) != int: + if type(fdm) != float: + raise Exception( + 'fdm (%s) must be an integer or a float.' % str(fdm)) - if type(obs_time) != int: - if type(obs_time) != float: - raise Exception('obs_time (%s) must be an integer or a float.' % str(obs_time)) + if type(pbh_mass) != int: + if type(pbh_mass) != float: + raise Exception( + 'pbh_mass (%s) must be an integer or a float.' % str(pbh_mass)) - if type(blend_rad) != int: - if type(blend_rad) != float: - raise Exception('blend_rad (%s) must be an integer or a float.' % str(blend_rad)) + if type(r_max) != int: + if type(r_max) != float: + raise Exception( + 'r_max (%s) must be an integer or a float.' % str(r_max)) - if type(n_obs) != int: - raise Exception('n_obs (%s) must be an integer.' % str(n_obs)) + if type(r_s) != int: + if type(r_s) != float: + raise Exception( + 'r_s (%s) must be an integer or a float.' % str(r_s)) - if type(n_proc) != int: - raise Exception('n_proc (%s) must be an integer.' % str(n_proc)) + if gamma not in [.25, .5, 1]: + raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) + + if type(v_esc) != int: + if type(v_esc) != float: + raise Exception( + 'v_esc (%s) must be an integer or a float.' % str(v_esc)) + + if type(rho_0) != int: + if type(rho_0) != float: + raise Exception( + 'rho_0 (%s) must be an integer or a float.' % str(rho_0)) + + if type(n_lin) != int: + raise Exception('n_lin (%s) must be an integerr.' % str(n_lin)) if type(overwrite) != bool: raise Exception('overwrite (%s) must be a boolean.' % str(overwrite)) - if type(theta_frac) != int: - if type(theta_frac) != float: - raise Exception('theta_frac (%s) must be an integer or a float.' % str(theta_frac)) + if seed is not None: + if type(seed) != int: + raise Exception( + 'seed (%s) must be None or an integer.' % str(seed)) -def calc_events(hdf5_file, output_root2, - radius_cut=2, obs_time=1000, n_obs=101, theta_frac=2, - blend_rad=0.65, n_proc=1, - overwrite=False): +def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, + r_max=16.6, r_s=18.6, gamma=1, v_esc=550, + rho_0=0.0093, n_lin=1000, overwrite=False, seed=None): """ - Calculate microlensing events + Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, + and saves them in a new HDF5 file with the PBHs added. Parameters ---------- - hdf5_file : str - Name of the HDF5 file. + hdf5_file : str or hdf5 file + str : name of the hdf5 file from the output of perform_pop_syn + + ebf_file : str or ebf file + str : name of the ebf file from Galaxia + ebf file : actually the ebf file from Galaxia output_root2 : str - The name for the h5 file + The thing you want the output files to be named + Examples: + 'myout' + '/some/path/to/myout' + '../back/to/some/path/myout' - radius_cut : float - Initial radius cut, in ARCSECONDS. + fdm : float + Fraction of dark matter. + The fraction of dark matter that you want to consist of PBHs. + Defaults to 1. - obs_time : float - Survey duration, in DAYS. + pbh_mass : int + The single mass that all PBHs will have (in units of Msun). + Defaults to 40 Msun (from LIGO detections thought to be primordial) - n_obs : int - Number of observations. + r_max : float + The maximum radius from the Earth that you want to find PBHs. + Defaults to 16.6 kpc. (2 * distance to galactic center) - theta_frac : float - Another cut, in multiples of Einstein radii. + r_s: float + The scale radius of the Milky Way (in kpc). r_s = r_vir / c (virial radius / concentration index) + Defaults to 18.6 kpc. The median value given in McMillan 2017. - blend_rad : float - Stars within this distance of the lens are said to be blended. - Units are in ARCSECONDS. + rho_0: float + The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). + Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2017. + + n_lin: int + The number of times you want the density determined along the line of sight when calculating PBH positions + Defaults to 1000. Will need to make large if you are closer to the galactic center. + + gamma: float + The inner slope of the MW dark matter halo as described in LaCroix 2018. + Gamma goes into the determination of the velocities and each value returns a slightly different distribution. + The default value is 1, corresponding to an NFW profile. + + v_esc: int + The escape velocity of the Milky Way (in km/s). + v_esc is used in calculating the velocities. + Default is 550 km/s. Most papers cite values of 515-575. Optional Parameters ------------------- - n_proc : int - Number of processors to use. Should not exceed the number of cores. - Default is one processor (no parallelization). - overwrite : bool If set to True, overwrites output files. If set to False, exists the function if output files are already on disk. Default is False. + seed : int + If set to non-None, all random sampling will be seeded with the + specified seed, forcing identical output for PyPopStar and PopSyCLE. + Default None. - Output - ------ - _events.fits : Astropy .fits table - Table of candidate microlensing events. The number of rows - corresponds to the number of candidate events. - + Outputs + ------- + .h5 : hdf5 file + The new .h5 file with PBHs injected in. """ - ########## # Error handling: check whether files exist and # whether input types are correct. ########## - # Check if .fits file exists already. If it does, throw an error message - # to complain and exit. + _check_add_pbh(hdf5_file=hdf5_file, ebf_file=ebf_file, + output_root2=output_root2, fdm=fdm, pbh_mass=pbh_mass, + r_max=r_max, r_s=r_s, gamma=gamma, v_esc=v_esc, + rho_0=rho_0, n_lin=n_lin, + overwrite=overwrite, seed=seed) + if not overwrite: - if os.path.isfile(output_root2 + '_events.fits'): - raise Exception( - 'That events.fits file name is taken! Either delete the .fits ' - 'file, or pick a new name.') - if os.path.isfile(output_root2 + '_blends.fits'): + # Check if HDF5 file exists already. If it does, throw an error message + # to complain and exit. + if os.path.isfile(output_root2 + '.h5'): raise Exception( - 'That blends.fits file name is taken! Either delete the .fits ' - 'file, or pick a new name.') + 'That .h5 file name is taken! Either delete the .h5 file, ' + 'or pick a new name.') - # Error handling/complaining if input types are not right. - _check_calc_events(hdf5_file, output_root2, - radius_cut, obs_time, n_obs, theta_frac, - blend_rad, n_proc, overwrite) + # Check to make sure that the output hdf5 file + # will not overwrite the input hdf5 file + output_hdf5_file = '%s.h5' % output_root2 + if hdf5_file == output_hdf5_file: + raise Exception('Output hdf5 file %s cannot be equal to ' + 'input hdf5 file %s' % (output_hdf5_file, hdf5_file)) ########## # Start of code ######### - t0 = time.time() - - # Initialize events_tmp and blends_tmp. - events_tmp = None - blends_tmp = None - - # Get the l and b from the HDF5 file. - hf = h5py.File(hdf5_file, 'r') - l_array = np.array(hf['long_bin_edges']) - b_array = np.array(hf['lat_bin_edges']) - hf.close() + # Set random seed + np.random.seed(seed) - # Converts radius_cut from arcseconds into milliarcseconds - radius_cut *= 1000.0 + t0 = time.time() - # Set up the multiprocessing - pool = Pool(n_proc) + # Read in the hdf5 file that doesn't have PBHs. Product of perform_pop_syn. + no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') + key_list = list(no_pbh_hdf5_file) + # Delete lat_bin_edges and long_bin_edges from key_list. + key_list = [key for key in key_list if 'bin_edges' not in key] - # Set up inputs to be able to be read by pool.map - nll = len(l_array[:]) - 2 - nbb = len(b_array[:]) - 2 + # Get data from lat_bin_edges and long_bin_edges + lat_bin = no_pbh_hdf5_file['lat_bin_edges'][:] + long_bin = no_pbh_hdf5_file['long_bin_edges'][:] + bin_edges_number = len(long_bin) - llbb = itertools.product(range(nll), range(nbb)) + # Getting the maximum ID from all of the stars and compact objects. + # Later used to set the IDs of the PBHs. + max_id_no_pbh = [] + for key in key_list: + # max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key][20])) + max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key]['obj_id'])) + max_id = np.amax(max_id_no_pbh) - reps = nll * nbb + hdf5_dset_names = no_pbh_hdf5_file[key_list[0]][:].dtype.names - hd = itertools.repeat(hdf5_file, reps) - ot = itertools.repeat(obs_time, reps) - no = itertools.repeat(n_obs, reps) - rc = itertools.repeat(radius_cut, reps) - tf = itertools.repeat(theta_frac, reps) - br = itertools.repeat(blend_rad, reps) + no_pbh_hdf5_file.close() - inputs = zip(llbb, hd, ot, no, rc, tf, br) + # Read in ebf file + t = ebf.read_ind(ebf_file, '/log', 0) + # Convert log to useful dictionary. + ebf_log = make_ebf_log(t) - ########## - # Loop through galactic latitude and longitude bins. For each bin vertex, - # take the nearest 4 bin samples and calculate microlensing events. - # We do this to properly handle bin edges - # (i.e. a sliding window analysis of 2x2 bins). - # Duplicate events are removed. - ########## - # Should I use starmap_async? - results = pool.starmap(_calc_event_time_loop, inputs) + # Obtain survey area and center latitude and longitude + b = float(ebf_log['latitude']) # deg + b_radian = b * np.pi / 180 # rad + l = float(ebf_log['longitude']) # deg + l_radian = l * np.pi / 180 # rad + surveyArea = float(ebf_log['surveyArea']) # deg^2 - pool.close() - pool.join() + # Calculate the size of the field of view we are running + field_of_view_radius = (surveyArea / np.pi) ** (1 / 2) - # Remove all the None values - # (occurs for patches with less than 10 objects) - results = [i for i in results if i is not None] + # Generate an array of heliocentric radii + # These radii will just be used to numerically integrate the density + n_lin = 1000 + if np.logical_and(np.logical_and(np.abs(l_radian) < 0.5 * np.pi / 180, + np.abs(b_radian) < 0.5 * np.pi / 180), + n_lin < 100000): + print( + 'Warning: for fields very near the center of the Milky Way it is reocmmended that the number of elements used to estimate the density be n_lin>100000') + r_h_linspace = np.linspace(0, r_max, num=n_lin) - results_ev = [] - results_bl = [] + # Represent the line of sight line in galactic coordinates + galactic_lin = coord.Galactic(l=l_radian * units.rad, + b=b_radian * units.rad, + distance=r_h_linspace * units.kpc) - for ii in range(len(results)): - if results[ii] is not None: - if results[ii][0] is not None: - results_ev.append(results[ii][0]) - if results[ii][1] is not None: - results_bl.append(results[ii][1]) + # convert the line of sight to galactocentric coordinates + # outputs l, b, and distance in degrees. + galactocen_lin = galactic_lin.transform_to( + coord.Galactocentric(representation_type='spherical')) - if len(results_ev) == 0: - print('No events!') - return - else: - events_tmp = np.concatenate(results_ev, axis=0) - if len(results_bl) == 0: - blends_tmp = np.array([]) - else: - blends_tmp = np.concatenate(results_bl, axis=0) + # Determine the dark matter density at all galactocentric radii along the line of sight. + rho_lin = rho_dmhalo(galactocen_lin.spherical.distance.value, + rho_0=rho_0, r_s=r_s, gamma=gamma) - # Convert the events numpy recarray into an - # Astropy Table for easier consumption. - events_tmp = unique_events(events_tmp) - events_final = Table(events_tmp) - N_events = len(events_final) - print('Candidate events detected: ', N_events) + # Estimate the total mass within the line-of-sight cylinder [units: M_sun kpc**-2] + # Projected density along line of light + # (multiply by projected area to get total mass) + rho_marg_r = np.trapz(rho_lin, dx=(r_max) / n_lin) * 1000 ** 3 + print( + "Projected density along line-of-sight = {0:0.2e} [M_sun kpc**-2]".format( + rho_marg_r)) + # LOS cylinder radius in kpc, assuming small angle approximation [units: kpc] + r_proj_los_cyl = field_of_view_radius * np.pi / 180 * (r_max) + # Projected area of the LOS cylinder [units: kpc**2] + area_proj_los_cyl = np.pi * r_proj_los_cyl ** 2 + # Mass within the total cylinder + mass_los_cyl = rho_marg_r * area_proj_los_cyl + print("Mass within line-of-sight cylinder = {0:0.2e} [M_sun]".format( + mass_los_cyl)) - if len(results_bl) != 0: - blends_tmp = unique_blends(blends_tmp) - blends_final = Table(blends_tmp) + # Total number of black holes to randomly draw + n_pbh = int(np.round(fdm * mass_los_cyl / pbh_mass)) - # Save out file - events_final.write(output_root2 + '_events.fits', overwrite=overwrite) - blends_final.write(output_root2 + '_blends.fits', overwrite=overwrite) + # Estimate the discrete CDF based on the discrete PDF + rho_marg_r_cum = integrate.cumtrapz(y=rho_lin, + x=galactic_lin.distance.kpc, + dx=(r_max) / n_lin) + cdf_los = rho_marg_r_cum / rho_marg_r_cum[-1] + # Since cumtrapz does not include zero for the first element insert it + cdf_los = np.insert(cdf_los, 0, 0) - t1 = time.time() + # Create a function to interpolate the CDF so that we can randomly sample from it + f_cdf_d = interpolate.interp1d(cdf_los, galactic_lin.distance.kpc) - ########## - # Make log file - ########## - now = datetime.datetime.now() - radius_cut = radius_cut / 1000.0 # back to arcsec - microlens_path = os.path.dirname(inspect.getfile(perform_pop_syn)) - microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=microlens_path).decode('ascii').strip() - dash_line = '-----------------------------' + '\n' - empty_line = '\n' - line0 = 'FUNCTION INPUT PARAMETERS' + '\n' - line1 = 'hdf5_file , ' + hdf5_file + '\n' - line2 = 'output_root2 , ' + output_root2 + '\n' - line3 = 'radius_cut , ' + str(radius_cut) + ' , (arcsec)' + '\n' - line4 = 'obs_time , ' + str(obs_time) + ' , (days)' + '\n' - line5 = 'n_obs , ' + str(n_obs) + '\n' - line6 = 'theta_frac , ' + str(theta_frac) + ' , (thetaE)' + '\n' - line7 = 'blend_rad , ' + str(blend_rad) + ' , (arcsec)' + '\n' - line8 = 'n_proc , ' + str(n_proc) + '\n' - line9 = 'VERSION INFORMATION' + '\n' - line10 = str(now) + ' : creation date' + '\n' - line11 = microlens_hash + ' : microlens commit' + '\n' + # Randomly sample galactic coordinates for the PBHs based on CDF + d_galac = f_cdf_d(np.random.uniform(size=n_pbh)) - line12 = 'OTHER INFORMATION' + '\n' - line13 = str(t1 - t0) + ' : total runtime (s)' + '\n' - line14 = str(N_events) + ' : total number of events' + '\n' + # Randomly assign a l & b galactic coordinate to each PBH, within the LOS cone + # sample the angle from 0 to 2pi + theta = np.random.uniform(size=n_pbh) * 2 * np.pi + # sample radius correcting for annular area to make uniform + r_cyl = r_proj_los_cyl * np.sqrt(np.random.uniform(size=n_pbh)) # kpc + y_cyl = r_cyl * np.sin(theta) # kpc + x_cyl = r_cyl * np.cos(theta) # kpc - line15 = 'FILES CREATED' + '\n' - line16 = output_root2 + '_events.fits : events file' + '\n' - line17 = output_root2 + '_blends.fits : blends file' + '\n' + # Mask out sampled PBH outside the observation cone + mask_obs_cone = r_cyl <= r_proj_los_cyl * d_galac / (r_max) + print( + 'Number of PBH before and after light cone masking: {0} and {1}, respectively'.format( + n_pbh, np.sum(mask_obs_cone))) - with open(output_root2 + '_calc_events.log', 'w') as out: - out.writelines([line0, dash_line, line1, line2, line3, - line4, line5, line6, line7, line8, empty_line, - line9, dash_line, line10, line11, empty_line, - line12, dash_line, line13, line14, empty_line, line15, - dash_line, line16, line17]) + # Assuming small angle approximation + b_galac = r_cyl * np.sin(theta) / d_galac + b_radian # rad + l_galac = r_cyl * np.cos(theta) / np.cos( + b_radian) / d_galac + l_radian # rad - print('Total runtime: {0:f} s'.format(t1 - t0)) + d_galac = d_galac[mask_obs_cone] + b_galac = b_galac[mask_obs_cone] + l_galac = l_galac[mask_obs_cone] - return + latitude = b_galac * (180 / np.pi) # degrees + longitude = l_galac * (180 / np.pi) # degrees + N_PBHs_in_field = len(d_galac) + print('%i PBHs in the field' % N_PBHs_in_field) -def _calc_event_time_loop(llbb, hdf5_file, obs_time, n_obs, radius_cut, - theta_frac, blend_rad): - """ - Parameters - ---------- - llbb : (int, int) - Indices of (l,b) bin. + if N_PBHs_in_field == 0: + print('-- No PBHs in the field') + print('-- Copying %s to %s' % (hdf5_file, output_hdf5_file)) + shutil.copy(hdf5_file, output_hdf5_file) + return - obs_time, n_obs, radius_cut, theta_frac, blend_rad - are all parameters of calc_events() + # Converting the PBH positions from the field of view back to galactocentric for determining velocities. + galactic_pbh = coord.Galactic(l=longitude * units.deg, + b=latitude * units.deg, + distance=d_galac * units.kpc) + galacto_pbh = galactic_pbh.transform_to( + coord.Galactocentric(representation_type='spherical')) + cart_pbh = astropy.coordinates.cartesian_to_spherical(galacto_pbh.x, + galacto_pbh.y, + galacto_pbh.z) + pbh_r_galacto = cart_pbh[0] - Return - ------ - events_llbb : array - Array of the unique events for the particular (l,b) patch. + # Inner slope of the MW halo + # From Lacroix et al 2018, Figure 11 (top left panel) + data_dir = '%s/data' % os.path.dirname(inspect.getfile(add_pbh)) + if gamma == 1: + vel_data = pd.read_csv( + '%s/radial_velocity_profile_steep.csv' % data_dir) + elif gamma == .25: + vel_data = pd.read_csv( + '%s/radial_velocity_profile_shallow.csv' % data_dir) + elif gamma == .5: + vel_data = pd.read_csv( + '%s/radial_velocity_profile_middle.csv' % data_dir) + else: + raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) - blends_llbb : array - Array of the unique blends for the particular (l,b) patch. + # Interpolating v values from the above data, given the PBH r values. + pbh_vrms = np.interp(pbh_r_galacto, vel_data['r'], vel_data['v']) + v_vals = np.arange(0, v_esc) # Goes from v to v_esc + a = (1 / 2) * pbh_vrms * ((np.pi / 2) ** (1 / 2)) - """ - #################### - # Loop through different time steps and figure out separations between - # all possible pairs of stars. Trim down to "events", which consist of - # those pairs that approach within one of each other. - # These will be the events we consider as candidate microlensing events. - #################### + # Calculating the v_rms velocities for the PBHs by randomly sampling from the CDF. + rand_cdf = np.array([]) - # Initialize events_llbb and blends_llbb. - events_llbb = None - blends_llbb = None + for a_val in a: + cdf = scipy.special.erf(v_vals / (a_val * 2 ** (1 / 2))) - ( + ((2 / np.pi) ** (1 / 2)) * ((v_vals * np.exp( + -v_vals ** 2 / 2 * a_val ** 2)) / a_val)) + rand_cdf = np.append(rand_cdf, np.random.uniform(0, np.amax(cdf))) + interpreted_rms_velocities = np.interp(rand_cdf, cdf, v_vals) - ll = llbb[0] - bb = llbb[1] + # Sampling random latitude and longitude values for velocity to complete the spherical velocities. + sin_lat_vel = np.random.uniform(-1, 1, len(d_galac)) + lat_vel = np.arcsin(sin_lat_vel) + long_vel = np.random.uniform(0, 2 * np.pi, len(d_galac)) - print('Working on loop ll, bb = ', ll, bb) - name00 = 'l' + str(ll) + 'b' + str(bb) - name01 = 'l' + str(ll) + 'b' + str(bb + 1) - name10 = 'l' + str(ll + 1) + 'b' + str(bb) - name11 = 'l' + str(ll + 1) + 'b' + str(bb + 1) + # Transforming velocities to cartesian to get vx, vy, and vz. + cart_vel = astropy.coordinates.spherical_to_cartesian( + interpreted_rms_velocities, lat_vel, long_vel) - hf = h5py.File(hdf5_file, 'r') - bigpatch = np.hstack((hf[name00], hf[name01], hf[name10], hf[name11])) - hf.close() + # Load up a numpy array + comp_dtype = _generate_comp_dtype(hdf5_dset_names) + pbh_data = np.empty(len(d_galac), dtype=comp_dtype) - # Skip patches with less than 10 objects - if len(bigpatch) < 10: - # continue - return + # Getting longitude into the right format for PopSyCLE + longitude = np.where(longitude > 180, longitude - 360, longitude) - time_array = np.linspace(-1 * obs_time / 2.0, obs_time / 2.0, n_obs) + pbh_data['rad'] = d_galac + pbh_data['glon'] = longitude + pbh_data['glat'] = latitude - for i in np.arange(len(time_array)): - # Find potential lenses and sources that fall within radius cut. - lens_id, sorc_id, r_t, sep, event_id1, c = _calc_event_cands_radius(bigpatch, - time_array[i], - radius_cut) + pbh_data['vx'] = cart_vel[0] + pbh_data['vy'] = cart_vel[1] + pbh_data['vz'] = cart_vel[2] - # Calculate einstein radius and lens-source separation - theta_E = einstein_radius(bigpatch['mass'][lens_id], - r_t[lens_id], r_t[sorc_id]) # mas - u = sep[event_id1] / theta_E + # Getting the rest of the PBH data for the combined .h5 file + pbh_data['mass'] = np.full(len(d_galac), pbh_mass) + pbh_data['zams_mass'] = np.full(len(d_galac), pbh_mass) + pbh_data['age'] = np.full(len(d_galac), np.nan) + pbh_data['popid'] = np.full(len(d_galac), 10) + pbh_data['rem_id'] = np.full(len(d_galac), 104) - # Trim down to those microlensing events that really get close enough - # to hope that we can detect them. Trim on a Theta_E criteria. - event_lbt = _calc_event_cands_thetaE(bigpatch, theta_E, u, theta_frac, - lens_id, sorc_id, time_array[i]) + cart_helio = astropy.coordinates.spherical_to_cartesian(d_galac, b_galac, + l_galac) + pbh_data['px'] = cart_helio[0] + pbh_data['py'] = cart_helio[1] + pbh_data['pz'] = cart_helio[2] - if event_lbt is not None: - # Concatenate the current event table - # (at this l, b, time) with the rest. - if events_llbb is not None: - events_llbb = np.hstack((events_llbb, event_lbt)) - else: - events_llbb = event_lbt + vr, mu_b, mu_lcosb = calc_sph_motion(pbh_data['vx'], + pbh_data['vy'], + pbh_data['vz'], + d_galac, b_galac, l_galac) + pbh_data['vr'] = vr + pbh_data['mu_b'] = mu_b + pbh_data['mu_lcosb'] = mu_lcosb + pbh_data['obj_id'] = np.arange((max_id + 1), (max_id + len(d_galac) + 1)) - # Keep only unique events within our different time stamps - events_llbb = unique_events(events_llbb) + pbh_data['exbv'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_K'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_J'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_I'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_U'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_R'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_B'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_H'] = np.full(len(d_galac), np.nan) + pbh_data['ubv_V'] = np.full(len(d_galac), np.nan) + pbh_data['teff'] = np.full(len(d_galac), np.nan) + pbh_data['grav'] = np.full(len(d_galac), np.nan) + pbh_data['mbol'] = np.full(len(d_galac), np.nan) + pbh_data['feh'] = np.full(len(d_galac), np.nan) + if any(['ztf' in n for n in hdf5_dset_names]): + pbh_data['ztf_g'] = np.full(len(d_galac), np.nan) + pbh_data['ztf_r'] = np.full(len(d_galac), np.nan) - ######### - # Get blending. - # Note 1: We are centering on the lens. - # Note 2: We don't want to include the lens itself, - # or the source, in the table. - ########## - blends_lbt = _calc_blends(bigpatch, c, event_lbt, blend_rad) + # Calculate the maximum and minimum l and b values for each dataset in the no PBH file, + # so that we can determine which datasets to correctly add the PBHs. + lat_long_list = [] + for idx in range(len(long_bin) - 1): + max_l = long_bin[idx + 1] + min_l = long_bin[idx] + for idx2 in range(len(lat_bin) - 1): + max_b = lat_bin[idx2 + 1] + min_b = lat_bin[idx2] + lat_long_list.append((min_l, max_l, min_b, max_b)) - if blends_lbt is not None: - # Concatenate the current blend table (at this l, b, time) - # with the rest. - if blends_llbb is not None: - blends_llbb = np.hstack((blends_llbb, blends_lbt)) - else: - blends_llbb = blends_lbt + # Opening the file with no PBHs and creating a new file for the PBHs added. + no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') + pbh_hdf5_file = h5py.File(output_hdf5_file, 'w') - # Keep only unique blends within our different time stamps - blends_llbb = unique_blends(blends_llbb) + # Appending the PBH data to the no PBH data and writing to the new .h5 file. + N_objs_no_pbh = 0 + N_objs_pbh = 0 + for idx, key in enumerate(key_list): + key_data = no_pbh_hdf5_file[key][:] + N_objs_no_pbh += key_data.shape[0] - # END of time loop + min_l, max_l, min_b, max_b = lat_long_list[idx] + mask = (pbh_data['glon'] >= min_l) & \ + (pbh_data['glon'] <= max_l) & \ + (pbh_data['glat'] >= min_b) & \ + (pbh_data['glat'] <= max_b) - return events_llbb, blends_llbb + if np.sum(mask) == 0: + combined_data = key_data + else: + pbh_data_in_key = pbh_data[mask] + combined_data = np.hstack((key_data, pbh_data_in_key)) + N_objs_pbh += combined_data.shape[0] + _ = pbh_hdf5_file.create_dataset(key, + shape=(combined_data.shape[0],), + dtype=comp_dtype, + data=combined_data) + _ = pbh_hdf5_file.create_dataset('lat_bin_edges', (len(lat_bin), 1), + data=lat_bin) + _ = pbh_hdf5_file.create_dataset('long_bin_edges', (len(lat_bin), 1), + data=long_bin) + no_pbh_hdf5_file.close() + pbh_hdf5_file.close() + print('Checking totals') + print('-- %i original objects' % N_objs_no_pbh) + print('-- %i PBHs in the field' % N_PBHs_in_field) + print('-- %i new total objects' % N_objs_pbh) -def _calc_event_cands_radius(bigpatch, timei, radius_cut): + if N_objs_pbh == N_objs_no_pbh + N_PBHs_in_field: + print('-- Totals match!') + else: + print('** MISSING PBHs!! **') + + t1 = time.time() + print('Total runtime: {0:f} s'.format(t1 - t0)) + + return + + +############################################################################ +########### Candidate event calculation and associated functions ########### +############################################################################ + +def _check_calc_events(hdf5_file, output_root2, + radius_cut, obs_time, n_obs, theta_frac, + blend_rad, n_proc, overwrite): """ - Get sources and lenses that pass the radius cut. + Checks that the inputs of calc_events are valid Parameters ---------- - bigpatch : array - Compilation of 4 .h5 datasets containing stars. + hdf5_file : str + Name of the HDF5 file. - timei : float - Time at which to evaluate. + output_root2 : str + The name for the h5 file radius_cut : float - Parameter of calc_events(). - Converted to mas + Initial radius cut, in ARCSECONDS. - Return - ------ - lens_id : array - Indices into bigpatch that indicate lenses + obs_time : float + Survey duration, in DAYS. - sorc_id : array - Indices into bigpatch that indicate sources + n_obs : int + Number of observations. - r_t : array - Radial coordinates for all objects in bigpatch at time t + theta_frac : float + Another cut, in multiples of Einstein radii. - sep : array - Separation between lens-source pairs (mas) + blend_rad : float + Stars within this distance of the lens are said to be blended. + Units are in ARCSECONDS. - event_id1 : array - Lens-source pairs where sep < radius_cut + n_proc : int + Number of processors to use. Should not exceed the number of cores. + Default is one processor (no parallelization). - c : SkyCoord object - Coordinates of all the stars. + overwrite : bool + If set to True, overwrites output files. If set to False, exits the + function if output files are already on disk. + Default is False. """ - # Propagate r, b, l positions forward in time. - r_t = bigpatch['rad'] + timei * bigpatch['vr'] * kms_to_kpcday # kpc - b_t = bigpatch['glat'] + timei * bigpatch['mu_b'] * masyr_to_degday # deg - l_t = bigpatch['glon'] + timei * (bigpatch['mu_lcosb'] / np.cos(np.radians(bigpatch['glat']))) * masyr_to_degday # deg - - ########## - # Determine nearest neighbor in spherical coordinates. - ########## - c = SkyCoord(frame='galactic', l=l_t * units.deg, b=b_t * units.deg) - - # NOTE: dist has no actual meaning since - # we didn't input distances from Earth. - # It's an auto output. just ignore it. - idx, sep, dist = coord.match_coordinates_sky(c, c, nthneighbor=2) + if hdf5_file[-3:] != '.h5': + raise Exception('hdf5_file (%s) must be an h5 file.' % str(hdf5_file)) - # Converts separations to milliarcseconds - sep = (sep.to(units.mas)) / units.mas + if type(output_root2) != str: + raise Exception('output_root2 (%s) must be a string.' % str(output_root2)) - ########## - # Error checking: calculate how many duplicate (l, b) pairs there are. - # (This is a problem for nearest neighbors.) - ########## - uni = np.unique((l_t, b_t), axis=1).shape[1] - tot = len(l_t) - dup = tot - uni - if dup != 0: - print('****************** WARNING!!! ********************') - print('There are ' + str(dup) + ' duplicate (l, b) pairs.') - print('**************************************************') + if type(radius_cut) != int: + if type(radius_cut) != float: + raise Exception('radius_cut (%s) must be an integer or a float.' % str(radius_cut)) - ########## - # Find all objects with a nearest neighbor within an angular distance - # equal to sep. The index of the object and its nearest neighbor are - # event_id1 and event_id2. (Indices correspond to those of idx and sep.) - ########## - # NOTE: event_id1/2 are indices into bigpatch - event_id1 = np.where(sep < radius_cut)[0] - event_id2 = idx[event_id1] + if type(obs_time) != int: + if type(obs_time) != float: + raise Exception('obs_time (%s) must be an integer or a float.' % str(obs_time)) - ########## - # We've got neighbors... figure out who's the lens and who's the source. - ########## - # NOTE: lens_id and sorc_id are indices into bigpatch - idx_l1 = np.where(r_t[event_id1] < r_t[event_id2])[0] - idx_l2 = np.where(r_t[event_id1] > r_t[event_id2])[0] + if type(blend_rad) != int: + if type(blend_rad) != float: + raise Exception('blend_rad (%s) must be an integer or a float.' % str(blend_rad)) - lens_id = np.zeros(len(event_id1), dtype='int') - sorc_id = np.zeros(len(event_id1), dtype='int') + if type(n_obs) != int: + raise Exception('n_obs (%s) must be an integer.' % str(n_obs)) - lens_id[idx_l1] = event_id1[idx_l1] - sorc_id[idx_l1] = event_id2[idx_l1] + if type(n_proc) != int: + raise Exception('n_proc (%s) must be an integer.' % str(n_proc)) - lens_id[idx_l2] = event_id2[idx_l2] - sorc_id[idx_l2] = event_id1[idx_l2] + if type(overwrite) != bool: + raise Exception('overwrite (%s) must be a boolean.' % str(overwrite)) - return lens_id, sorc_id, r_t, sep, event_id1, c + if type(theta_frac) != int: + if type(theta_frac) != float: + raise Exception('theta_frac (%s) must be an integer or a float.' % str(theta_frac)) -def _calc_event_cands_thetaE(bigpatch, theta_E, u, theta_frac, lens_id, - sorc_id, timei): +def calc_events(hdf5_file, output_root2, + radius_cut=2, obs_time=1000, n_obs=101, theta_frac=2, + blend_rad=0.65, n_proc=1, + overwrite=False): """ - Get sources and lenses that pass the radius cut. + Calculate microlensing events Parameters ---------- - bigpatch : array - Compilation of 4 .h5 datasets containing stars. + hdf5_file : str + Name of the HDF5 file. - theta_E : array - Einstein radius of the events that pass the radius cut + output_root2 : str + The name for the h5 file - u : array - Impact parameters at time t of the events that pass the radius cut + radius_cut : float + Initial radius cut, in ARCSECONDS. + + obs_time : float + Survey duration, in DAYS. + + n_obs : int + Number of observations. theta_frac : float - Parameter of calc_events() Another cut, in multiples of Einstein radii. - lens_id : array - Indices into bigpatch that indicate lenses + blend_rad : float + Stars within this distance of the lens are said to be blended. + Units are in ARCSECONDS. - sorc_id : array - Indices into bigpatch that indicate sources + Optional Parameters + ------------------- + n_proc : int + Number of processors to use. Should not exceed the number of cores. + Default is one processor (no parallelization). - timei : float - Time at which to evaluate. + overwrite : bool + If set to True, overwrites output files. If set to False, exists the + function if output files are already on disk. + Default is False. - Return + + Output ------ - event_lbt : array - Lenses and sources at a particular time t. + _events.fits : Astropy .fits table + Table of candidate microlensing events. The number of rows + corresponds to the number of candidate events. """ - # NOTE: adx is an index into lens_id or event_id (NOT bigpatch) - adx = np.where(u < theta_frac)[0] - if len(adx > 0): - # Narrow down to unique pairs of stars... don't double calculate - # an event. - lens_sorc_id_pairs = np.stack((bigpatch['obj_id'][lens_id][adx], - bigpatch['obj_id'][sorc_id][adx]), - axis=-1) - - unique_returns = np.unique(lens_sorc_id_pairs, - return_index=True, return_inverse=True, - return_counts=True, axis=0) - unique_tab = unique_returns[0] - unique_indices = unique_returns[1] - unique_inverse = unique_returns[2] - unique_counts = unique_returns[3] - - # Define all tables we will need to calculate event properties. - # These will all have a length of N_events. - lens_table = bigpatch[lens_id][adx][unique_indices] - sorc_table = bigpatch[sorc_id][adx][unique_indices] - theta_E = theta_E[adx][unique_indices] - u = u[adx][unique_indices] - - mu_b_rel = sorc_table['mu_b'] - lens_table['mu_b'] # mas/yr - mu_lcosb_rel = sorc_table['mu_lcosb'] - lens_table['mu_lcosb'] # mas/yr - mu_rel = np.sqrt(mu_b_rel ** 2 + mu_lcosb_rel ** 2) # mas/yr - t_event = np.ones(len(mu_rel), dtype=float) * timei # days - # This is all the events for this l, b, time - # Loop through the lens table and append '_L' to the end of each field - lens_rename_dct = {} - for name in lens_table.dtype.names: - lens_rename_dct[name] = name + '_L' - lens_table = rfn.rename_fields(lens_table, lens_rename_dct) + ########## + # Error handling: check whether files exist and + # whether input types are correct. + ########## - # Loop through the source table and append '_S' to the end of each field - sorc_rename_dct = {} - for name in sorc_table.dtype.names: - sorc_rename_dct[name] = name + '_S' - sorc_table = rfn.rename_fields(sorc_table, sorc_rename_dct) + # Check if .fits file exists already. If it does, throw an error message + # to complain and exit. + if not overwrite: + if os.path.isfile(output_root2 + '_events.fits'): + raise Exception( + 'That events.fits file name is taken! Either delete the .fits ' + 'file, or pick a new name.') + if os.path.isfile(output_root2 + '_blends.fits'): + raise Exception( + 'That blends.fits file name is taken! Either delete the .fits ' + 'file, or pick a new name.') - # Combine the lens and source tables into the events table - event_lbt = rfn.merge_arrays((lens_table, sorc_table), - flatten=True) + # Error handling/complaining if input types are not right. + _check_calc_events(hdf5_file, output_root2, + radius_cut, obs_time, n_obs, theta_frac, + blend_rad, n_proc, overwrite) - # Add additional microlensing parameters to the events table - event_lbt = rfn.append_fields(event_lbt, 'theta_E', - theta_E, usemask=False) - event_lbt = rfn.append_fields(event_lbt, 'u0', - u, usemask=False) - event_lbt = rfn.append_fields(event_lbt, 'mu_rel', - mu_rel, usemask=False) - event_lbt = rfn.append_fields(event_lbt, 't0', - t_event, usemask=False) - return event_lbt + ########## + # Start of code + ######### - else: - return None + t0 = time.time() + # Initialize events_tmp and blends_tmp. + events_tmp = None + blends_tmp = None -def _calc_blends(bigpatch, c, event_lbt, blend_rad): - """ - Create a table containing the blended stars for each event. - Note 1: We are centering on the lens. - Note 2: We don't want to include the lens itself, - or the source, in the table. + # Get the l and b from the HDF5 file. + hf = h5py.File(hdf5_file, 'r') + l_array = np.array(hf['long_bin_edges']) + b_array = np.array(hf['lat_bin_edges']) + hf.close() - Parameters - ---------- - bigpatch : array - Compilation of 4 .h5 datasets containing stars. + # Converts radius_cut from arcseconds into milliarcseconds + radius_cut *= 1000.0 - c : SkyCoord object - Coordinates of all the stars. + # Set up the multiprocessing + pool = Pool(n_proc) - event_lbt : array - Lenses and sources at a particular time t. + # Set up inputs to be able to be read by pool.map + nll = len(l_array[:]) - 2 + nbb = len(b_array[:]) - 2 - blend_rad : float - Parameter of calc_events(). + llbb = itertools.product(range(nll), range(nbb)) - Return - ------ - blends_lbt : array - Array of neighbor stars for each lens-source pair. + reps = nll * nbb - """ - ########## - # Get the cached KD-Tree to make things run faster. - ########## - # This way, we don't have to remake a tree that already exists. - # (We didn't just do the neighbor calculation initially, because - # it would be expensive to hold onto all the unnecessary neighbors) - kdtree_cache = c.cache['kdtree_sky'] + hd = itertools.repeat(hdf5_file, reps) + ot = itertools.repeat(obs_time, reps) + no = itertools.repeat(n_obs, reps) + rc = itertools.repeat(radius_cut, reps) + tf = itertools.repeat(theta_frac, reps) + br = itertools.repeat(blend_rad, reps) - # Define the center of the blending disk (the lens) - coords_lbt = SkyCoord(frame='galactic', - l=np.array(event_lbt['glon_L']) * units.deg, - b=np.array(event_lbt['glat_L']) * units.deg) + inputs = zip(llbb, hd, ot, no, rc, tf, br) ########## - # Replicate astropy's search_around_sky. + # Loop through galactic latitude and longitude bins. For each bin vertex, + # take the nearest 4 bin samples and calculate microlensing events. + # We do this to properly handle bin edges + # (i.e. a sliding window analysis of 2x2 bins). + # Duplicate events are removed. ########## - # Make the coordinates to query around - seplimit = blend_rad * units.arcsec - coords1 = coords_lbt - coords1 = coords1.transform_to(c) - urepr1 = coords1.data.represent_as(UnitSphericalRepresentation) - ucoords1 = coords1.realize_frame(urepr1) - cartxyz1 = ucoords1.cartesian.xyz - flatxyz1 = cartxyz1.reshape((3, np.prod(cartxyz1.shape) // 3)) + # Should I use starmap_async? + results = pool.starmap(_calc_event_time_loop, inputs) - # Define the query distance. - r_kdt = (2 * np.sin(Angle(seplimit) / 2.0)).value + pool.close() + pool.join() - # Query ball against the existing (cached) tree. - # NOTE: results is an array of lists. - #results = kdtree_cache.query_ball_point(flatxyz1.T.copy(order='C'), r_kdt) - results = kdtree_cache.query_ball_point(flatxyz1.T, r_kdt) + # Remove all the None values + # (occurs for patches with less than 10 objects) + results = [i for i in results if i is not None] - # Figure out the number of blends for each lens. - blend_lens_obj_id = [] - blend_sorc_obj_id = [] - blend_neigh_obj_id = [] - blend_neigh_idx = [] - sep_LN_list = [] + results_ev = [] + results_bl = [] for ii in range(len(results)): - # results indexes into bigpatch. - # ii corresponds to coords_lbt. - if len(results[ii]) == 0: - continue - - # bidx indexes into results. - # It should be that len(results[ii]) == len(bidx) + 2 (we get rid of source and lens.) - # neighbor star object id - nid = bigpatch['obj_id'][results[ii]] - # lens star object id - lid = np.array(event_lbt['obj_id_L'][ii]) - # source star object id - sid = np.array(event_lbt['obj_id_S'][ii]) - - # Fetch the things that are not the lens and the source. - bidx = np.where((nid != lid) & - (nid != sid))[0] - - if len(bidx) == 0: - continue - - # Make a list of the lens and source IDs for each neighbor... these are just repeats. - tmp_lens_id = np.repeat(lid, len(bidx)).tolist() - tmp_sorc_id = np.repeat(sid, len(bidx)).tolist() - - # Calculate the distance from lens to each neighbor. - lens_lb = SkyCoord(frame='galactic', - l=np.array(event_lbt['glon_L'][ii]) * units.deg, - b=np.array(event_lbt['glat_L'][ii]) * units.deg) - neigh_lb = SkyCoord(frame='galactic', - l=bigpatch['glon'][results[ii]][bidx] * units.deg, - b=bigpatch['glat'][results[ii]][bidx] * units.deg) - sep_LN = lens_lb.separation(neigh_lb) - sep_LN = (sep_LN.to(units.arcsec)) / units.arcsec - - # Add the non-lens, non-source blended object IDs to a list. - # Add the lens and the source object ID to a new list as well. - # Add the index of the neighbors. - blend_neigh_obj_id.extend(nid[bidx].tolist()) - blend_lens_obj_id.extend(tmp_lens_id) - blend_sorc_obj_id.extend(tmp_sorc_id) - blend_neigh_idx.extend([results[ii][bb] for bb in bidx]) - sep_LN_list.extend(sep_LN.value.tolist()) - - # Convert our lists into arrays. - blend_neigh_obj_id = np.array(blend_neigh_obj_id) - blend_lens_obj_id = np.array(blend_lens_obj_id) - blend_sorc_obj_id = np.array(blend_sorc_obj_id) - blend_neigh_idx = np.array(blend_neigh_idx) - sep_LN_list = np.array(sep_LN_list) - - if len(blend_neigh_obj_id) > 0: - # Grab the rows of bigpatch that are neighbors - blends_lbt = bigpatch[blend_neigh_idx] - - # Append '_N' to each column in blends_lbt - blends_rename_dct = {} - for name in blends_lbt.dtype.names: - blends_rename_dct[name] = name + '_N' - blends_lbt = rfn.rename_fields(blends_lbt, blends_rename_dct) + if results[ii] is not None: + if results[ii][0] is not None: + results_ev.append(results[ii][0]) + if results[ii][1] is not None: + results_bl.append(results[ii][1]) - # Add additional columns into blends_lbt - blends_lbt = rfn.append_fields(blends_lbt, 'obj_id_L', - blend_lens_obj_id, usemask=False) - blends_lbt = rfn.append_fields(blends_lbt, 'obj_id_S', - blend_sorc_obj_id, usemask=False) - blends_lbt = rfn.append_fields(blends_lbt, 'sep_LN', - sep_LN_list, usemask=False) + if len(results_ev) == 0: + print('No events!') + return else: - blends_lbt = None - - return blends_lbt + events_tmp = np.concatenate(results_ev, axis=0) + if len(results_bl) == 0: + blends_tmp = np.array([]) + else: + blends_tmp = np.concatenate(results_bl, axis=0) + # Convert the events numpy recarray into an + # Astropy Table for easier consumption. + events_tmp = unique_events(events_tmp) + events_final = Table(events_tmp) + N_events = len(events_final) + print('Candidate events detected: ', N_events) -def unique_events(event_table): - """ - Given an event table, there might be a single microlensing event listed - multiple times (it might have been observed at different timesteps, or - the nearest neighbor pair might have been symmetric for source and lens.) - This function will eliminate duplicates, only keeping an event once. It - is picked to be the particular observed event with the smallest source- - lens separation. + if len(results_bl) != 0: + blends_tmp = unique_blends(blends_tmp) + blends_final = Table(blends_tmp) - Parameters - --------- - event_table : numpy array - A table with all the events. There are equal numbers of columns - containing info about the source and the lens, and four additional - columns with theta_E, u0, mu_rel, and t0. The number of rows - corresponds to the number of events. + # Save out file + events_final.write(output_root2 + '_events.fits', overwrite=overwrite) + blends_final.write(output_root2 + '_blends.fits', overwrite=overwrite) - Return - ------ - new_event_table : numpy array - Same as event_table, but all duplicate events have been trimmed out, - such that each event only is listed once (at the observed time where - the source-lens separation is smallest.) + t1 = time.time() - """ - # Pull the unique ID numbers for the lens and source and put them into a - # table of 2 x N_lenses. - lens_uid = event_table['obj_id_L'] - sorc_uid = event_table['obj_id_S'] - events_uid = np.swapaxes(np.vstack((lens_uid, sorc_uid)), 0, 1) + ########## + # Make log file + ########## + now = datetime.datetime.now() + radius_cut = radius_cut / 1000.0 # back to arcsec + microlens_path = os.path.dirname(inspect.getfile(perform_pop_syn)) + microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=microlens_path).decode('ascii').strip() + dash_line = '-----------------------------' + '\n' + empty_line = '\n' + line0 = 'FUNCTION INPUT PARAMETERS' + '\n' + line1 = 'hdf5_file , ' + hdf5_file + '\n' + line2 = 'output_root2 , ' + output_root2 + '\n' + line3 = 'radius_cut , ' + str(radius_cut) + ' , (arcsec)' + '\n' + line4 = 'obs_time , ' + str(obs_time) + ' , (days)' + '\n' + line5 = 'n_obs , ' + str(n_obs) + '\n' + line6 = 'theta_frac , ' + str(theta_frac) + ' , (thetaE)' + '\n' + line7 = 'blend_rad , ' + str(blend_rad) + ' , (arcsec)' + '\n' + line8 = 'n_proc , ' + str(n_proc) + '\n' + line9 = 'VERSION INFORMATION' + '\n' + line10 = str(now) + ' : creation date' + '\n' + line11 = microlens_hash + ' : microlens commit' + '\n' - # Determine if we have unique events (and how many duplicates there are). - unique_returns = np.unique(events_uid, - return_index=True, return_inverse=True, - return_counts=True, axis=0) - unique_tab = unique_returns[0] - unique_indices = unique_returns[1] - unique_inverse = unique_returns[2] - unique_counts = unique_returns[3] + line12 = 'OTHER INFORMATION' + '\n' + line13 = str(t1 - t0) + ' : total runtime (s)' + '\n' + line14 = str(N_events) + ' : total number of events' + '\n' - new_event_table = event_table[unique_indices] + line15 = 'FILES CREATED' + '\n' + line16 = output_root2 + '_events.fits : events file' + '\n' + line17 = output_root2 + '_blends.fits : blends file' + '\n' - # Check for duplicate events and keep the one with the closest u. - dpdx = np.where(unique_counts > 1)[0] - for ii in range(len(dpdx)): - # Fetch the duplicates for this event. - dup_idx = np.where(unique_inverse == dpdx[ii])[0] - dup_events = event_table[dup_idx] - min_idx = np.argmin(dup_events['u0']) - new_event_table[dpdx[ii]] = event_table[dup_idx[min_idx]] + with open(output_root2 + '_calc_events.log', 'w') as out: + out.writelines([line0, dash_line, line1, line2, line3, + line4, line5, line6, line7, line8, empty_line, + line9, dash_line, line10, line11, empty_line, + line12, dash_line, line13, line14, empty_line, line15, + dash_line, line16, line17]) - return new_event_table + print('Total runtime: {0:f} s'.format(t1 - t0)) + + return -def unique_blends(blend_table): +def _calc_event_time_loop(llbb, hdf5_file, obs_time, n_obs, radius_cut, + theta_frac, blend_rad): """ - Given an blends table, there might be a single lens-source-neighbor triple - multiple times (it might have been observed at different timesteps.) - This function will eliminate duplicates, only keeping an event once. It - is picked to be the first occurence. - Parameters - --------- - blend_table : blend array - A table with all the events. There is 1 column with the unique - source ID, 1 with the unique lens ID lens, 1 with the lens-neighbor - separation, and the remaining columns contain info about the neighbors. + ---------- + llbb : (int, int) + Indices of (l,b) bin. + obs_time, n_obs, radius_cut, theta_frac, blend_rad + are all parameters of calc_events() Return ------ - new_blend_table : numpy array - Same as blend_table, but all duplicate events have been trimmed out, - such that each event only is listed once (at the observed time where - the source-lens separation is smallest.) + events_llbb : array + Array of the unique events for the particular (l,b) patch. + + blends_llbb : array + Array of the unique blends for the particular (l,b) patch. """ - # Pull the unique ID numbers for the lens, source, and neighbors and put - # them into a table of 3 x N_blends. - lens_obj_id = blend_table['obj_id_L'] - sorc_obj_id = blend_table['obj_id_S'] - neigh_obj_id = blend_table['obj_id_N'] + #################### + # Loop through different time steps and figure out separations between + # all possible pairs of stars. Trim down to "events", which consist of + # those pairs that approach within one of each other. + # These will be the events we consider as candidate microlensing events. + #################### - triples_obj_id = np.swapaxes(np.vstack((lens_obj_id, - sorc_obj_id, - neigh_obj_id)), - 0, 1) + # Initialize events_llbb and blends_llbb. + events_llbb = None + blends_llbb = None - # Determine if we have unique events (and how many duplicates there are). - # We will keep the first occurence. - unique_returns = np.unique(triples_obj_id, return_index=True, axis=0) - unique_tab = unique_returns[0] - unique_indices = unique_returns[1] + ll = llbb[0] + bb = llbb[1] - unique_blend_tab = blend_table[unique_indices] + print('Working on loop ll, bb = ', ll, bb) + name00 = 'l' + str(ll) + 'b' + str(bb) + name01 = 'l' + str(ll) + 'b' + str(bb + 1) + name10 = 'l' + str(ll + 1) + 'b' + str(bb) + name11 = 'l' + str(ll + 1) + 'b' + str(bb + 1) - return unique_blend_tab + hf = h5py.File(hdf5_file, 'r') + bigpatch = np.hstack((hf[name00], hf[name01], hf[name10], hf[name11])) + hf.close() + # Skip patches with less than 10 objects + if len(bigpatch) < 10: + # continue + return -def calc_diff_limit_blend(event_fits_file, blend_fits_file, blend_rad): - """ - Given a table of events and blends, calculate what the blending would be - for a smaller blending radius (e.g. table is for seeing limited, - and you want what the diffraction limit blend is.) - """ - diff_limit_blend = None + time_array = np.linspace(-1 * obs_time / 2.0, obs_time / 2.0, n_obs) - event = Table.read(event_fits_file) - blend = Table.read(blend_fits_file) + for i in np.arange(len(time_array)): + # Find potential lenses and sources that fall within radius cut. + lens_id, sorc_id, r_t, sep, event_id1, c = _calc_event_cands_radius(bigpatch, + time_array[i], + radius_cut) - for i in range(len(blend)): - lens_id = blend[i]['obj_id_L'] - sorc_id = blend[i]['obj_id_S'] - event_idx = np.where( - (event['obj_id_L'] == lens_id) & (event['obj_id_S'] == sorc_id))[0] + # Calculate einstein radius and lens-source separation + theta_E = einstein_radius(bigpatch['mass'][lens_id], + r_t[lens_id], r_t[sorc_id]) # mas + u = sep[event_id1] / theta_E - l_L = event[event_idx]['glon_L'] - b_L = event[event_idx]['glat_L'] - c_L = SkyCoord(frame='galactic', l=l_L * units.deg, b=b_L * units.deg) + # Trim down to those microlensing events that really get close enough + # to hope that we can detect them. Trim on a Theta_E criteria. + event_lbt = _calc_event_cands_thetaE(bigpatch, theta_E, u, theta_frac, + lens_id, sorc_id, time_array[i]) - l_N = blend[i]['glon_N'] - b_N = blend[i]['glat_N'] - c_N = SkyCoord(frame='galactic', l=l_N * units.deg, b=b_N * units.deg) + if event_lbt is not None: + # Concatenate the current event table + # (at this l, b, time) with the rest. + if events_llbb is not None: + events_llbb = np.hstack((events_llbb, event_lbt)) + else: + events_llbb = event_lbt - sep = (c_L.separation(c_N)).arcsec + # Keep only unique events within our different time stamps + events_llbb = unique_events(events_llbb) - if sep < blend_rad: - if diff_limit_blend is not None: - diff_limit_blend = vstack((diff_limit_blend, blend[i])) - else: - diff_limit_blend = blend[i] + ######### + # Get blending. + # Note 1: We are centering on the lens. + # Note 2: We don't want to include the lens itself, + # or the source, in the table. + ########## + blends_lbt = _calc_blends(bigpatch, c, event_lbt, blend_rad) - diff_limit_blend.write('diff_limit_blend_' + str(blend_rad) + '.fits') + if blends_lbt is not None: + # Concatenate the current blend table (at this l, b, time) + # with the rest. + if blends_llbb is not None: + blends_llbb = np.hstack((blends_llbb, blends_lbt)) + else: + blends_llbb = blends_lbt - return + # Keep only unique blends within our different time stamps + blends_llbb = unique_blends(blends_llbb) + # END of time loop -def reduce_blend_rad(blend_tab, new_blend_rad, output_root, overwrite=False): + return events_llbb, blends_llbb + + +def _calc_event_cands_radius(bigpatch, timei, radius_cut): """ - Creates a new blend table for some blending radius r_new - that is smaller than the original blend radius r_orig, - i.e. r_new < r_orig. Also makes a corresponding log and events. + Get sources and lenses that pass the radius cut. Parameters ---------- - blend_tab : str - The name of the blend table. + bigpatch : array + Compilation of 4 .h5 datasets containing stars. - new_blend_rad : float or int - The new (smaller) blend radius. - Units are in ARCSECONDS. + timei : float + Time at which to evaluate. - output_root : str - The name for the new blend table - (and corresponding event table) + radius_cut : float + Parameter of calc_events(). + Converted to mas Return ------ - new_blend : .fits table - New table with smaller blend radius. + lens_id : array + Indices into bigpatch that indicate lenses - """ - input_root = blend_tab.rstrip('_blends.fits') + sorc_id : array + Indices into bigpatch that indicate sources - event_tab_name = input_root + '_events.fits' - event_log_name = input_root + '_calc_events.log' + r_t : array + Radial coordinates for all objects in bigpatch at time t - new_event_tab_name = output_root + '_events.fits' - new_event_log_name = output_root + '_calc_events.log' - new_blend_tab_name = output_root + '_blends.fits' + sep : array + Separation between lens-source pairs (mas) - os.system('cp %s %s' % (event_tab_name, new_event_tab_name)) - os.system('cp %s %s' % (event_log_name, new_event_log_name)) + event_id1 : array + Lens-source pairs where sep < radius_cut - now = datetime.datetime.now() - dash_line = '-----------------------------' + '\n' + c : SkyCoord object + Coordinates of all the stars. + """ + # Propagate r, b, l positions forward in time. + r_t = bigpatch['rad'] + timei * bigpatch['vr'] * kms_to_kpcday # kpc + b_t = bigpatch['glat'] + timei * bigpatch['mu_b'] * masyr_to_degday # deg + l_t = bigpatch['glon'] + timei * (bigpatch['mu_lcosb'] / np.cos(np.radians(bigpatch['glat']))) * masyr_to_degday # deg - line0 = 'NEW BLEND RADIUS INFO AND FILES CREATED' + '\n' - line1 = 'new blend rad : ' + str(new_blend_rad) + ' (arcsec)' + '\n' - line2 = 'blend file : ' + new_blend_tab_name + '\n' - line3 = 'event file : ' + new_event_tab_name + '\n' - line4 = 'creation time: ' + str(now) + '\n' + ########## + # Determine nearest neighbor in spherical coordinates. + ########## + c = SkyCoord(frame='galactic', l=l_t * units.deg, b=b_t * units.deg) - with open(new_event_log_name, 'a') as my_file: - my_file.writelines(['\n', line0, dash_line, - line1, line2, line3, line4]) + # NOTE: dist has no actual meaning since + # we didn't input distances from Earth. + # It's an auto output. just ignore it. + idx, sep, dist = coord.match_coordinates_sky(c, c, nthneighbor=2) - old_blends = Table.read(blend_tab) - good_idx = np.where(old_blends['sep_LN'] < new_blend_rad)[0] - new_blends = old_blends[good_idx] + # Converts separations to milliarcseconds + sep = (sep.to(units.mas)) / units.mas - new_blends.write(new_blend_tab_name, overwrite=overwrite) + ########## + # Error checking: calculate how many duplicate (l, b) pairs there are. + # (This is a problem for nearest neighbors.) + ########## + uni = np.unique((l_t, b_t), axis=1).shape[1] + tot = len(l_t) + dup = tot - uni + if dup != 0: + print('****************** WARNING!!! ********************') + print('There are ' + str(dup) + ' duplicate (l, b) pairs.') + print('**************************************************') + + ########## + # Find all objects with a nearest neighbor within an angular distance + # equal to sep. The index of the object and its nearest neighbor are + # event_id1 and event_id2. (Indices correspond to those of idx and sep.) + ########## + # NOTE: event_id1/2 are indices into bigpatch + event_id1 = np.where(sep < radius_cut)[0] + event_id2 = idx[event_id1] - return + ########## + # We've got neighbors... figure out who's the lens and who's the source. + ########## + # NOTE: lens_id and sorc_id are indices into bigpatch + idx_l1 = np.where(r_t[event_id1] < r_t[event_id2])[0] + idx_l2 = np.where(r_t[event_id1] > r_t[event_id2])[0] + lens_id = np.zeros(len(event_id1), dtype='int') + sorc_id = np.zeros(len(event_id1), dtype='int') -############################################################################ -######### Refined event rate calculation and associated functions ########## -############################################################################ + lens_id[idx_l1] = event_id1[idx_l1] + sorc_id[idx_l1] = event_id2[idx_l1] + lens_id[idx_l2] = event_id2[idx_l2] + sorc_id[idx_l2] = event_id1[idx_l2] -def _convert_photometric_99_to_nan(table, photometric_system='ubv'): - for name in table.colnames: - if ('exbv' in name) or (photometric_system in name): - cond = np.where(table[name] == -99)[0] - table[name][cond] = np.nan + return lens_id, sorc_id, r_t, sep, event_id1, c -def _check_refine_events(input_root, filter_name, - photometric_system, red_law, overwrite, - output_file): +def _calc_event_cands_thetaE(bigpatch, theta_E, u, theta_frac, lens_id, + sorc_id, timei): """ - Checks that the inputs of refine_events are valid + Get sources and lenses that pass the radius cut. Parameters ---------- - input_root : str - The root path and name of the *_events.fits and *_blends.fits. - Don't include those suffixes yet. - - filter_name : str - The name of the filter in which to calculate all the - microlensing events. The filter name convention is set - in the global filt_dict parameter at the top of this module. - - photometric_system : str - The name of the photometric system in which the filter exists. - - red_law : str - The name of the reddening law to use from PopStar. - - overwrite : bool - If set to True, overwrites output files. If set to False, exists the - function if output files are already on disk. - Default is False. - """ - if type(input_root) != str: - raise Exception('input_root (%s) must be a string.' % str(input_root)) - - if type(filter_name) != str: - raise Exception('filter_name (%s) must be a string.' % str(filter_name)) - - if type(photometric_system) != str: - raise Exception('photometric_system (%s) must be a string.' % str(photometric_system)) + bigpatch : array + Compilation of 4 .h5 datasets containing stars. - if type(red_law) != str: - raise Exception('red_law (%s) must be a string.' % str(red_law)) + theta_E : array + Einstein radius of the events that pass the radius cut - if type(output_file) != str: - raise Exception('output_file (%s) must be a string.' % str(output_file)) + u : array + Impact parameters at time t of the events that pass the radius cut - if type(overwrite) != bool: - raise Exception('overwrite (%s) must be a boolean.' % str(overwrite)) + theta_frac : float + Parameter of calc_events() + Another cut, in multiples of Einstein radii. - # Check to see that the filter name, photometric system, red_law are valid - if photometric_system not in photometric_system_dict: - exception_str = 'photometric_system must be a key in ' \ - 'photometric_system_dict. \n' \ - 'Acceptable values are : ' - for photometric_system in photometric_system_dict: - exception_str += '%s, ' % photometric_system - exception_str = exception_str[:-2] - raise Exception(exception_str) + lens_id : array + Indices into bigpatch that indicate lenses - if filter_name not in photometric_system_dict[photometric_system]: - exception_str = 'filter_name must be a value in ' \ - 'photometric_system_dict[%s]. \n' \ - 'Acceptable values are : ' % photometric_system - for filter_name in photometric_system_dict[photometric_system]: - exception_str += '%s, ' % filter_name - exception_str = exception_str[:-2] - raise Exception(exception_str) + sorc_id : array + Indices into bigpatch that indicate sources - key = photometric_system + '_' + filter_name - if red_law not in filt_dict[key]: - exception_str = 'red_law must be a value in ' \ - 'filt_dict[%s]. \n' \ - 'Acceptable values are : ' % key - for red_law in filt_dict[key]: - exception_str += '%s, ' % red_law - exception_str = exception_str[:-2] - raise Exception(exception_str) + timei : float + Time at which to evaluate. + Return + ------ + event_lbt : array + Lenses and sources at a particular time t. -def refine_events(input_root, filter_name, photometric_system, red_law, - overwrite=False, - output_file='default'): """ - Takes the output Astropy table from calc_events, and from that - calculates the time of closest approach. Will also return source-lens - separation at this time. + # NOTE: adx is an index into lens_id or event_id (NOT bigpatch) + adx = np.where(u < theta_frac)[0] + if len(adx > 0): + # Narrow down to unique pairs of stars... don't double calculate + # an event. + lens_sorc_id_pairs = np.stack((bigpatch['obj_id'][lens_id][adx], + bigpatch['obj_id'][sorc_id][adx]), + axis=-1) - Parameters - ---------- - input_root : str - The root path and name of the *_events.fits and *_blends.fits. - Don't include those suffixes yet. + unique_returns = np.unique(lens_sorc_id_pairs, + return_index=True, return_inverse=True, + return_counts=True, axis=0) + unique_tab = unique_returns[0] + unique_indices = unique_returns[1] + unique_inverse = unique_returns[2] + unique_counts = unique_returns[3] - filter_name : str - The name of the filter in which to calculate all the - microlensing events. The filter name convention is set - in the global filt_dict parameter at the top of this module. + # Define all tables we will need to calculate event properties. + # These will all have a length of N_events. + lens_table = bigpatch[lens_id][adx][unique_indices] + sorc_table = bigpatch[sorc_id][adx][unique_indices] + theta_E = theta_E[adx][unique_indices] + u = u[adx][unique_indices] - photometric_system : str - The name of the photometric system in which the filter exists. + mu_b_rel = sorc_table['mu_b'] - lens_table['mu_b'] # mas/yr + mu_lcosb_rel = sorc_table['mu_lcosb'] - lens_table['mu_lcosb'] # mas/yr + mu_rel = np.sqrt(mu_b_rel ** 2 + mu_lcosb_rel ** 2) # mas/yr + t_event = np.ones(len(mu_rel), dtype=float) * timei # days - red_law : str - The name of the reddening law to use from PopStar. + # This is all the events for this l, b, time + # Loop through the lens table and append '_L' to the end of each field + lens_rename_dct = {} + for name in lens_table.dtype.names: + lens_rename_dct[name] = name + '_L' + lens_table = rfn.rename_fields(lens_table, lens_rename_dct) - Optional Parameters - ------------------- - overwrite : bool - If set to True, overwrites output files. If set to False, exists the - function if output files are already on disk. - Default is False. + # Loop through the source table and append '_S' to the end of each field + sorc_rename_dct = {} + for name in sorc_table.dtype.names: + sorc_rename_dct[name] = name + '_S' + sorc_table = rfn.rename_fields(sorc_table, sorc_rename_dct) - output_file : str - The name of the final refined_events file. - If set to 'default', the format will be: - _refined_events___.fits + # Combine the lens and source tables into the events table + event_lbt = rfn.merge_arrays((lens_table, sorc_table), + flatten=True) - Output: - ---------- - A file will be created named - _refined_events___.fits - that contains all the same objects, only now with lots of extra - columns of data. + # Add additional microlensing parameters to the events table + event_lbt = rfn.append_fields(event_lbt, 'theta_E', + theta_E, usemask=False) + event_lbt = rfn.append_fields(event_lbt, 'u0', + u, usemask=False) + event_lbt = rfn.append_fields(event_lbt, 'mu_rel', + mu_rel, usemask=False) + event_lbt = rfn.append_fields(event_lbt, 't0', + t_event, usemask=False) + return event_lbt - """ - # Check if .fits file exists already. If it does, throw an error message - # to complain and exit. - if not overwrite and os.path.isfile(output_file): - raise Exception('That refined_events.fits file name is taken! ' - 'Either delete the .fits file, or pick a new name.') + else: + return None - # Error handling/complaining if input types are not right. - _check_refine_events(input_root, filter_name, - photometric_system, red_law, - overwrite, output_file) - if output_file == 'default': - output_file = '{0:s}_refined_events_{1:s}_{2:s}_{3:s}.fits'.format(input_root, - photometric_system, - filter_name, - red_law) +def _calc_blends(bigpatch, c, event_lbt, blend_rad): + """ + Create a table containing the blended stars for each event. + Note 1: We are centering on the lens. + Note 2: We don't want to include the lens itself, + or the source, in the table. - t_0 = time.time() + Parameters + ---------- + bigpatch : array + Compilation of 4 .h5 datasets containing stars. - event_fits_file = input_root + '_events.fits' - blend_fits_file = input_root + '_blends.fits' - log_file = input_root + '_calc_events.log' + c : SkyCoord object + Coordinates of all the stars. - event_tab = Table.read(event_fits_file) - blend_tab = Table.read(blend_fits_file) + event_lbt : array + Lenses and sources at a particular time t. - # If photometric fields contain -99, convert to nan - _convert_photometric_99_to_nan(event_tab, photometric_system) - _convert_photometric_99_to_nan(blend_tab, photometric_system) + blend_rad : float + Parameter of calc_events(). - # Only keep events with luminous sources - event_tab = event_tab[~np.isnan(event_tab[photometric_system + '_' + filter_name + '_S'])] + Return + ------ + blends_lbt : array + Array of neighbor stars for each lens-source pair. - with open(log_file) as my_file: - for num, line in enumerate(my_file): - if 'obs_time' in line: - obs_time = line.split(',')[1] - obs_time = float(obs_time) - break + """ + ########## + # Get the cached KD-Tree to make things run faster. + ########## + # This way, we don't have to remake a tree that already exists. + # (We didn't just do the neighbor calculation initially, because + # it would be expensive to hold onto all the unnecessary neighbors) + kdtree_cache = c.cache['kdtree_sky'] - # Calculate time and separation at closest approach, add to table - # NOTE: calc_closest_approach modifies the event table! - # It trims out events that peak outside the survey range! - N_events_original = len(event_tab) - print('Original candidate events: ', N_events_original) - u0, t0 = calc_closest_approach(event_tab, obs_time) - N_events_survey = len(event_tab) - print('Candidate events in survey window: ', N_events_survey) - event_tab['t0'] = t0 # days - event_tab['u0'] = u0 - if len(event_tab) == 0: - print('No events!') - return + # Define the center of the blending disk (the lens) + coords_lbt = SkyCoord(frame='galactic', + l=np.array(event_lbt['glon_L']) * units.deg, + b=np.array(event_lbt['glat_L']) * units.deg) ########## - # Calculate apparent magnitudes + # Replicate astropy's search_around_sky. ########## + # Make the coordinates to query around + seplimit = blend_rad * units.arcsec + coords1 = coords_lbt + coords1 = coords1.transform_to(c) + urepr1 = coords1.data.represent_as(UnitSphericalRepresentation) + ucoords1 = coords1.realize_frame(urepr1) + cartxyz1 = ucoords1.cartesian.xyz + flatxyz1 = cartxyz1.reshape((3, np.prod(cartxyz1.shape) // 3)) - # Einstein crossing time - # THIS HAS TO GO BEFORE _CALC_OBSERVABLES - t_E = event_tab['theta_E'] / event_tab['mu_rel'] # yr - t_E *= 365.25 # days - event_tab['t_E'] = t_E # days + # Define the query distance. + r_kdt = (2 * np.sin(Angle(seplimit) / 2.0)).value - # Add stuff to event_tab... shouldn't have any direct outputs - _calc_observables(filter_name, red_law, event_tab, blend_tab, photometric_system) + # Query ball against the existing (cached) tree. + # NOTE: results is an array of lists. + #results = kdtree_cache.query_ball_point(flatxyz1.T.copy(order='C'), r_kdt) + results = kdtree_cache.query_ball_point(flatxyz1.T, r_kdt) - # Relative parallax - pi_rel = event_tab['rad_L'] ** -1 - event_tab['rad_S'] ** -1 - event_tab['pi_rel'] = pi_rel # mas + # Figure out the number of blends for each lens. + blend_lens_obj_id = [] + blend_sorc_obj_id = [] + blend_neigh_obj_id = [] + blend_neigh_idx = [] + sep_LN_list = [] - # Microlensing parallax - pi_E = pi_rel / event_tab['theta_E'] - event_tab['pi_E'] = pi_E # dim'less + for ii in range(len(results)): + # results indexes into bigpatch. + # ii corresponds to coords_lbt. + if len(results[ii]) == 0: + continue - event_tab.write(output_file, overwrite=overwrite) + # bidx indexes into results. + # It should be that len(results[ii]) == len(bidx) + 2 (we get rid of source and lens.) + # neighbor star object id + nid = bigpatch['obj_id'][results[ii]] + # lens star object id + lid = np.array(event_lbt['obj_id_L'][ii]) + # source star object id + sid = np.array(event_lbt['obj_id_S'][ii]) - t_1 = time.time() + # Fetch the things that are not the lens and the source. + bidx = np.where((nid != lid) & + (nid != sid))[0] - ########## - # Make log file - ########## - now = datetime.datetime.now() - microlens_path = os.path.dirname(inspect.getfile(perform_pop_syn)) - popstar_path = os.path.dirname(inspect.getfile(imf)) - microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=microlens_path).decode('ascii').strip() - popstar_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=popstar_path).decode('ascii').strip() - dash_line = '-----------------------------' + '\n' - empty_line = '\n' + if len(bidx) == 0: + continue - line0 = 'FUNCTION INPUT PARAMETERS' + '\n' - line1 = 'input_root : ' + input_root + '\n' - line2 = 'filter_name : ' + filter_name + '\n' - line3 = 'red_law : ' + red_law + '\n' + # Make a list of the lens and source IDs for each neighbor... these are just repeats. + tmp_lens_id = np.repeat(lid, len(bidx)).tolist() + tmp_sorc_id = np.repeat(sid, len(bidx)).tolist() - line4 = 'VERSION INFORMATION' + '\n' - line5 = str(now) + ' : creation date' + '\n' - line6 = popstar_hash + ' : PopStar commit' + '\n' - line7 = microlens_hash + ' : microlens commit' + '\n' + # Calculate the distance from lens to each neighbor. + lens_lb = SkyCoord(frame='galactic', + l=np.array(event_lbt['glon_L'][ii]) * units.deg, + b=np.array(event_lbt['glat_L'][ii]) * units.deg) + neigh_lb = SkyCoord(frame='galactic', + l=bigpatch['glon'][results[ii]][bidx] * units.deg, + b=bigpatch['glat'][results[ii]][bidx] * units.deg) + sep_LN = lens_lb.separation(neigh_lb) + sep_LN = (sep_LN.to(units.arcsec)) / units.arcsec - line8 = 'OTHER INFORMATION' + '\n' - line9 = str(t_1 - t_0) + ' : total runtime (s)' + '\n' - line10 = str(N_events_original) + ' : original candidate events (dark sources removed)' + '\n' - line11 = str(N_events_survey) + ' : candidate events in survey window' + '\n' + # Add the non-lens, non-source blended object IDs to a list. + # Add the lens and the source object ID to a new list as well. + # Add the index of the neighbors. + blend_neigh_obj_id.extend(nid[bidx].tolist()) + blend_lens_obj_id.extend(tmp_lens_id) + blend_sorc_obj_id.extend(tmp_sorc_id) + blend_neigh_idx.extend([results[ii][bb] for bb in bidx]) + sep_LN_list.extend(sep_LN.value.tolist()) - line12 = 'FILES CREATED' + '\n' - line13 = output_file + ' : refined events' + # Convert our lists into arrays. + blend_neigh_obj_id = np.array(blend_neigh_obj_id) + blend_lens_obj_id = np.array(blend_lens_obj_id) + blend_sorc_obj_id = np.array(blend_sorc_obj_id) + blend_neigh_idx = np.array(blend_neigh_idx) + sep_LN_list = np.array(sep_LN_list) - with open(input_root + '_refined_events_' + photometric_system + '_' + filter_name + '_' + red_law + '.log', 'w') as out: - out.writelines([line0, dash_line, line1, line2, line3, empty_line, - line4, dash_line, line5, line6, line7, empty_line, - line8, dash_line, line9, line10, line11, empty_line, - line12, dash_line, line13]) + if len(blend_neigh_obj_id) > 0: + # Grab the rows of bigpatch that are neighbors + blends_lbt = bigpatch[blend_neigh_idx] - print('Total runtime: {0:f} s'.format(t_1 - t_0)) - return + # Append '_N' to each column in blends_lbt + blends_rename_dct = {} + for name in blends_lbt.dtype.names: + blends_rename_dct[name] = name + '_N' + blends_lbt = rfn.rename_fields(blends_lbt, blends_rename_dct) + # Add additional columns into blends_lbt + blends_lbt = rfn.append_fields(blends_lbt, 'obj_id_L', + blend_lens_obj_id, usemask=False) + blends_lbt = rfn.append_fields(blends_lbt, 'obj_id_S', + blend_sorc_obj_id, usemask=False) + blends_lbt = rfn.append_fields(blends_lbt, 'sep_LN', + sep_LN_list, usemask=False) + else: + blends_lbt = None -def calc_closest_approach(event_tab, survey_duration): + return blends_lbt + + +def unique_events(event_table): """ - Calculate the point of closest approach, during a given interval. - Return the position and time of closest approach. - This algorithm comes from a little writeup called - "Point of Closest Approach for the Linear Motion of Two Particles in 2-D" + Given an event table, there might be a single microlensing event listed + multiple times (it might have been observed at different timesteps, or + the nearest neighbor pair might have been symmetric for source and lens.) + This function will eliminate duplicates, only keeping an event once. It + is picked to be the particular observed event with the smallest source- + lens separation. Parameters - ---------- - event_tab : astropy table - This should take the astropy table that comes from the - output of calc_events. - - survey_duration : int or float - Survey duration, in days + --------- + event_table : numpy array + A table with all the events. There are equal numbers of columns + containing info about the source and the lens, and four additional + columns with theta_E, u0, mu_rel, and t0. The number of rows + corresponds to the number of events. Return ------ - u0 : array (same length as event_tab) - Minimum separation (normalized to Einstein radius) - - t0 : array (same length as event_tab) - Time at minimum separation (days) + new_event_table : numpy array + Same as event_table, but all duplicate events have been trimmed out, + such that each event only is listed once (at the observed time where + the source-lens separation is smallest.) """ - # Survey start and end time - start_time = -1 * survey_duration / 2.0 # days - end_time = survey_duration / 2.0 # days - - # Parametrization coefficients - cos_glat_S = np.cos(np.radians(event_tab['glat_S'])) - cos_glat_L = np.cos(np.radians(event_tab['glat_L'])) - - l_0 = (event_tab['glon_S'] - event_tab['glon_L']) * cos_glat_L * 3600.0 * 1.0e3 # position, mas - l_1 = (event_tab['mu_lcosb_S'] - event_tab['mu_lcosb_L']) # pm, mas/yr - b_0 = (event_tab['glat_S'] - event_tab['glat_L']) * 3600.0 * 1.0e3 # position, mas - b_1 = (event_tab['mu_b_S'] - event_tab['mu_b_L']) # pm, mas/yr - - # Calculate the critical time (time of closest approach), global minimum - t_crit = -(l_0 * l_1 + b_0 * b_1) / (l_1 ** 2 + b_1 ** 2) # years - t_crit *= 365.25 # days - - # Only include events that peak in survey window, - # since we can't easily fit parameters for events where we don’t have the peak. - # FIXME: more efficient way to do this?? - good = np.where((t_crit > start_time) & (t_crit < end_time))[0] - bad = np.where((t_crit <= start_time) | (t_crit >= end_time))[0] + # Pull the unique ID numbers for the lens and source and put them into a + # table of 2 x N_lenses. + lens_uid = event_table['obj_id_L'] + sorc_uid = event_table['obj_id_S'] + events_uid = np.swapaxes(np.vstack((lens_uid, sorc_uid)), 0, 1) - event_tab.remove_rows(bad) + # Determine if we have unique events (and how many duplicates there are). + unique_returns = np.unique(events_uid, + return_index=True, return_inverse=True, + return_counts=True, axis=0) + unique_tab = unique_returns[0] + unique_indices = unique_returns[1] + unique_inverse = unique_returns[2] + unique_counts = unique_returns[3] - # Initialize the time of closest approach vector, minimum of survey - # NOTE: THIS WILL OVERWRITE CURRENT VALUES - t0 = np.zeros(len(event_tab)) - u0 = np.zeros(len(event_tab)) + new_event_table = event_table[unique_indices] - # Assign values to closest approach values - t0 = t_crit[good] - u0 = calc_distance(event_tab, t0) + # Check for duplicate events and keep the one with the closest u. + dpdx = np.where(unique_counts > 1)[0] + for ii in range(len(dpdx)): + # Fetch the duplicates for this event. + dup_idx = np.where(unique_inverse == dpdx[ii])[0] + dup_events = event_table[dup_idx] + min_idx = np.argmin(dup_events['u0']) + new_event_table[dpdx[ii]] = event_table[dup_idx[min_idx]] - return u0, t0 + return new_event_table -def calc_distance(event_tab, time): +def unique_blends(blend_table): """ - Calculate the separation of two different objects at some given time. + Given an blends table, there might be a single lens-source-neighbor triple + multiple times (it might have been observed at different timesteps.) + This function will eliminate duplicates, only keeping an event once. It + is picked to be the first occurence. Parameters - ---------- - event_tab : astropy table + --------- + blend_table : blend array + A table with all the events. There is 1 column with the unique + source ID, 1 with the unique lens ID lens, 1 with the lens-neighbor + separation, and the remaining columns contain info about the neighbors. - time : int/float Return ------ - u : array (same length as astropy table) + new_blend_table : numpy array + Same as blend_table, but all duplicate events have been trimmed out, + such that each event only is listed once (at the observed time where + the source-lens separation is smallest.) """ - # Calculate sep - cos_glat_S = np.cos(np.radians(event_tab['glat_S'])) - cos_glat_L = np.cos(np.radians(event_tab['glat_L'])) + # Pull the unique ID numbers for the lens, source, and neighbors and put + # them into a table of 3 x N_blends. + lens_obj_id = blend_table['obj_id_L'] + sorc_obj_id = blend_table['obj_id_S'] + neigh_obj_id = blend_table['obj_id_N'] - l_L = (event_tab['glon_L'] + time * event_tab['mu_lcosb_L'] * masyr_to_degday / cos_glat_L) - b_L = (event_tab['glat_L'] + time * event_tab['mu_b_L'] * masyr_to_degday) - l_S = (event_tab['glon_S'] + time * event_tab['mu_lcosb_S'] * masyr_to_degday / cos_glat_S) - b_S = (event_tab['glat_S'] + time * event_tab['mu_b_S'] * masyr_to_degday) + triples_obj_id = np.swapaxes(np.vstack((lens_obj_id, + sorc_obj_id, + neigh_obj_id)), + 0, 1) - c_L = SkyCoord(frame='galactic', l=l_L * units.deg, b=b_L * units.deg) - c_S = SkyCoord(frame='galactic', l=l_S * units.deg, b=b_S * units.deg) + # Determine if we have unique events (and how many duplicates there are). + # We will keep the first occurence. + unique_returns = np.unique(triples_obj_id, return_index=True, axis=0) + unique_tab = unique_returns[0] + unique_indices = unique_returns[1] - sep = (c_L.separation(c_S)).mas - u = sep / event_tab['theta_E'] + unique_blend_tab = blend_table[unique_indices] - return u + return unique_blend_tab -def calc_blend_and_centroid(filter_name, red_law, blend_tab, photometric_system='ubv'): +def calc_diff_limit_blend(event_fits_file, blend_fits_file, blend_rad): """ - Given the absolute magnitudes of a bunch of things, - calculate their blended apparent magnitude and flux. - Also calculate the centroid of these things. - - Filter name is 'j', 'i', etc. - red_law is Damineli16, Schlegel99, etc. + Given a table of events and blends, calculate what the blending would be + for a smaller blending radius (e.g. table is for seeing limited, + and you want what the diffraction limit blend is.) """ - f_i = filt_dict[photometric_system + '_' + filter_name][red_law] + diff_limit_blend = None - # Calculate apparent magnitudes - app_N = calc_app_mag(blend_tab['rad_N'], - blend_tab[photometric_system + '_' + filter_name + '_N'], - blend_tab['exbv_N'], f_i) + event = Table.read(event_fits_file) + blend = Table.read(blend_fits_file) - # Convert absolute magnitudes to fluxes, and fix bad values - flux_N = 10 ** (app_N / -2.5) - flux_N = np.nan_to_num(flux_N) + for i in range(len(blend)): + lens_id = blend[i]['obj_id_L'] + sorc_id = blend[i]['obj_id_S'] + event_idx = np.where( + (event['obj_id_L'] == lens_id) & (event['obj_id_S'] == sorc_id))[0] - # Get total flux - flux_N_tot = np.sum(flux_N) + l_L = event[event_idx]['glon_L'] + b_L = event[event_idx]['glat_L'] + c_L = SkyCoord(frame='galactic', l=l_L * units.deg, b=b_L * units.deg) - # Get centroid - # If there is no flux, then the centroid is set to (l, b) = (0, 0). - if flux_N_tot == 0: - cent_l = 0.0 - cent_b = 0.0 - else: - cent_l = np.sum(flux_N * blend_tab['glon_N']) / flux_N_tot - cent_b = np.sum(flux_N * blend_tab['glat_N']) / flux_N_tot + l_N = blend[i]['glon_N'] + b_N = blend[i]['glat_N'] + c_N = SkyCoord(frame='galactic', l=l_N * units.deg, b=b_N * units.deg) - # Total blended magnitude - app_blended = -2.5 * np.log10(flux_N_tot) + sep = (c_L.separation(c_N)).arcsec - return app_blended, flux_N_tot, cent_l, cent_b + if sep < blend_rad: + if diff_limit_blend is not None: + diff_limit_blend = vstack((diff_limit_blend, blend[i])) + else: + diff_limit_blend = blend[i] + diff_limit_blend.write('diff_limit_blend_' + str(blend_rad) + '.fits') -def _calc_observables(filter_name, red_law, event_tab, blend_tab, photometric_system='ubv'): + return + + +def reduce_blend_rad(blend_tab, new_blend_rad, output_root, overwrite=False): """ - Calculate a bunch of observable quantities we get out from microlensing + Creates a new blend table for some blending radius r_new + that is smaller than the original blend radius r_orig, + i.e. r_new < r_orig. Also makes a corresponding log and events. Parameters ---------- - filter_name : str + blend_tab : str + The name of the blend table. - event_tab : + new_blend_rad : float or int + The new (smaller) blend radius. + Units are in ARCSECONDS. - blend_tab : + output_root : str + The name for the new blend table + (and corresponding event table) Return ------ - """ - f_i = filt_dict[photometric_system + '_' + filter_name][red_law] - - # Calculate apparent magnitude of lens and source, and fix bad values - app_S = calc_app_mag(event_tab['rad_S'], - event_tab[photometric_system + '_' + filter_name + '_S'], - event_tab['exbv_S'], f_i) - app_L = calc_app_mag(event_tab['rad_L'], - event_tab[photometric_system + '_' + filter_name + '_L'], - event_tab['exbv_L'], f_i) - event_tab[photometric_system + '_' + filter_name + '_app_S'] = app_S - event_tab[photometric_system + '_' + filter_name + '_app_L'] = app_L - - # Convert absolute magnitude to fluxes, and fix bad values - flux_L = 10 ** (app_L / -2.5) - flux_S = 10 ** (app_S / -2.5) + new_blend : .fits table + New table with smaller blend radius. - flux_L = np.nan_to_num(flux_L) - flux_S = np.nan_to_num(flux_S) + """ + input_root = blend_tab.rstrip('_blends.fits') - # Find the blends. - LS_pairs = np.stack((event_tab['obj_id_L'], event_tab['obj_id_S']), - axis=-1) - blend_pairs = np.stack((blend_tab['obj_id_L'], blend_tab['obj_id_S']), - axis=-1) + event_tab_name = input_root + '_events.fits' + event_log_name = input_root + '_calc_events.log' - flux_N = np.zeros(len(app_L)) - event_tab['cent_glon_' + filter_name + '_N'] = np.zeros(len(app_L)) - event_tab['cent_glat_' + filter_name + '_N'] = np.zeros(len(app_L)) + new_event_tab_name = output_root + '_events.fits' + new_event_log_name = output_root + '_calc_events.log' + new_blend_tab_name = output_root + '_blends.fits' - # Get the unique blend_pairs and the first instance of each. - if len(blend_pairs) > 0: - uni_blends, uni_bidx = np.unique(blend_pairs, return_index=True, - axis=0) - # Add a "dummy" idx onto uni_bidx so that I can do idx+1 later. - uni_bidxx = np.append(uni_bidx, len(blend_pairs)) - else: - # obj_id always >= 0, so these negative values serve as dummies that - # cannot be matched in the below `where` statement - uni_blends = np.array([[-1, -1]]) - uni_bidxx = np.array([]) + os.system('cp %s %s' % (event_tab_name, new_event_tab_name)) + os.system('cp %s %s' % (event_log_name, new_event_log_name)) - # FIXME: Put in a check that uni_bidx is monotonically increasing??? - # np.sort(uni_bidx) - uni_bidx == np.zeros(len(uni_bidx)) ??? + now = datetime.datetime.now() + dash_line = '-----------------------------' + '\n' - # FIXME: This seems more complicated than necessary... - for pp in range(len(LS_pairs)): - # Search for where the blend pairs have the nth unique LS value - idx = np.where(uni_blends == LS_pairs[pp])[0] - if len(idx) > 0: - start = uni_bidxx[idx][0] - end = uni_bidxx[idx + 1][0] - app_blended, flux_N_tot, cent_l, cent_b = calc_blend_and_centroid(filter_name, - red_law, - blend_tab[start:end], - photometric_system) - flux_N[pp] = flux_N_tot - event_tab['cent_glon_' + filter_name + '_N'][pp] = cent_l - event_tab['cent_glat_' + filter_name + '_N'][pp] = cent_b + line0 = 'NEW BLEND RADIUS INFO AND FILES CREATED' + '\n' + line1 = 'new blend rad : ' + str(new_blend_rad) + ' (arcsec)' + '\n' + line2 = 'blend file : ' + new_blend_tab_name + '\n' + line3 = 'event file : ' + new_event_tab_name + '\n' + line4 = 'creation time: ' + str(now) + '\n' - # Total blended flux in i-band - flux_tot = flux_L + flux_S + flux_N + with open(new_event_log_name, 'a') as my_file: + my_file.writelines(['\n', line0, dash_line, + line1, line2, line3, line4]) - # Total blended magnitude, neighbor - app_N = np.zeros(len(flux_N)) - bad_N_idx = np.where(flux_N == 0)[0] - good_N_idx = np.where(flux_N != 0)[0] - app_N[good_N_idx] = -2.5 * np.log10(flux_N[good_N_idx]) - app_N[bad_N_idx] = np.full(len(bad_N_idx), np.nan) + old_blends = Table.read(blend_tab) + good_idx = np.where(old_blends['sep_LN'] < new_blend_rad)[0] + new_blends = old_blends[good_idx] - event_tab[photometric_system + '_' + filter_name + '_app_N'] = app_N + new_blends.write(new_blend_tab_name, overwrite=overwrite) - # Total blended magnitude - app_LSN = -2.5 * np.log10(flux_tot) - event_tab[photometric_system + '_' + filter_name + '_app_LSN'] = app_LSN + return - # Bump amplitude (in magnitudes) - delta_m = calc_bump_amp(event_tab['u0'], flux_S, flux_L, flux_N) - event_tab['delta_m_' + filter_name] = delta_m - # Calculate the blend fraction - # (commonly used in microlensing): f_source / f_total - f_blend = flux_S / flux_tot - event_tab['f_blend_' + filter_name] = f_blend +############################################################################ +######### Refined event rate calculation and associated functions ########## +############################################################################ - return +def _convert_photometric_99_to_nan(table, photometric_system='ubv'): + for name in table.colnames: + if ('exbv' in name) or (photometric_system in name): + cond = np.where(table[name] == -99)[0] + table[name][cond] = np.nan -################################################################## -############ Reading/writing and format functions ############### -################################################################## -def make_ebf_log(ebf_table): +def _check_refine_events(input_root, filter_name, + photometric_system, red_law, overwrite, + output_file): """ - Converts log from Galaxia ebf output into dictionary + Checks that the inputs of refine_events are valid Parameters ---------- - ebf_table : processed ebf file - The ebf file that galaxia outputs, AFTER it's been read + input_root : str + The root path and name of the *_events.fits and *_blends.fits. + Don't include those suffixes yet. - Returns - ------- - ebf_log : dictionary - The ebf file log, in dictionary form + filter_name : str + The name of the filter in which to calculate all the + microlensing events. The filter name convention is set + in the global filt_dict parameter at the top of this module. - """ - if type(ebf_table) == dict: - log_list = ebf_table['log'][0].split(b'\n') + photometric_system : str + The name of the photometric system in which the filter exists. - if ebf_table[-4:] == '.ebf': - log_list = ebf.read(ebf_table, '/log')[0].split(b'\n') + red_law : str + The name of the reddening law to use from PopStar. - if type(ebf_table) == np.bytes_: - log_list = ebf_table.split(b'\n') + overwrite : bool + If set to True, overwrites output files. If set to False, exists the + function if output files are already on disk. + Default is False. + """ + if type(input_root) != str: + raise Exception('input_root (%s) must be a string.' % str(input_root)) - ebf_log = {} - for ii in range(len(log_list)): - if len(log_list[ii]) == 0: - continue + if type(filter_name) != str: + raise Exception('filter_name (%s) must be a string.' % str(filter_name)) - if log_list[ii].startswith(b'#'): - continue + if type(photometric_system) != str: + raise Exception('photometric_system (%s) must be a string.' % str(photometric_system)) - log_list_entry = log_list[ii].split() + if type(red_law) != str: + raise Exception('red_law (%s) must be a string.' % str(red_law)) - ebf_log[log_list_entry[0].decode('utf-8')] = log_list_entry[1].decode( - 'utf-8') + if type(output_file) != str: + raise Exception('output_file (%s) must be a string.' % str(output_file)) - return ebf_log + if type(overwrite) != bool: + raise Exception('overwrite (%s) must be a boolean.' % str(overwrite)) + # Check to see that the filter name, photometric system, red_law are valid + if photometric_system not in photometric_system_dict: + exception_str = 'photometric_system must be a key in ' \ + 'photometric_system_dict. \n' \ + 'Acceptable values are : ' + for photometric_system in photometric_system_dict: + exception_str += '%s, ' % photometric_system + exception_str = exception_str[:-2] + raise Exception(exception_str) -def make_label_file(h5file_name, overwrite=False): - """ - Given an output root for an .h5 file, creates a table of - dataset name, l, b, and number of objects. - Writes out the Astropy table as a .fits file. + if filter_name not in photometric_system_dict[photometric_system]: + exception_str = 'filter_name must be a value in ' \ + 'photometric_system_dict[%s]. \n' \ + 'Acceptable values are : ' % photometric_system + for filter_name in photometric_system_dict[photometric_system]: + exception_str += '%s, ' % filter_name + exception_str = exception_str[:-2] + raise Exception(exception_str) - Parameters - ---------- - h5file_name : string - The path and name of the input h5file containing the - population synthesis results. We will read this in and - make a new output file entitled: - _label.fits (after removing the *.h5). + key = photometric_system + '_' + filter_name + if red_law not in filt_dict[key]: + exception_str = 'red_law must be a value in ' \ + 'filt_dict[%s]. \n' \ + 'Acceptable values are : ' % key + for red_law in filt_dict[key]: + exception_str += '%s, ' % red_law + exception_str = exception_str[:-2] + raise Exception(exception_str) - Return - ------ - label_file : Astropy table - Table containing the dataset name, and corresponding l, b, and number of objects. +def refine_events(input_root, filter_name, photometric_system, red_law, + overwrite=False, + output_file='default'): """ - data_dict = {'file_name': [], 'long_start': [], 'long_end': [], - 'lat_start': [], 'lat_end': [], 'objects': [], - 'N_stars': [], 'N_WD': [], 'N_NS': [], 'N_BH': []} + Takes the output Astropy table from calc_events, and from that + calculates the time of closest approach. Will also return source-lens + separation at this time. - hf = h5py.File(h5file_name + '.h5', 'r') - l_array = hf['long_bin_edges'] - b_array = hf['lat_bin_edges'] + Parameters + ---------- + input_root : str + The root path and name of the *_events.fits and *_blends.fits. + Don't include those suffixes yet. - for ll in range(len(l_array) - 1): - for bb in range(len(b_array) - 1): - dset_name = 'l' + str(ll) + 'b' + str(bb) - dataset = hf[dset_name] + filter_name : str + The name of the filter in which to calculate all the + microlensing events. The filter name convention is set + in the global filt_dict parameter at the top of this module. - if len(dataset) == 0: - N_stars = 0 - N_WD = 0 - N_NS = 0 - N_BH = 0 - else: - N_stars = np.sum(dataset['rem_id'] == 0) - N_WD = np.sum(dataset['rem_id'] == 101) - N_NS = np.sum(dataset['rem_id'] == 102) - N_BH = np.sum(dataset['rem_id'] == 103) + photometric_system : str + The name of the photometric system in which the filter exists. - data_dict['file_name'].append(dset_name) - data_dict['long_start'].append(l_array[ll]) - data_dict['long_end'].append(l_array[ll + 1]) - data_dict['lat_start'].append(b_array[bb]) - data_dict['lat_end'].append(b_array[bb + 1]) - data_dict['objects'].append(dataset.shape[0]) - data_dict['N_stars'].append(N_stars) - data_dict['N_WD'].append(N_WD) - data_dict['N_NS'].append(N_NS) - data_dict['N_BH'].append(N_BH) + red_law : str + The name of the reddening law to use from PopStar. - hf.close() + Optional Parameters + ------------------- + overwrite : bool + If set to True, overwrites output files. If set to False, exists the + function if output files are already on disk. + Default is False. - label_file = Table(data_dict, names=('file_name', 'long_start', 'long_end', - 'lat_start', 'lat_end', 'objects', - 'N_stars', 'N_WD', 'N_NS', 'N_BH')) - label_file['long_start'].format = '08.3f' - label_file['long_end'].format = '08.3f' - label_file['lat_start'].format = '07.3f' - label_file['lat_end'].format = '07.3f' + output_file : str + The name of the final refined_events file. + If set to 'default', the format will be: + _refined_events___.fits - now = datetime.datetime.now() - label_file.meta['label'] = 'label.fits file creation time: ' + str(now) + Output: + ---------- + A file will be created named + _refined_events___.fits + that contains all the same objects, only now with lots of extra + columns of data. - label_file.write(h5file_name + '_label.fits', overwrite=overwrite) + """ + # Check if .fits file exists already. If it does, throw an error message + # to complain and exit. + if not overwrite and os.path.isfile(output_file): + raise Exception('That refined_events.fits file name is taken! ' + 'Either delete the .fits file, or pick a new name.') - return + # Error handling/complaining if input types are not right. + _check_refine_events(input_root, filter_name, + photometric_system, red_law, + overwrite, output_file) + if output_file == 'default': + output_file = '{0:s}_refined_events_{1:s}_{2:s}_{3:s}.fits'.format(input_root, + photometric_system, + filter_name, + red_law) -########################################################################### -############ General formulas, conversions, and calculations ############## -########################################################################### + t_0 = time.time() -def heliocentric_to_galactic(x, y, z): - """ - Converts from heliocentric coordinates to galactic coordinates. + event_fits_file = input_root + '_events.fits' + blend_fits_file = input_root + '_blends.fits' + log_file = input_root + '_calc_events.log' - Parameters - ---------- - x, y, z : float or array - Heliocentric coordinates x, y, and z (in kpc) + event_tab = Table.read(event_fits_file) + blend_tab = Table.read(blend_fits_file) - Returns - ------- - r, b, l : float or array - Galactic coordinates r, b, and l (in kpc and degrees) + # If photometric fields contain -99, convert to nan + _convert_photometric_99_to_nan(event_tab, photometric_system) + _convert_photometric_99_to_nan(blend_tab, photometric_system) - """ - r = (x ** 2 + y ** 2 + z ** 2) ** 0.5 + # Only keep events with luminous sources + event_tab = event_tab[~np.isnan(event_tab[photometric_system + '_' + filter_name + '_S'])] - # np.arccos will return a value between 0 and pi - # so b will be between -90 and 90 degrees - b = -np.degrees(np.arccos(z / r)) + 90 + with open(log_file) as my_file: + for num, line in enumerate(my_file): + if 'obs_time' in line: + obs_time = line.split(',')[1] + obs_time = float(obs_time) + break - # np.arctan2 will return a value between -pi and pi - # arctan2 takes care of choosing the correct branch of the arctan function - # %360 will return it between 0 and 360 degrees - l = np.degrees((np.arctan2(y, x))) % 360 + # Calculate time and separation at closest approach, add to table + # NOTE: calc_closest_approach modifies the event table! + # It trims out events that peak outside the survey range! + N_events_original = len(event_tab) + print('Original candidate events: ', N_events_original) + u0, t0 = calc_closest_approach(event_tab, obs_time) + N_events_survey = len(event_tab) + print('Candidate events in survey window: ', N_events_survey) + event_tab['t0'] = t0 # days + event_tab['u0'] = u0 + if len(event_tab) == 0: + print('No events!') + return - return r, b, l + ########## + # Calculate apparent magnitudes + ########## + # Einstein crossing time + # THIS HAS TO GO BEFORE _CALC_OBSERVABLES + t_E = event_tab['theta_E'] / event_tab['mu_rel'] # yr + t_E *= 365.25 # days + event_tab['t_E'] = t_E # days -def galactic_to_heliocentric(r, b, l): - """ - Converts from galactic coordinates to heliocentric coordinates. + # Add stuff to event_tab... shouldn't have any direct outputs + _calc_observables(filter_name, red_law, event_tab, blend_tab, photometric_system) - Parameters - --------- - r, b, l : float or array - Galactic coordinates r, b and l (in kpc and degrees) + # Relative parallax + pi_rel = event_tab['rad_L'] ** -1 - event_tab['rad_S'] ** -1 + event_tab['pi_rel'] = pi_rel # mas - Returns - ------- - x, y, z : float or array - Heliocentric coordinates x, y, and z (in kpc) + # Microlensing parallax + pi_E = pi_rel / event_tab['theta_E'] + event_tab['pi_E'] = pi_E # dim'less - """ - # Convert input from degrees to radians. - l = np.radians(l) - b = np.radians(b) + event_tab.write(output_file, overwrite=overwrite) - x = r * np.cos(b) * np.cos(l) - y = r * np.cos(b) * np.sin(l) - z = r * np.sin(b) + t_1 = time.time() - return x, y, z + ########## + # Make log file + ########## + now = datetime.datetime.now() + microlens_path = os.path.dirname(inspect.getfile(perform_pop_syn)) + popstar_path = os.path.dirname(inspect.getfile(imf)) + microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=microlens_path).decode('ascii').strip() + popstar_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=popstar_path).decode('ascii').strip() + dash_line = '-----------------------------' + '\n' + empty_line = '\n' + line0 = 'FUNCTION INPUT PARAMETERS' + '\n' + line1 = 'input_root : ' + input_root + '\n' + line2 = 'filter_name : ' + filter_name + '\n' + line3 = 'red_law : ' + red_law + '\n' -def einstein_radius(M, d_L, d_S): - """ - Calculates the einstein radius, in mas + line4 = 'VERSION INFORMATION' + '\n' + line5 = str(now) + ' : creation date' + '\n' + line6 = popstar_hash + ' : PopStar commit' + '\n' + line7 = microlens_hash + ' : microlens commit' + '\n' - Parameters - ---------- - M : float - Lens mass, in solar masses + line8 = 'OTHER INFORMATION' + '\n' + line9 = str(t_1 - t_0) + ' : total runtime (s)' + '\n' + line10 = str(N_events_original) + ' : original candidate events (dark sources removed)' + '\n' + line11 = str(N_events_survey) + ' : candidate events in survey window' + '\n' - d_L : float - Distance to lens, in kpc + line12 = 'FILES CREATED' + '\n' + line13 = output_file + ' : refined events' - d_S : float - Distance to source, in kpc + with open(input_root + '_refined_events_' + photometric_system + '_' + filter_name + '_' + red_law + '.log', 'w') as out: + out.writelines([line0, dash_line, line1, line2, line3, empty_line, + line4, dash_line, line5, line6, line7, empty_line, + line8, dash_line, line9, line10, line11, empty_line, + line12, dash_line, line13]) - Return - ------ - Einstein radius, in mas - """ - return 2.85 * M ** 0.5 * (1 / d_L - 1 / d_S) ** 0.5 + print('Total runtime: {0:f} s'.format(t_1 - t_0)) + return -def calc_sph_motion(vx, vy, vz, r, b, l): +def calc_closest_approach(event_tab, survey_duration): """ - Calculate velocities in the r directions and proper motions - in l, b directions. + Calculate the point of closest approach, during a given interval. + Return the position and time of closest approach. + This algorithm comes from a little writeup called + "Point of Closest Approach for the Linear Motion of Two Particles in 2-D" Parameters ---------- - vx, vy, vz : float or array - Heliocentric velocities vx, vy, and vz (in km/s) + event_tab : astropy table + This should take the astropy table that comes from the + output of calc_events. - r, b, l : float or array - Galactic coordinates r, b, and l (in kpc and degrees) + survey_duration : int or float + Survey duration, in days Return ------ - vr, mu_b, mu_lcosb : float or array (in km/s and mas/yr) - Radial velocity and proper motions - - """ - b = np.radians(b) - l = np.radians(l) - - cosb = np.cos(b) - sinb = np.sin(b) - cosl = np.cos(l) - sinl = np.sin(l) - - vr = vz * sinb + cosb * (vx * cosl + vy * sinl) - vb = vz * cosb - sinb * (vx * cosl + vy * sinl) - vl = vy * cosl - vx * sinl - - mu_b = vb / r - mu_lcosb = vl / r - - ########## - # Convert between radians*(km/s)/kpc into mas/year. - # 1 kpc = 3.086 * 10^16 km, 1 year = 3.154 * 10^7 s, 2.063 * 10^8 mas = 1 rad - # 1 radian*(km/s)/kpc = 0.2108 mas/yr - ######### - conversion_fact = 0.2108 - mu_b = mu_b * conversion_fact - mu_lcosb = mu_lcosb * conversion_fact - - return vr, mu_b, mu_lcosb + u0 : array (same length as event_tab) + Minimum separation (normalized to Einstein radius) + t0 : array (same length as event_tab) + Time at minimum separation (days) -def calc_normalized_counts(mag): - """ - CHANGE THIS CODE IN THE FUTURE TO TAKE IN DIFFERENT ZERO POINTS! - Right now this is only applicable for OGLE I band - See confluence wiki for where these values come from... """ - a = -0.67815949 - b = 15.37993393 - counts = 10 ** (b + a * mag) - return counts + # Survey start and end time + start_time = -1 * survey_duration / 2.0 # days + end_time = survey_duration / 2.0 # days + # Parametrization coefficients + cos_glat_S = np.cos(np.radians(event_tab['glat_S'])) + cos_glat_L = np.cos(np.radians(event_tab['glat_L'])) -def calc_magnification(u): - """ - Calculate the magnification factor A(u) + l_0 = (event_tab['glon_S'] - event_tab['glon_L']) * cos_glat_L * 3600.0 * 1.0e3 # position, mas + l_1 = (event_tab['mu_lcosb_S'] - event_tab['mu_lcosb_L']) # pm, mas/yr + b_0 = (event_tab['glat_S'] - event_tab['glat_L']) * 3600.0 * 1.0e3 # position, mas + b_1 = (event_tab['mu_b_S'] - event_tab['mu_b_L']) # pm, mas/yr - Parameters - ---------- - u : float or array - Dimensionless lens-source separation, normalized in Einstein radius units + # Calculate the critical time (time of closest approach), global minimum + t_crit = -(l_0 * l_1 + b_0 * b_1) / (l_1 ** 2 + b_1 ** 2) # years + t_crit *= 365.25 # days - Return - ------ - Magnification factor : float or array + # Only include events that peak in survey window, + # since we can't easily fit parameters for events where we don’t have the peak. + # FIXME: more efficient way to do this?? + good = np.where((t_crit > start_time) & (t_crit < end_time))[0] + bad = np.where((t_crit <= start_time) | (t_crit >= end_time))[0] - """ - return (u ** 2 + 2) / (u * np.sqrt(u ** 2 + 4)) + event_tab.remove_rows(bad) + # Initialize the time of closest approach vector, minimum of survey + # NOTE: THIS WILL OVERWRITE CURRENT VALUES + t0 = np.zeros(len(event_tab)) + u0 = np.zeros(len(event_tab)) -def calc_delta_c(u, thetaE): - """ - Calculate the maximum centroid shift for a dark lens, - no neighbors - """ - delta_c = u * thetaE / (u ** 2 + 2) + # Assign values to closest approach values + t0 = t_crit[good] + u0 = calc_distance(event_tab, t0) - return delta_c + return u0, t0 -def calc_delta_c_LL(fratio, u0, thetaE): +def calc_distance(event_tab, time): """ - Calculate the maximum-ish centroid shift for a luminous - lens, no neighbors + Calculate the separation of two different objects at some given time. Parameters ---------- - fratio : flux ratio of the lens to source, i.e. f_L/f_S + event_tab : astropy table - u0 : impact parameter + time : int/float - thetaE : Einstein radius + Return + ------ + u : array (same length as astropy table) """ - final_delta_array = np.zeros(len(u0)) - final_u_array = np.zeros(len(u0)) - for j in np.arange(len(u0)): - f = fratio[j] - u_array = np.linspace(0, u0[j], 20) - delta_array = np.zeros(len(u_array)) - for i in np.arange(len(u_array)): - u = u_array[i] - delta_array[i] = (u - f * u ** 2 * np.sqrt(u ** 2 + 4)) / (2 + u ** 2 + f * u * np.sqrt(u ** 2 + 4)) + (u * f) / (1 + f) - max_idx = np.argmax(delta_array) - final_delta_array[j] = delta_array[max_idx] - final_u_array[j] = u_array[max_idx] - final_delta_array = final_delta_array * thetaE - - return final_delta_array, final_u_array - + # Calculate sep + cos_glat_S = np.cos(np.radians(event_tab['glat_S'])) + cos_glat_L = np.cos(np.radians(event_tab['glat_L'])) -def max_delta_c(u0, thetaE): - max_delta_c_array = np.zeros(len(u0)) + l_L = (event_tab['glon_L'] + time * event_tab['mu_lcosb_L'] * masyr_to_degday / cos_glat_L) + b_L = (event_tab['glat_L'] + time * event_tab['mu_b_L'] * masyr_to_degday) + l_S = (event_tab['glon_S'] + time * event_tab['mu_lcosb_S'] * masyr_to_degday / cos_glat_S) + b_S = (event_tab['glat_S'] + time * event_tab['mu_b_S'] * masyr_to_degday) - big_idx = np.where(u0 > np.sqrt(2))[0] - small_idx = np.where(u0 <= np.sqrt(2))[0] + c_L = SkyCoord(frame='galactic', l=l_L * units.deg, b=b_L * units.deg) + c_S = SkyCoord(frame='galactic', l=l_S * units.deg, b=b_S * units.deg) - max_delta_c_array[big_idx] = calc_delta_c(u0[big_idx], thetaE[big_idx]) - max_delta_c_array[small_idx] = calc_delta_c(np.ones(len(small_idx)) * np.sqrt(2), - thetaE[small_idx]) + sep = (c_L.separation(c_S)).mas + u = sep / event_tab['theta_E'] - return max_delta_c_array + return u -def get_u_from_t(u0, t0, tE, t): +def calc_blend_and_centroid(filter_name, red_law, blend_tab, photometric_system='ubv'): """ - Given the time and separation at closest approach of lens and source - and the Einstein radius, calculate the separation as a function of time. - - NOTE 1: You need to be consistent with your units for t0, tE, and t, - i.e. pick whatever you want (days, years, etc.) but be self consistent. - - NOTE 2: There is a positive and negative solution for u. - We return the positive solution. - - Parameters - ---------- - u0 : float or array - Minimum separation of lens and source (normalized to Einstein radius) + Given the absolute magnitudes of a bunch of things, + calculate their blended apparent magnitude and flux. + Also calculate the centroid of these things. - t0 : float or array - Time of minimum separation of lens and source + Filter name is 'j', 'i', etc. + red_law is Damineli16, Schlegel99, etc. + """ + f_i = filt_dict[photometric_system + '_' + filter_name][red_law] - tE : float or array - Einstein crossing time of microlensing event + # Calculate apparent magnitudes + app_N = calc_app_mag(blend_tab['rad_N'], + blend_tab[photometric_system + '_' + filter_name + '_N'], + blend_tab['exbv_N'], f_i) - t : float or array - Time at which you want to calculate the separation + # Convert absolute magnitudes to fluxes, and fix bad values + flux_N = 10 ** (app_N / -2.5) + flux_N = np.nan_to_num(flux_N) - Return - ------ - u : float or array - Separation of lens and source (normalized to Einstein radius) - """ - tau = (t - t0) / tE - u = np.sqrt(u0 ** 2 + tau ** 2) - return u + # Get total flux + flux_N_tot = np.sum(flux_N) + # Get centroid + # If there is no flux, then the centroid is set to (l, b) = (0, 0). + if flux_N_tot == 0: + cent_l = 0.0 + cent_b = 0.0 + else: + cent_l = np.sum(flux_N * blend_tab['glon_N']) / flux_N_tot + cent_b = np.sum(flux_N * blend_tab['glat_N']) / flux_N_tot -def get_t_from_u(u0, t0, tE, u): - """ - Given the time and separation at closest approach of lens and source - and the Einstein radius, calculate the time as a function of separation. + # Total blended magnitude + app_blended = -2.5 * np.log10(flux_N_tot) - NOTE 1: You need to be consistent with your units for t0, tE, and t, - i.e. pick whatever you want (days, years, etc.) but be self consistent. + return app_blended, flux_N_tot, cent_l, cent_b - NOTE 2: There is a positive and negative solution for t. - We return the positive solution. + +def _calc_observables(filter_name, red_law, event_tab, blend_tab, photometric_system='ubv'): + """ + Calculate a bunch of observable quantities we get out from microlensing Parameters ---------- - u0 : float or array - Minimum separation of lens and source (normalized to Einstein radius) - - t0 : float or array - Time of minimum separation of lens and source + filter_name : str - tE : float or array - Einstein crossing time of microlensing event + event_tab : - u : float or array - Separation of lens and source (normalized to Einstein radius) + blend_tab : Return ------ - t : float or array - Time corresponding to the separation u """ - t = tE * np.sqrt(u ** 2 - u0 ** 2) + t0 - return t + f_i = filt_dict[photometric_system + '_' + filter_name][red_law] + # Calculate apparent magnitude of lens and source, and fix bad values + app_S = calc_app_mag(event_tab['rad_S'], + event_tab[photometric_system + '_' + filter_name + '_S'], + event_tab['exbv_S'], f_i) + app_L = calc_app_mag(event_tab['rad_L'], + event_tab[photometric_system + '_' + filter_name + '_L'], + event_tab['exbv_L'], f_i) + event_tab[photometric_system + '_' + filter_name + '_app_S'] = app_S + event_tab[photometric_system + '_' + filter_name + '_app_L'] = app_L -def calc_new_position(l0, b0, mu_lcosb, mu_b, t): - """ - Given an initial position and proper motions in l and b, - calculate the new position at some later time. + # Convert absolute magnitude to fluxes, and fix bad values + flux_L = 10 ** (app_L / -2.5) + flux_S = 10 ** (app_S / -2.5) - Parameters - ---------- - l0 : float or array - Initial longitude, in DEGREES + flux_L = np.nan_to_num(flux_L) + flux_S = np.nan_to_num(flux_S) - b0 : float or array - Initial latitude, in DEGREES + # Find the blends. + LS_pairs = np.stack((event_tab['obj_id_L'], event_tab['obj_id_S']), + axis=-1) + blend_pairs = np.stack((blend_tab['obj_id_L'], blend_tab['obj_id_S']), + axis=-1) - mu_lcosb : float or array - Longitudinal proper motion l * cos(b), in MAS/YEAR + flux_N = np.zeros(len(app_L)) + event_tab['cent_glon_' + filter_name + '_N'] = np.zeros(len(app_L)) + event_tab['cent_glat_' + filter_name + '_N'] = np.zeros(len(app_L)) - mu_b : float or array - Latitudinal roper motion, in MAS/YEAR + # Get the unique blend_pairs and the first instance of each. + if len(blend_pairs) > 0: + uni_blends, uni_bidx = np.unique(blend_pairs, return_index=True, + axis=0) + # Add a "dummy" idx onto uni_bidx so that I can do idx+1 later. + uni_bidxx = np.append(uni_bidx, len(blend_pairs)) + else: + # obj_id always >= 0, so these negative values serve as dummies that + # cannot be matched in the below `where` statement + uni_blends = np.array([[-1, -1]]) + uni_bidxx = np.array([]) - t : float or array - Time, in DAYS + # FIXME: Put in a check that uni_bidx is monotonically increasing??? + # np.sort(uni_bidx) - uni_bidx == np.zeros(len(uni_bidx)) ??? - Return - ------ - l : float or array - Latitude, in DEGREES + # FIXME: This seems more complicated than necessary... + for pp in range(len(LS_pairs)): + # Search for where the blend pairs have the nth unique LS value + idx = np.where(uni_blends == LS_pairs[pp])[0] + if len(idx) > 0: + start = uni_bidxx[idx][0] + end = uni_bidxx[idx + 1][0] + app_blended, flux_N_tot, cent_l, cent_b = calc_blend_and_centroid(filter_name, + red_law, + blend_tab[start:end], + photometric_system) + flux_N[pp] = flux_N_tot + event_tab['cent_glon_' + filter_name + '_N'][pp] = cent_l + event_tab['cent_glat_' + filter_name + '_N'][pp] = cent_b - b : float or array - Longitude, in DEGREES - """ + # Total blended flux in i-band + flux_tot = flux_L + flux_S + flux_N - cos_b0 = np.cos(np.radians(b0)) + # Total blended magnitude, neighbor + app_N = np.zeros(len(flux_N)) + bad_N_idx = np.where(flux_N == 0)[0] + good_N_idx = np.where(flux_N != 0)[0] + app_N[good_N_idx] = -2.5 * np.log10(flux_N[good_N_idx]) + app_N[bad_N_idx] = np.full(len(bad_N_idx), np.nan) - l = l0 + t * mu_lcosb * masyr_to_degday / cos_b0 - b = b0 + t * mu_b * masyr_to_degday + event_tab[photometric_system + '_' + filter_name + '_app_N'] = app_N - return l, b + # Total blended magnitude + app_LSN = -2.5 * np.log10(flux_tot) + event_tab[photometric_system + '_' + filter_name + '_app_LSN'] = app_LSN + + # Bump amplitude (in magnitudes) + delta_m = calc_bump_amp(event_tab['u0'], flux_S, flux_L, flux_N) + event_tab['delta_m_' + filter_name] = delta_m + + # Calculate the blend fraction + # (commonly used in microlensing): f_source / f_total + f_blend = flux_S / flux_tot + event_tab['f_blend_' + filter_name] = f_blend + return -def calc_centroid_shift(glat_S, glon_S, glat_N, glon_N, f_L, f_S, f_N, u): + +################################################################## +############ Reading/writing and format functions ############### +################################################################## + +def make_ebf_log(ebf_table): """ - Calculate the centroid (astrometric) shift - for a luminous lens and neighbors + Converts log from Galaxia ebf output into dictionary Parameters ---------- - glat_x : float or array - Longitude of x (L = lens, S = source, N = neighbor centroid) - - glon_x : float or array - Latitude of x (L = lens, S = source, N = neighbor centroid) - - f_x : float or array - Flux of x (L = lens, S = source, N = all neighbors) + ebf_table : processed ebf file + The ebf file that galaxia outputs, AFTER it's been read - u : float or array - Dimensionless separation + Returns + ------- + ebf_log : dictionary + The ebf file log, in dictionary form - Return - ------ - delta_c_obs : float or array - Magnitude of observed astrometric shift, in mas """ - ########## - # Calculating the centroid shift in the frame of the lens - ########## + if type(ebf_table) == dict: + log_list = ebf_table['log'][0].split(b'\n') - t1a1_t2a2 = (u ** 2 + 3) / (u * np.sqrt(u ** 2 + 4)) - a1_a2 = calc_magnification(u) + if ebf_table[-4:] == '.ebf': + log_list = ebf.read(ebf_table, '/log')[0].split(b'\n') - glat_c_lensed = (t1a1_t2a2 * f_S * glat_S + glat_N * f_N) / ( - a1_a2 * f_S + f_L + f_N) - glon_c_lensed = (t1a1_t2a2 * f_S * glon_S + glon_N * f_N) / ( - a1_a2 * f_S + f_L + f_N) + if type(ebf_table) == np.bytes_: + log_list = ebf_table.split(b'\n') - glat_c_unlensed = (glat_S * f_S + glat_N * f_N) / (f_S + f_L + f_N) - glon_c_unlensed = (glon_S * f_S + glon_N * f_N) / (f_S + f_L + f_N) + ebf_log = {} + for ii in range(len(log_list)): + if len(log_list[ii]) == 0: + continue - glat_delta_c = glat_c_lensed - glat_c_unlensed - glon_delta_c = glon_c_lensed - glon_c_unlensed + if log_list[ii].startswith(b'#'): + continue - delta_c_obs = np.sqrt(glat_delta_c ** 2 + glon_delta_c ** 2 * np.cos( - np.radians(glat_S)) ** 2) + log_list_entry = log_list[ii].split() - # Convert from degrees to mas - delta_c_obs *= 3.6 * 10 ** 6 + ebf_log[log_list_entry[0].decode('utf-8')] = log_list_entry[1].decode( + 'utf-8') - return delta_c_obs + return ebf_log -def calc_bump_amp(u0, f_S, f_L, f_N): +def make_label_file(h5file_name, overwrite=False): """ - Calculate the "bump" amplitude, given the minimum separation and the fluxes - of the (unmagnified) source, lens, and neighbors. - The bump amplitude (mags) is: - |m_peak - m_base| = -2.5 log10 ((A(u0) * f_S + f_L + f_N)/(f_S + f_L + f_N)) + Given an output root for an .h5 file, creates a table of + dataset name, l, b, and number of objects. + Writes out the Astropy table as a .fits file. Parameters ---------- - u0 : float or array - Dimensionless source-lens angular separation, closest approach - f_S : float or array - Flux from source (arbitrary units) - f_L : float or array - Flux from lens (arbitrary units) - f_N : float or array - Flux from neighbors (arbitrary units) + h5file_name : string + The path and name of the input h5file containing the + population synthesis results. We will read this in and + make a new output file entitled: + _label.fits (after removing the *.h5). Return ------ - m_bump : float or array - Bump magnitude + label_file : Astropy table + Table containing the dataset name, and corresponding l, b, and number of objects. + """ - A = calc_magnification(u0) - m_bump = 2.5 * np.log10((A * f_S + f_L + f_N) / (f_S + f_L + f_N)) + data_dict = {'file_name': [], 'long_start': [], 'long_end': [], + 'lat_start': [], 'lat_end': [], 'objects': [], + 'N_stars': [], 'N_WD': [], 'N_NS': [], 'N_BH': []} + + hf = h5py.File(h5file_name + '.h5', 'r') + l_array = hf['long_bin_edges'] + b_array = hf['lat_bin_edges'] + + for ll in range(len(l_array) - 1): + for bb in range(len(b_array) - 1): + dset_name = 'l' + str(ll) + 'b' + str(bb) + dataset = hf[dset_name] + + if len(dataset) == 0: + N_stars = 0 + N_WD = 0 + N_NS = 0 + N_BH = 0 + else: + N_stars = np.sum(dataset['rem_id'] == 0) + N_WD = np.sum(dataset['rem_id'] == 101) + N_NS = np.sum(dataset['rem_id'] == 102) + N_BH = np.sum(dataset['rem_id'] == 103) + + data_dict['file_name'].append(dset_name) + data_dict['long_start'].append(l_array[ll]) + data_dict['long_end'].append(l_array[ll + 1]) + data_dict['lat_start'].append(b_array[bb]) + data_dict['lat_end'].append(b_array[bb + 1]) + data_dict['objects'].append(dataset.shape[0]) + data_dict['N_stars'].append(N_stars) + data_dict['N_WD'].append(N_WD) + data_dict['N_NS'].append(N_NS) + data_dict['N_BH'].append(N_BH) + + hf.close() + + label_file = Table(data_dict, names=('file_name', 'long_start', 'long_end', + 'lat_start', 'lat_end', 'objects', + 'N_stars', 'N_WD', 'N_NS', 'N_BH')) + label_file['long_start'].format = '08.3f' + label_file['long_end'].format = '08.3f' + label_file['lat_start'].format = '07.3f' + label_file['lat_end'].format = '07.3f' - return m_bump + now = datetime.datetime.now() + label_file.meta['label'] = 'label.fits file creation time: ' + str(now) + + label_file.write(h5file_name + '_label.fits', overwrite=overwrite) + + return -def calc_app_mag(r, M, E, f): +########################################################################### +############ General formulas, conversions, and calculations ############## +########################################################################### + +def heliocentric_to_galactic(x, y, z): """ - Calculate the apparent magnitude (i.e. distance modulus - plus extinction) + Converts from heliocentric coordinates to galactic coordinates. Parameters ---------- - M : float or array - Absolute magnitude of star - - r : float or array - Distance of star from sun (in kpc) + x, y, z : float or array + Heliocentric coordinates x, y, and z (in kpc) - E : float or array - Extinction law + Returns + ------- + r, b, l : float or array + Galactic coordinates r, b, and l (in kpc and degrees) - f : float or array - Coefficient for that particular band or whatever + """ + r = (x ** 2 + y ** 2 + z ** 2) ** 0.5 - Return - ------ - m : float or array - Apparent magnitude of star + # np.arccos will return a value between 0 and pi + # so b will be between -90 and 90 degrees + b = -np.degrees(np.arccos(z / r)) + 90 - """ - m = calc_DM(r, M) + calc_ext(E, f) + # np.arctan2 will return a value between -pi and pi + # arctan2 takes care of choosing the correct branch of the arctan function + # %360 will return it between 0 and 360 degrees + l = np.degrees((np.arctan2(y, x))) % 360 - return m + return r, b, l -def calc_DM(r, M): +def galactic_to_heliocentric(r, b, l): """ - Calculate the distance modulus: m = M + 5log10(100*r/kpc) + Converts from galactic coordinates to heliocentric coordinates. Parameters - ---------- - M : float or array - Absolute magnitude of star - - r : float or array - Distance of star from sun (in kpc) + --------- + r, b, l : float or array + Galactic coordinates r, b and l (in kpc and degrees) - Return - ------ - m : float or array - Apparent magnitude of star + Returns + ------- + x, y, z : float or array + Heliocentric coordinates x, y, and z (in kpc) """ - m = M + 5 * np.log10(100 * r) + # Convert input from degrees to radians. + l = np.radians(l) + b = np.radians(b) - return m + x = r * np.cos(b) * np.cos(l) + y = r * np.cos(b) * np.sin(l) + z = r * np.sin(b) + + return x, y, z -def calc_ext(E, f): +def einstein_radius(M, d_L, d_S): """ - Calculate the magnitude of extinction. + Calculates the einstein radius, in mas Parameters ---------- - E : float or array - Extinction law + M : float + Lens mass, in solar masses - f : float or array - Coefficient for that particular band or whatever + d_L : float + Distance to lens, in kpc + + d_S : float + Distance to source, in kpc Return ------ - m_E : float or array - Magnitude of extinction - + Einstein radius, in mas """ - m_E = E * f - - return m_E + return 2.85 * M ** 0.5 * (1 / d_L - 1 / d_S) ** 0.5 -def get_Alambda_AKs(red_law_name, lambda_eff): +def calc_sph_motion(vx, vy, vz, r, b, l): """ - Get Alambda/AKs. NOTE: this doesn't work for every law in PopStar! - Naming convention is not consistent. Change PopStar or add if statements? + Calculate velocities in the r directions and proper motions + in l, b directions. Parameters ---------- - red_law_name : str - The name of the reddening law - lambda_eff : float - Wavelength in microns + vx, vy, vz : float or array + Heliocentric velocities vx, vy, and vz (in km/s) + + r, b, l : float or array + Galactic coordinates r, b, and l (in kpc and degrees) Return ------ - Alambda_AKs : float - Alambda/AKs + vr, mu_b, mu_lcosb : float or array (in km/s and mas/yr) + Radial velocity and proper motions """ - red_law_class = getattr(reddening, 'RedLaw' + red_law_name) - red_law = red_law_class() - red_law_method = getattr(red_law, red_law_name) - Alambda_AKs = red_law_method(lambda_eff, 1) + b = np.radians(b) + l = np.radians(l) - return Alambda_AKs + cosb = np.cos(b) + sinb = np.sin(b) + cosl = np.cos(l) + sinl = np.sin(l) + vr = vz * sinb + cosb * (vx * cosl + vy * sinl) + vb = vz * cosb - sinb * (vx * cosl + vy * sinl) + vl = vy * cosl - vx * sinl -def calc_f(lambda_eff): - """ - Calculate that coefficient f that multiples E(B-V) to get the - extinction in magnitudes - """ - B = get_Alambda_AKs('Damineli16', 0.445) - V = get_Alambda_AKs('Damineli16', 0.551) - L = get_Alambda_AKs('Damineli16', lambda_eff) + mu_b = vb / r + mu_lcosb = vl / r - f = L * (B - V) ** -1 + ########## + # Convert between radians*(km/s)/kpc into mas/year. + # 1 kpc = 3.086 * 10^16 km, 1 year = 3.154 * 10^7 s, 2.063 * 10^8 mas = 1 rad + # 1 radian*(km/s)/kpc = 0.2108 mas/yr + ######### + conversion_fact = 0.2108 + mu_b = mu_b * conversion_fact + mu_lcosb = mu_lcosb * conversion_fact - return f + return vr, mu_b, mu_lcosb -def rho_dmhalo(r, rho_0=.0093, r_s=18.6, gamma=1): + +def calc_normalized_counts(mag): """ - Density profile of the dark matter halo. - We are using the parametrization from McMillan (2017) Equation 5, - with defaults based on the mean values in Table 2. - r: galactocentric radius [units: kpc] - rho_0: characteristic density in [units: m_sun / pc**3] - r_s: scale radius in [units: kpc]. - gamma: gamma=1 for NFW, gamma > 1 cuspy, gamma < 1 cored - - returns: density at r [units: m_sun / pc**3] + CHANGE THIS CODE IN THE FUTURE TO TAKE IN DIFFERENT ZERO POINTS! + Right now this is only applicable for OGLE I band + See confluence wiki for where these values come from... """ - x = r / r_s - rho = rho_0 / (x**gamma * (1 + x)**(3 - gamma)) - return rho + a = -0.67815949 + b = 15.37993393 + counts = 10 ** (b + a * mag) + return counts -def _check_add_pbh(hdf5_file, ebf_file, output_root2, - fdm, pbh_mass, - r_max, r_s, gamma, v_esc, - rho_0, n_lin, overwrite, seed): +def calc_magnification(u): """ - Checks that the inputs of add_pbj are valid + Calculate the magnification factor A(u) Parameters ---------- - hdf5_file : str or hdf5 file - str : name of the hdf5 file from the output of perform_pop_syn + u : float or array + Dimensionless lens-source separation, normalized in Einstein radius units - ebf_file : str or ebf file - str : name of the ebf file from Galaxia - ebf file : actually the ebf file from Galaxia + Return + ------ + Magnification factor : float or array - output_root2 : str - The thing you want the output files to be named - Examples: - 'myout' - '/some/path/to/myout' - '../back/to/some/path/myout' + """ + return (u ** 2 + 2) / (u * np.sqrt(u ** 2 + 4)) - fdm : float - Fraction of dark matter. - The fraction of dark matter that you want to consist of PBHs. - Defaults to 1. - pbh_mass : int - The single mass that all PBHs will have (in units of Msun). - Defaults to 40 Msun (from LIGO detections thought to be primordial) +def calc_delta_c(u, thetaE): + """ + Calculate the maximum centroid shift for a dark lens, + no neighbors + """ + delta_c = u * thetaE / (u ** 2 + 2) - r_max : float - The maximum radius from the Earth that you want to find PBHs. - Defaults to 16.6 kpc. (2 * distance to galactic center) + return delta_c - r_s: float - The scale radius of the Milky Way (in kpc). r_s = r_vir / c (virial radius / concentration index) - Defaults to 18.6 kpc. The median value given in McMillan 2017. - rho_0: float - The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). - Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2017. +def calc_delta_c_LL(fratio, u0, thetaE): + """ + Calculate the maximum-ish centroid shift for a luminous + lens, no neighbors - n_lin: int - The number of times you want the density determined along the line of sight when calculating PBH positions - Defaults to 1000. Will need to make large if you are closer to the galactic center. + Parameters + ---------- + fratio : flux ratio of the lens to source, i.e. f_L/f_S - gamma: float - The inner slope of the MW dark matter halo as described in LaCroix 2018. - Gamma goes into the determination of the velocities and each value returns a slightly different distribution. - The default value is 1, corresponding to an NFW profile. + u0 : impact parameter - v_esc: int - The escape velocity of the Milky Way (in km/s). - v_esc is used in calculating the velocities. - Default is 550 km/s. Most papers cite values of 515-575. + thetaE : Einstein radius + + """ + final_delta_array = np.zeros(len(u0)) + final_u_array = np.zeros(len(u0)) + for j in np.arange(len(u0)): + f = fratio[j] + u_array = np.linspace(0, u0[j], 20) + delta_array = np.zeros(len(u_array)) + for i in np.arange(len(u_array)): + u = u_array[i] + delta_array[i] = (u - f * u ** 2 * np.sqrt(u ** 2 + 4)) / (2 + u ** 2 + f * u * np.sqrt(u ** 2 + 4)) + (u * f) / (1 + f) + max_idx = np.argmax(delta_array) + final_delta_array[j] = delta_array[max_idx] + final_u_array[j] = u_array[max_idx] + final_delta_array = final_delta_array * thetaE - overwrite : bool - If set to True, overwrites output files. If set to False, exists the - function if output files are already on disk. - Default is False. + return final_delta_array, final_u_array - seed : int - If set to non-None, all random sampling will be seeded with the - specified seed, forcing identical output for PyPopStar and PopSyCLE. - Default None. - """ - if hdf5_file[-3:] != '.h5': - raise Exception('hdf5_file (%s) must be an ebf file.' % str(hdf5_file)) - if ebf_file[-4:] != '.ebf': - raise Exception('ebf_file (%s) must be an ebf file.' % str(ebf_file)) +def max_delta_c(u0, thetaE): + max_delta_c_array = np.zeros(len(u0)) - if type(output_root2) != str: - raise Exception('output_root2 (%s) must be a string.' % str(output_root)) + big_idx = np.where(u0 > np.sqrt(2))[0] + small_idx = np.where(u0 <= np.sqrt(2))[0] - if type(fdm) != int: - if type(fdm) != float: - raise Exception('fdm (%s) must be an integer or a float.' % str(fdm)) + max_delta_c_array[big_idx] = calc_delta_c(u0[big_idx], thetaE[big_idx]) + max_delta_c_array[small_idx] = calc_delta_c(np.ones(len(small_idx)) * np.sqrt(2), + thetaE[small_idx]) - if type(pbh_mass) != int: - if type(pbh_mass) != float: - raise Exception('pbh_mass (%s) must be an integer or a float.' % str(pbh_mass)) + return max_delta_c_array - if type(r_max) != int: - if type(r_max) != float: - raise Exception('r_max (%s) must be an integer or a float.' % str(r_max)) - if type(r_s) != int: - if type(r_s) != float: - raise Exception('r_s (%s) must be an integer or a float.' % str(r_s)) +def get_u_from_t(u0, t0, tE, t): + """ + Given the time and separation at closest approach of lens and source + and the Einstein radius, calculate the separation as a function of time. - if gamma not in [.25, .5, 1]: - raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) + NOTE 1: You need to be consistent with your units for t0, tE, and t, + i.e. pick whatever you want (days, years, etc.) but be self consistent. - if type(v_esc) != int: - if type(v_esc) != float: - raise Exception('v_esc (%s) must be an integer or a float.' % str(v_esc)) + NOTE 2: There is a positive and negative solution for u. + We return the positive solution. - if type(rho_0) != int: - if type(rho_0) != float: - raise Exception('rho_0 (%s) must be an integer or a float.' % str(rho_0)) + Parameters + ---------- + u0 : float or array + Minimum separation of lens and source (normalized to Einstein radius) - if type(n_lin) != int: - raise Exception('n_lin (%s) must be an integerr.' % str(n_lin)) + t0 : float or array + Time of minimum separation of lens and source - if type(overwrite) != bool: - raise Exception('overwrite (%s) must be a boolean.' % str(overwrite)) + tE : float or array + Einstein crossing time of microlensing event - if seed is not None: - if type(seed) != int: - raise Exception('seed (%s) must be None or an integer.' % str(seed)) + t : float or array + Time at which you want to calculate the separation + Return + ------ + u : float or array + Separation of lens and source (normalized to Einstein radius) + """ + tau = (t - t0) / tE + u = np.sqrt(u0 ** 2 + tau ** 2) + return u -def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, - r_max=16.6, r_s=18.6, gamma=1, v_esc=550, - rho_0=0.0093, n_lin=1000, overwrite=False, seed=None): +def get_t_from_u(u0, t0, tE, u): """ - Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, - and saves them in a new HDF5 file with the PBHs added. + Given the time and separation at closest approach of lens and source + and the Einstein radius, calculate the time as a function of separation. + + NOTE 1: You need to be consistent with your units for t0, tE, and t, + i.e. pick whatever you want (days, years, etc.) but be self consistent. + + NOTE 2: There is a positive and negative solution for t. + We return the positive solution. Parameters ---------- - hdf5_file : str or hdf5 file - str : name of the hdf5 file from the output of perform_pop_syn + u0 : float or array + Minimum separation of lens and source (normalized to Einstein radius) - ebf_file : str or ebf file - str : name of the ebf file from Galaxia - ebf file : actually the ebf file from Galaxia + t0 : float or array + Time of minimum separation of lens and source - output_root2 : str - The thing you want the output files to be named - Examples: - 'myout' - '/some/path/to/myout' - '../back/to/some/path/myout' + tE : float or array + Einstein crossing time of microlensing event - fdm : float - Fraction of dark matter. - The fraction of dark matter that you want to consist of PBHs. - Defaults to 1. + u : float or array + Separation of lens and source (normalized to Einstein radius) - pbh_mass : int - The single mass that all PBHs will have (in units of Msun). - Defaults to 40 Msun (from LIGO detections thought to be primordial) + Return + ------ + t : float or array + Time corresponding to the separation u + """ + t = tE * np.sqrt(u ** 2 - u0 ** 2) + t0 + return t - r_max : float - The maximum radius from the Earth that you want to find PBHs. - Defaults to 16.6 kpc. (2 * distance to galactic center) - r_s: float - The scale radius of the Milky Way (in kpc). r_s = r_vir / c (virial radius / concentration index) - Defaults to 18.6 kpc. The median value given in McMillan 2017. +def calc_new_position(l0, b0, mu_lcosb, mu_b, t): + """ + Given an initial position and proper motions in l and b, + calculate the new position at some later time. - rho_0: float - The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). - Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2017. + Parameters + ---------- + l0 : float or array + Initial longitude, in DEGREES - n_lin: int - The number of times you want the density determined along the line of sight when calculating PBH positions - Defaults to 1000. Will need to make large if you are closer to the galactic center. + b0 : float or array + Initial latitude, in DEGREES - gamma: float - The inner slope of the MW dark matter halo as described in LaCroix 2018. - Gamma goes into the determination of the velocities and each value returns a slightly different distribution. - The default value is 1, corresponding to an NFW profile. + mu_lcosb : float or array + Longitudinal proper motion l * cos(b), in MAS/YEAR - v_esc: int - The escape velocity of the Milky Way (in km/s). - v_esc is used in calculating the velocities. - Default is 550 km/s. Most papers cite values of 515-575. + mu_b : float or array + Latitudinal roper motion, in MAS/YEAR - Optional Parameters - ------------------- - overwrite : bool - If set to True, overwrites output files. If set to False, exists the - function if output files are already on disk. - Default is False. + t : float or array + Time, in DAYS - seed : int - If set to non-None, all random sampling will be seeded with the - specified seed, forcing identical output for PyPopStar and PopSyCLE. - Default None. + Return + ------ + l : float or array + Latitude, in DEGREES - Outputs - ------- - .h5 : hdf5 file - The new .h5 file with PBHs injected in. + b : float or array + Longitude, in DEGREES """ - ########## - # Error handling: check whether files exist and - # whether input types are correct. - ########## - _check_add_pbh(hdf5_file=hdf5_file, ebf_file=ebf_file, - output_root2=output_root2, fdm=fdm, pbh_mass=pbh_mass, - r_max=r_max, r_s=r_s, gamma=gamma, v_esc=v_esc, - rho_0=rho_0, n_lin=n_lin, - overwrite=overwrite, seed=seed) + cos_b0 = np.cos(np.radians(b0)) - if not overwrite: - # Check if HDF5 file exists already. If it does, throw an error message - # to complain and exit. - if os.path.isfile(output_root2 + '.h5'): - raise Exception( - 'That .h5 file name is taken! Either delete the .h5 file, ' - 'or pick a new name.') + l = l0 + t * mu_lcosb * masyr_to_degday / cos_b0 + b = b0 + t * mu_b * masyr_to_degday - # Check to make sure that the output hdf5 file - # will not overwrite the input hdf5 file - output_hdf5_file = '%s.h5' % output_root2 - if hdf5_file == output_hdf5_file: - raise Exception('Output hdf5 file %s cannot be equal to ' - 'input hdf5 file %s' % (output_hdf5_file, hdf5_file)) + return l, b - ########## - # Start of code - ######### +def calc_centroid_shift(glat_S, glon_S, glat_N, glon_N, f_L, f_S, f_N, u): + """ + Calculate the centroid (astrometric) shift + for a luminous lens and neighbors - # Set random seed - np.random.seed(seed) + Parameters + ---------- + glat_x : float or array + Longitude of x (L = lens, S = source, N = neighbor centroid) - t0 = time.time() + glon_x : float or array + Latitude of x (L = lens, S = source, N = neighbor centroid) - #Read in the hdf5 file that doesn't have PBHs. Product of perform_pop_syn. - no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') - key_list = list(no_pbh_hdf5_file) - #Delete lat_bin_edges and long_bin_edges from key_list. - key_list = [key for key in key_list if 'bin_edges' not in key] + f_x : float or array + Flux of x (L = lens, S = source, N = all neighbors) - #Get data from lat_bin_edges and long_bin_edges - lat_bin = no_pbh_hdf5_file['lat_bin_edges'][:] - long_bin = no_pbh_hdf5_file['long_bin_edges'][:] - bin_edges_number = len(long_bin) + u : float or array + Dimensionless separation - #Getting the maximum ID from all of the stars and compact objects. - #Later used to set the IDs of the PBHs. - max_id_no_pbh = [] - for key in key_list: - # max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key][20])) - max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key]['obj_id'])) - max_id = np.amax(max_id_no_pbh) + Return + ------ + delta_c_obs : float or array + Magnitude of observed astrometric shift, in mas + """ + ########## + # Calculating the centroid shift in the frame of the lens + ########## - hdf5_dset_names = no_pbh_hdf5_file[key_list[0]][:].dtype.names + t1a1_t2a2 = (u ** 2 + 3) / (u * np.sqrt(u ** 2 + 4)) + a1_a2 = calc_magnification(u) - no_pbh_hdf5_file.close() + glat_c_lensed = (t1a1_t2a2 * f_S * glat_S + glat_N * f_N) / ( + a1_a2 * f_S + f_L + f_N) + glon_c_lensed = (t1a1_t2a2 * f_S * glon_S + glon_N * f_N) / ( + a1_a2 * f_S + f_L + f_N) - #Read in ebf file - t = ebf.read_ind(ebf_file, '/log', 0) - # Convert log to useful dictionary. - ebf_log = make_ebf_log(t) + glat_c_unlensed = (glat_S * f_S + glat_N * f_N) / (f_S + f_L + f_N) + glon_c_unlensed = (glon_S * f_S + glon_N * f_N) / (f_S + f_L + f_N) + + glat_delta_c = glat_c_lensed - glat_c_unlensed + glon_delta_c = glon_c_lensed - glon_c_unlensed + + delta_c_obs = np.sqrt(glat_delta_c ** 2 + glon_delta_c ** 2 * np.cos( + np.radians(glat_S)) ** 2) - #Obtain survey area and center latitude and longitude - b = float(ebf_log['latitude']) #deg - b_radian = b * np.pi / 180 #rad - l = float(ebf_log['longitude']) #deg - l_radian = l * np.pi /180 #rad - surveyArea = float(ebf_log['surveyArea']) #deg^2 + # Convert from degrees to mas + delta_c_obs *= 3.6 * 10 ** 6 - #Calculate the size of the field of view we are running - field_of_view_radius = (surveyArea/np.pi)**(1/2) + return delta_c_obs - # Generate an array of heliocentric radii - # These radii will just be used to numerically integrate the density - n_lin = 1000 - if np.logical_and(np.logical_and(np.abs(l_radian)<0.5 * np.pi / 180, - np.abs(b_radian)<0.5 * np.pi / 180), - n_lin<100000): - print('Warning: for fields very near the center of the Milky Way it is reocmmended that the number of elements used to estimate the density be n_lin>100000') - r_h_linspace = np.linspace(0, r_max, num=n_lin) - # Represent the line of sight line in galactic coordinates - galactic_lin = coord.Galactic(l=l_radian * units.rad, - b=b_radian * units.rad, - distance=r_h_linspace * units.kpc) +def calc_bump_amp(u0, f_S, f_L, f_N): + """ + Calculate the "bump" amplitude, given the minimum separation and the fluxes + of the (unmagnified) source, lens, and neighbors. + The bump amplitude (mags) is: + |m_peak - m_base| = -2.5 log10 ((A(u0) * f_S + f_L + f_N)/(f_S + f_L + f_N)) - # convert the line of sight to galactocentric coordinates - #outputs l, b, and distance in degrees. - galactocen_lin = galactic_lin.transform_to(coord.Galactocentric(representation_type='spherical')) + Parameters + ---------- + u0 : float or array + Dimensionless source-lens angular separation, closest approach + f_S : float or array + Flux from source (arbitrary units) + f_L : float or array + Flux from lens (arbitrary units) + f_N : float or array + Flux from neighbors (arbitrary units) - # Determine the dark matter density at all galactocentric radii along the line of sight. - rho_lin = rho_dmhalo(galactocen_lin.spherical.distance.value, - rho_0=rho_0, r_s=r_s, gamma=gamma) + Return + ------ + m_bump : float or array + Bump magnitude + """ + A = calc_magnification(u0) + m_bump = 2.5 * np.log10((A * f_S + f_L + f_N) / (f_S + f_L + f_N)) - # Estimate the total mass within the line-of-sight cylinder [units: M_sun kpc**-2] - # Projected density along line of light - # (multiply by projected area to get total mass) - rho_marg_r = np.trapz(rho_lin, dx=(r_max) / n_lin) * 1000**3 - print("Projected density along line-of-sight = {0:0.2e} [M_sun kpc**-2]".format(rho_marg_r)) - # LOS cylinder radius in kpc, assuming small angle approximation [units: kpc] - r_proj_los_cyl = field_of_view_radius * np.pi / 180 * (r_max) - # Projected area of the LOS cylinder [units: kpc**2] - area_proj_los_cyl = np.pi * r_proj_los_cyl**2 - # Mass within the total cylinder - mass_los_cyl = rho_marg_r * area_proj_los_cyl - print("Mass within line-of-sight cylinder = {0:0.2e} [M_sun]".format(mass_los_cyl)) + return m_bump - # Total number of black holes to randomly draw - n_pbh = int(np.round(fdm * mass_los_cyl / pbh_mass)) - # Estimate the discrete CDF based on the discrete PDF - rho_marg_r_cum = integrate.cumtrapz(y=rho_lin, - x=galactic_lin.distance.kpc, - dx=(r_max) / n_lin) - cdf_los = rho_marg_r_cum / rho_marg_r_cum[-1] - # Since cumtrapz does not include zero for the first element insert it - cdf_los = np.insert(cdf_los,0,0) +def calc_app_mag(r, M, E, f): + """ + Calculate the apparent magnitude (i.e. distance modulus + plus extinction) - # Create a function to interpolate the CDF so that we can randomly sample from it - f_cdf_d = interpolate.interp1d(cdf_los, galactic_lin.distance.kpc) + Parameters + ---------- + M : float or array + Absolute magnitude of star - # Randomly sample galactic coordinates for the PBHs based on CDF - d_galac = f_cdf_d(np.random.uniform(size=n_pbh)) + r : float or array + Distance of star from sun (in kpc) - # Randomly assign a l & b galactic coordinate to each PBH, within the LOS cone - # sample the angle from 0 to 2pi - theta = np.random.uniform(size=n_pbh) * 2 * np.pi - # sample radius correcting for annular area to make uniform - r_cyl = r_proj_los_cyl * np.sqrt(np.random.uniform(size=n_pbh)) # kpc - y_cyl = r_cyl * np.sin(theta) # kpc - x_cyl = r_cyl * np.cos(theta) # kpc + E : float or array + Extinction law - # Mask out sampled PBH outside the observation cone - mask_obs_cone = r_cyl <= r_proj_los_cyl * d_galac / (r_max) - print('Number of PBH before and after light cone masking: {0} and {1}, respectively'.format(n_pbh, np.sum(mask_obs_cone))) - - # Assuming small angle approximation - b_galac = r_cyl * np.sin(theta) / d_galac + b_radian # rad - l_galac = r_cyl * np.cos(theta) / np.cos(b_radian) / d_galac + l_radian # rad + f : float or array + Coefficient for that particular band or whatever - d_galac = d_galac[mask_obs_cone] - b_galac = b_galac[mask_obs_cone] - l_galac = l_galac[mask_obs_cone] + Return + ------ + m : float or array + Apparent magnitude of star - latitude = b_galac * (180/np.pi) #degrees - longitude = l_galac * (180/np.pi) #degrees - - N_PBHs_in_field = len(d_galac) - print('%i PBHs in the field' % N_PBHs_in_field) + """ + m = calc_DM(r, M) + calc_ext(E, f) - if N_PBHs_in_field == 0: - print('-- No PBHs in the field') - print('-- Copying %s to %s' % (hdf5_file, output_hdf5_file)) - shutil.copy(hdf5_file, output_hdf5_file) - return + return m - #Converting the PBH positions from the field of view back to galactocentric for determining velocities. - galactic_pbh = coord.Galactic(l=longitude * units.deg, b=latitude * units.deg, distance=d_galac * units.kpc) - galacto_pbh = galactic_pbh.transform_to(coord.Galactocentric(representation_type='spherical')) - cart_pbh = astropy.coordinates.cartesian_to_spherical(galacto_pbh.x, galacto_pbh.y, galacto_pbh.z) - pbh_r_galacto = cart_pbh[0] - #Inner slope of the MW halo - #From Lacroix et al 2018, Figure 11 (top left panel) - data_dir = '%s/data' % os.path.dirname(inspect.getfile(add_pbh)) - if gamma == 1: - vel_data = pd.read_csv('%s/radial_velocity_profile_steep.csv' % data_dir) - elif gamma == .25: - vel_data = pd.read_csv('%s/radial_velocity_profile_shallow.csv' % data_dir) - elif gamma == .5: - vel_data = pd.read_csv('%s/radial_velocity_profile_middle.csv' % data_dir) - else: - raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) +def calc_DM(r, M): + """ + Calculate the distance modulus: m = M + 5log10(100*r/kpc) - #Interpolating v values from the above data, given the PBH r values. - pbh_vrms = np.interp(pbh_r_galacto, vel_data['r'], vel_data['v']) - v_vals = np.arange(0, v_esc) #Goes from v to v_esc - a = (1/2)*pbh_vrms*((np.pi/2)**(1/2)) + Parameters + ---------- + M : float or array + Absolute magnitude of star - #Calculating the v_rms velocities for the PBHs by randomly sampling from the CDF. - rand_cdf = np.array([]) + r : float or array + Distance of star from sun (in kpc) - for a_val in a: - cdf = scipy.special.erf(v_vals/(a_val*2**(1/2)))-(((2/np.pi)**(1/2))*((v_vals*np.exp(-v_vals**2/2*a_val**2))/a_val)) - rand_cdf = np.append(rand_cdf, np.random.uniform(0, np.amax(cdf))) - interpreted_rms_velocities = np.interp(rand_cdf, cdf, v_vals) + Return + ------ + m : float or array + Apparent magnitude of star - #Sampling random latitude and longitude values for velocity to complete the spherical velocities. - sin_lat_vel = np.random.uniform(-1, 1, len(d_galac)) - lat_vel = np.arcsin(sin_lat_vel) - long_vel = np.random.uniform(0, 2*np.pi, len(d_galac)) + """ + m = M + 5 * np.log10(100 * r) - #Transforming velocities to cartesian to get vx, vy, and vz. - cart_vel = astropy.coordinates.spherical_to_cartesian(interpreted_rms_velocities, lat_vel, long_vel) + return m - #Load up a numpy array - comp_dtype = _generate_comp_dtype(hdf5_dset_names) - pbh_data = np.empty(len(d_galac), dtype=comp_dtype) - #Getting longitude into the right format for PopSyCLE - longitude = np.where(longitude>180, longitude-360, longitude) +def calc_ext(E, f): + """ + Calculate the magnitude of extinction. - pbh_data['rad'] = d_galac - pbh_data['glon'] = longitude - pbh_data['glat'] = latitude + Parameters + ---------- + E : float or array + Extinction law - pbh_data['vx'] = cart_vel[0] - pbh_data['vy'] = cart_vel[1] - pbh_data['vz'] = cart_vel[2] + f : float or array + Coefficient for that particular band or whatever - #Getting the rest of the PBH data for the combined .h5 file - pbh_data['mass'] = np.full(len(d_galac), pbh_mass) - pbh_data['zams_mass'] = np.full(len(d_galac), pbh_mass) - pbh_data['age'] = np.full(len(d_galac), np.nan) - pbh_data['popid'] = np.full(len(d_galac), 10) - pbh_data['rem_id'] = np.full(len(d_galac), 104) + Return + ------ + m_E : float or array + Magnitude of extinction - cart_helio = astropy.coordinates.spherical_to_cartesian(d_galac, b_galac, l_galac) - pbh_data['px'] = cart_helio[0] - pbh_data['py'] = cart_helio[1] - pbh_data['pz'] = cart_helio[2] + """ + m_E = E * f - vr, mu_b, mu_lcosb = calc_sph_motion(pbh_data['vx'], - pbh_data['vy'], - pbh_data['vz'], - d_galac, b_galac, l_galac) - pbh_data['vr'] = vr - pbh_data['mu_b'] = mu_b - pbh_data['mu_lcosb'] = mu_lcosb - pbh_data['obj_id'] = np.arange((max_id+1), (max_id+len(d_galac)+1)) + return m_E - pbh_data['exbv'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_K'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_J'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_I'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_U'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_R'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_B'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_H'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_V'] = np.full(len(d_galac), np.nan) - pbh_data['teff'] = np.full(len(d_galac), np.nan) - pbh_data['grav'] = np.full(len(d_galac), np.nan) - pbh_data['mbol'] = np.full(len(d_galac), np.nan) - pbh_data['feh'] = np.full(len(d_galac), np.nan) - if any(['ztf' in n for n in hdf5_dset_names]): - pbh_data['ztf_g'] = np.full(len(d_galac), np.nan) - pbh_data['ztf_r'] = np.full(len(d_galac), np.nan) - #Calculate the maximum and minimum l and b values for each dataset in the no PBH file, - #so that we can determine which datasets to correctly add the PBHs. - lat_long_list = [] - for idx in range(len(long_bin) - 1): - max_l = long_bin[idx + 1] - min_l = long_bin[idx] - for idx2 in range(len(lat_bin)-1): - max_b = lat_bin[idx2 + 1] - min_b = lat_bin[idx2] - lat_long_list.append((min_l, max_l, min_b, max_b)) +def get_Alambda_AKs(red_law_name, lambda_eff): + """ + Get Alambda/AKs. NOTE: this doesn't work for every law in PopStar! + Naming convention is not consistent. Change PopStar or add if statements? - - #Opening the file with no PBHs and creating a new file for the PBHs added. - no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') - pbh_hdf5_file = h5py.File(output_hdf5_file, 'w') + Parameters + ---------- + red_law_name : str + The name of the reddening law + lambda_eff : float + Wavelength in microns - #Appending the PBH data to the no PBH data and writing to the new .h5 file. - N_objs_no_pbh = 0 - N_objs_pbh = 0 - for idx, key in enumerate(key_list): - key_data = no_pbh_hdf5_file[key][:] - N_objs_no_pbh += key_data.shape[0] + Return + ------ + Alambda_AKs : float + Alambda/AKs - min_l, max_l, min_b, max_b = lat_long_list[idx] - mask = (pbh_data['glon'] >= min_l) & \ - (pbh_data['glon'] <= max_l) & \ - (pbh_data['glat'] >= min_b) & \ - (pbh_data['glat'] <= max_b) + """ + red_law_class = getattr(reddening, 'RedLaw' + red_law_name) + red_law = red_law_class() + red_law_method = getattr(red_law, red_law_name) + Alambda_AKs = red_law_method(lambda_eff, 1) - if np.sum(mask) == 0: - combined_data = key_data - else: - pbh_data_in_key = pbh_data[mask] - combined_data = np.hstack((key_data, pbh_data_in_key)) - N_objs_pbh += combined_data.shape[0] - _ = pbh_hdf5_file.create_dataset(key, - shape=(combined_data.shape[0],), - dtype=comp_dtype, - data=combined_data) - _ = pbh_hdf5_file.create_dataset('lat_bin_edges', (len(lat_bin), 1), data=lat_bin) - _ = pbh_hdf5_file.create_dataset('long_bin_edges', (len(lat_bin), 1), data=long_bin) - no_pbh_hdf5_file.close() - pbh_hdf5_file.close() + return Alambda_AKs - print('Checking totals') - print('-- %i original objects' % N_objs_no_pbh) - print('-- %i PBHs in the field' % N_PBHs_in_field) - print('-- %i new total objects' % N_objs_pbh) - if N_objs_pbh == N_objs_no_pbh + N_PBHs_in_field: - print('-- Totals match!') - else: - print('** MISSING PBHs!! **') +def calc_f(lambda_eff): + """ + Calculate that coefficient f that multiples E(B-V) to get the + extinction in magnitudes + """ + B = get_Alambda_AKs('Damineli16', 0.445) + V = get_Alambda_AKs('Damineli16', 0.551) + L = get_Alambda_AKs('Damineli16', lambda_eff) - t1 = time.time() - print('Total runtime: {0:f} s'.format(t1 - t0)) + f = L * (B - V) ** -1 - return + return f From 27599b23b14f1a3ec15c6b1ef900c338faf75d0b Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 18 Mar 2020 14:08:31 -0700 Subject: [PATCH 057/125] attempt to --- popsycle/data/slurm_config.yaml | 3 +++ popsycle/run.py | 32 ++++++++++++++++++++++++++++++++ popsycle/synthetic.py | 9 +++------ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/popsycle/data/slurm_config.yaml b/popsycle/data/slurm_config.yaml index 83fbb641..80b7939c 100644 --- a/popsycle/data/slurm_config.yaml +++ b/popsycle/data/slurm_config.yaml @@ -9,6 +9,9 @@ account: ulens queue: regular # Name of the resource that will be used for the run resource: haswell +# Set to true if slurm scheduler requires a CONSTRAINT specification +# Set to false if slurm scheduler does not +include_constraint: true # Additional lines to be run before executing run.py additional_lines: - module load cray-hdf5/1.10.5.2 diff --git a/popsycle/run.py b/popsycle/run.py index 0cd74378..bb445d3a 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -136,6 +136,7 @@ def generate_field_config_file(longitude, latitude, area, def generate_slurm_config_file(path_python='python', account='ulens', queue='regular', resource='haswell', + include_constraint=True, n_cores_per_node=32, n_nodes_max=2388, walltime_max='48:00:00', additional_lines=['module load cray-hdf5/1.10.5.2', @@ -186,6 +187,7 @@ def generate_slurm_config_file(path_python='python', account='ulens', 'account': account, 'queue': queue, 'resource': resource, + 'include_constraint': bool(include_constraint), 'additional_lines': additional_lines, resource: {'n_cores_per_node': n_cores_per_node, 'n_nodes_max': n_nodes_max, @@ -392,15 +394,38 @@ def _check_slurm_config(slurm_config, walltime): if 'path_python' not in slurm_config: raise Exception('path_python must be set in slurm_config') + path_python = slurm_config['path_python'] + if type('path_python') != str: + raise Exception('path_python (%s) must be a string.' % str(path_python)) + if 'account' not in slurm_config: raise Exception('account must be set in slurm_config') + account = slurm_config['account'] + if type('account') != str: + raise Exception('account (%s) must be a string.' % str(account)) + if 'queue' not in slurm_config: raise Exception('queue must be set in slurm_config') + queue = slurm_config['queue'] + if type('queue') != str: + raise Exception('queue (%s) must be a string.' % str(queue)) + if 'resource' not in slurm_config: raise Exception('resource must be set in slurm_config') + resource = slurm_config['resource'] + if type('resource') != str: + raise Exception('resource (%s) must be a string.' % str(resource)) + + if 'include_constraint' not in slurm_config: + raise Exception('include_constraint must be set in slurm_config') + + include_constraint = slurm_config['include_constraint'] + if type('include_constraint') != bool: + raise Exception('include_constraint (%s) must be a boolean.' % str(include_constraint)) + if 'n_cores_per_node' not in slurm_config[slurm_config['resource']]: raise Exception('n_cores_per_node must be set in slurm_config') @@ -670,6 +695,13 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, # Template for writing slurm script. Text must be left adjusted. slurm_template = """#!/bin/sh + # Job name + #SBATCH --account={account} + #SBATCH --qos={queue} + """ + if slurm_config['include_constraint']: + slurm_template += '#SBATCH --constraint={resource}\n' + slurm_template += """#!/bin/sh # Job name #SBATCH --account={account} #SBATCH --qos={queue} diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index e336898d..45ba4270 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1760,14 +1760,11 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, # From Lacroix et al 2018, Figure 11 (top left panel) data_dir = '%s/data' % os.path.dirname(inspect.getfile(add_pbh)) if gamma == 1: - vel_data = pd.read_csv( - '%s/radial_velocity_profile_steep.csv' % data_dir) + vel_data = pd.read_csv('%s/radial_velocity_profile_steep.csv' % data_dir) elif gamma == .25: - vel_data = pd.read_csv( - '%s/radial_velocity_profile_shallow.csv' % data_dir) + vel_data = pd.read_csv('%s/radial_velocity_profile_shallow.csv' % data_dir) elif gamma == .5: - vel_data = pd.read_csv( - '%s/radial_velocity_profile_middle.csv' % data_dir) + vel_data = pd.read_csv('%s/radial_velocity_profile_middle.csv' % data_dir) else: raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) From 5ed3a790f465dcaed6fafd26d16143bfd9ee74ea Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 18 Mar 2020 14:17:27 -0700 Subject: [PATCH 058/125] set include_contraint to bool --- popsycle/run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/popsycle/run.py b/popsycle/run.py index bb445d3a..2e8f4700 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -584,6 +584,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, # Load the slurm configuration file slurm_config = load_config_file(slurm_config_filename) + slurm_config['include_constraint'] = bool(slurm_config['include_constraint']) # Check pipeline stages for valid inputs _check_slurm_config(slurm_config, walltime) From 5d15e9be684aee200291ab0d828f643a1728107e Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 18 Mar 2020 14:21:08 -0700 Subject: [PATCH 059/125] set include_contraint to bool --- popsycle/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index 2e8f4700..4add278e 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -424,7 +424,8 @@ def _check_slurm_config(slurm_config, walltime): include_constraint = slurm_config['include_constraint'] if type('include_constraint') != bool: - raise Exception('include_constraint (%s) must be a boolean.' % str(include_constraint)) + if type('include_constraint') != str: + raise Exception('include_constraint (%s) must be a boolean.' % str(include_constraint)) if 'n_cores_per_node' not in slurm_config[slurm_config['resource']]: raise Exception('n_cores_per_node must be set in slurm_config') From 9f37613be6d8ccc5a7ead8aaae6dd10b78591c83 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 18 Mar 2020 14:26:50 -0700 Subject: [PATCH 060/125] change slurm_template header --- popsycle/run.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index 4add278e..6db0301e 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -697,18 +697,13 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, # Template for writing slurm script. Text must be left adjusted. slurm_template = """#!/bin/sh - # Job name - #SBATCH --account={account} - #SBATCH --qos={queue} - """ - if slurm_config['include_constraint']: - slurm_template += '#SBATCH --constraint={resource}\n' - slurm_template += """#!/bin/sh # Job name #SBATCH --account={account} #SBATCH --qos={queue} -#SBATCH --constraint={resource} -#SBATCH --nodes=1 + """ + if slurm_config['include_constraint']: + slurm_template += '#SBATCH --constraint={resource}\n' + slurm_template += """#SBATCH --nodes=1 #SBATCH --time={walltime} #SBATCH --job-name={jobname} #SBATCH --output={jobname}.out From e59e529e6091ae01b909841efef00450d703a86e Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 18 Mar 2020 14:33:41 -0700 Subject: [PATCH 061/125] fix typo --- popsycle/run.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index 6db0301e..fa2c80f3 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -699,8 +699,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, slurm_template = """#!/bin/sh # Job name #SBATCH --account={account} -#SBATCH --qos={queue} - """ +#SBATCH --qos={queue}""" if slurm_config['include_constraint']: slurm_template += '#SBATCH --constraint={resource}\n' slurm_template += """#SBATCH --nodes=1 From 519b478c3b7ea3874b1bb97a36391755413a4bd3 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 18 Mar 2020 14:34:24 -0700 Subject: [PATCH 062/125] fix typo --- popsycle/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index fa2c80f3..1e6cf4c1 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -699,7 +699,8 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, slurm_template = """#!/bin/sh # Job name #SBATCH --account={account} -#SBATCH --qos={queue}""" +#SBATCH --qos={queue} +""" if slurm_config['include_constraint']: slurm_template += '#SBATCH --constraint={resource}\n' slurm_template += """#SBATCH --nodes=1 From 09e6766c56e6a0c947e297794c8507741159de81 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Wed, 18 Mar 2020 16:58:44 -0700 Subject: [PATCH 063/125] changed file permissions --- popsycle/ebf.py | 0 popsycle/run.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 popsycle/ebf.py mode change 100644 => 100755 popsycle/run.py diff --git a/popsycle/ebf.py b/popsycle/ebf.py old mode 100644 new mode 100755 diff --git a/popsycle/run.py b/popsycle/run.py old mode 100644 new mode 100755 From 9d75e4cb5bb916ef8e5c853ad435a20a958c3cb8 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 20 Mar 2020 09:49:29 -0700 Subject: [PATCH 064/125] add comments to read_ind --- popsycle/ebf.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/popsycle/ebf.py b/popsycle/ebf.py index 5736ec04..9e14c624 100644 --- a/popsycle/ebf.py +++ b/popsycle/ebf.py @@ -2753,18 +2753,36 @@ def read(self,i,nsize=1): return None def read_ind(self,ind): + # This method looks for groups of contiguous indices in 'ind' and + # loads those blocks of memory with a single copy command. + # This method is ~__% faster than looping over each index and running: + # data[i] = self.read(ind[i]) if numpy.max(ind) Date: Fri, 20 Mar 2020 09:53:35 -0700 Subject: [PATCH 065/125] change name of kdt_star_exbv to exbv_arr4kdt --- popsycle/synthetic.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index ea88589f..1f3bb110 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -607,7 +607,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, # Create the KDTree used for calculating extinction from the first # sample of stars kdt_star_p = None - kdt_star_exbv = None + exbv_arr4kdt = None if len_adx > 0: num_kdtree_samples = int(min(len_adx, 2e6)) kdt_idx = np.random.choice(np.arange(len_adx), @@ -619,7 +619,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, star_pz = ebf.read_ind(ebf_file, '/pz', bin_idx) star_xyz = np.array([star_px, star_py, star_pz]).T kdt_star_p = cKDTree(star_xyz) - kdt_star_exbv = ebf.read_ind(ebf_file, '/exbv_schlegel', bin_idx) + exbv_arr4kdt = ebf.read_ind(ebf_file, '/exbv_schlegel', bin_idx) del bin_idx, star_px, star_py, star_pz ########## @@ -724,7 +724,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, comp_dict, next_id = _make_comp_dict(iso_dir, age_of_bin, mass_in_bin, stars_in_bin, next_id, - kdt_star_p, kdt_star_exbv, + kdt_star_p, exbv_arr4kdt, BH_kick_speed_mean=BH_kick_speed_mean, NS_kick_speed_mean=NS_kick_speed_mean, additional_photometric_systems=additional_photometric_systems, @@ -746,7 +746,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, ########## del star_dict gc.collect() - del kdt_star_p, kdt_star_exbv + del kdt_star_p, exbv_arr4kdt t1 = time.time() print('Total run time is {0:f} s'.format(t1 - t0)) @@ -957,7 +957,7 @@ def current_initial_ratio(logage, ratio_file, iso_dir, seed=None): def _make_comp_dict(iso_dir, log_age, currentClusterMass, star_dict, next_id, - kdt_star_p, kdt_star_exbv, + kdt_star_p, exbv_arr4kdt, BH_kick_speed_mean=50, NS_kick_speed_mean=400, additional_photometric_systems=None, seed=None): @@ -986,7 +986,7 @@ def _make_comp_dict(iso_dir, log_age, currentClusterMass, KDTree constructed from the positions of randomly selected stars that all share the same popid and similar log_age. - kdt_star_exbv : numpy + exbv_arr4kdt : numpy Array of galactic extinctions for the stars in kdt_star_p Optional Parameters @@ -1225,7 +1225,7 @@ def _make_comp_dict(iso_dir, log_age, currentClusterMass, comp_dict['py'][lum_WD_idx], comp_dict['pz'][lum_WD_idx]]).T dist, indices = kdt_star_p.query(comp_xyz) - comp_dict['exbv'][lum_WD_idx] = kdt_star_exbv[indices.T] + comp_dict['exbv'][lum_WD_idx] = exbv_arr4kdt[indices.T] comp_dict['ubv_I'][lum_WD_idx] = comp_table['m_ubv_I'][lum_WD_idx].data comp_dict['ubv_K'][lum_WD_idx] = comp_table['m_ukirt_K'][lum_WD_idx].data From f24ad2c8e3e60b4cd2d8eb875026e64326c260ec Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 20 Mar 2020 09:55:30 -0700 Subject: [PATCH 066/125] add comments --- popsycle/ebf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/ebf.py b/popsycle/ebf.py index 9e14c624..49a266d2 100644 --- a/popsycle/ebf.py +++ b/popsycle/ebf.py @@ -2755,7 +2755,7 @@ def read(self,i,nsize=1): def read_ind(self,ind): # This method looks for groups of contiguous indices in 'ind' and # loads those blocks of memory with a single copy command. - # This method is ~__% faster than looping over each index and running: + # This method is ~50% faster than looping over each index and running: # data[i] = self.read(ind[i]) if numpy.max(ind) Date: Fri, 20 Mar 2020 10:13:41 -0700 Subject: [PATCH 067/125] add comments --- popsycle/synthetic.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 1f3bb110..2a2f34c7 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -604,8 +604,13 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, num_stars_in_bin = 2e6 num_bins = int(math.ceil(len_adx / num_stars_in_bin)) - # Create the KDTree used for calculating extinction from the first - # sample of stars + # Create a KDTree from randomly selected stars in the + # pop_id / age_bin used for calculating extinction to luminous + # white dwarfs. Because the same KDTree is used for each sub-bin, + # two compact objects randomly selected to have nearly identical + # positions would have identical extinctions. This low + # probability event is a reasonable trade-off for the reduced + # compute time gained by only constructing the KDTree once. kdt_star_p = None exbv_arr4kdt = None if len_adx > 0: @@ -1221,6 +1226,9 @@ def _make_comp_dict(iso_dir, log_age, currentClusterMass, lum_WD_idx = np.argwhere(~np.isnan(comp_table['m_ubv_I'])) if len(lum_WD_idx) > 0: + # The extinction to the luminous white dwarfs is calculated + # by finding the nearest star in the pop_id / age_bin KDTree + # to the compact object and copying that star's extinction. comp_xyz = np.array([comp_dict['px'][lum_WD_idx], comp_dict['py'][lum_WD_idx], comp_dict['pz'][lum_WD_idx]]).T From 208481acb2331dce9755d8585c47e82cb23abd8a Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 20 Mar 2020 10:22:27 -0700 Subject: [PATCH 068/125] change spacing --- popsycle/ebf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/popsycle/ebf.py b/popsycle/ebf.py index 49a266d2..25874bfa 100644 --- a/popsycle/ebf.py +++ b/popsycle/ebf.py @@ -2752,16 +2752,16 @@ def read(self,i,nsize=1): else: return None - def read_ind(self,ind): + def read_ind(self, ind): # This method looks for groups of contiguous indices in 'ind' and # loads those blocks of memory with a single copy command. # This method is ~50% faster than looping over each index and running: # data[i] = self.read(ind[i]) - if numpy.max(ind) Date: Mon, 23 Mar 2020 11:44:33 -0700 Subject: [PATCH 069/125] remove PBHs from h5 file before adding in new ones --- popsycle/synthetic.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 9d233566..07732cf9 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1683,8 +1683,7 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, if np.logical_and(np.logical_and(np.abs(l_radian) < 0.5 * np.pi / 180, np.abs(b_radian) < 0.5 * np.pi / 180), n_lin < 100000): - print( - 'Warning: for fields very near the center of the Milky Way it is reocmmended that the number of elements used to estimate the density be n_lin>100000') + print('Warning: for fields very near the center of the Milky Way it is reocmmended that the number of elements used to estimate the density be n_lin>100000') r_h_linspace = np.linspace(0, r_max, num=n_lin) # Represent the line of sight line in galactic coordinates @@ -1890,16 +1889,25 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, N_objs_pbh = 0 for idx, key in enumerate(key_list): key_data = no_pbh_hdf5_file[key][:] + + # Remove any PBHs that are already in the h5 file + cond = key_data['rem_id'] != 104 + key_data = key_data[cond] + + # Count the number of objects in the key N_objs_no_pbh += key_data.shape[0] + # Build a mask on the PBHs that fits the bounds of the key min_l, max_l, min_b, max_b = lat_long_list[idx] mask = (pbh_data['glon'] >= min_l) & \ (pbh_data['glon'] <= max_l) & \ (pbh_data['glat'] >= min_b) & \ (pbh_data['glat'] <= max_b) + # If there are no PBHs in the key, copy over the original key if np.sum(mask) == 0: combined_data = key_data + # If there are PBHs in the key, append them to the original key else: pbh_data_in_key = pbh_data[mask] combined_data = np.hstack((key_data, pbh_data_in_key)) From c2d768d53ba7fe033c9cc92487d5926da82a791b Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 12:12:40 -0700 Subject: [PATCH 070/125] overwrite hdf5 file in add_pbh, unless new_output_root provided --- popsycle/run.py | 74 ++++++++++++----------------- popsycle/synthetic.py | 106 +++++++++++++++++------------------------- 2 files changed, 72 insertions(+), 108 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index ea023f9c..cf865034 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -20,7 +20,7 @@ from popsycle.synthetic import _check_add_pbh -def _return_filename_dict(output_root, add_pbh_flag=False): +def _return_filename_dict(output_root): """ Return the filenames of the files output by the pipeline @@ -42,16 +42,9 @@ def _return_filename_dict(output_root, add_pbh_flag=False): # Write out all of the filenames using the output_root ebf_filename = '%s.ebf' % output_root hdf5_filename = '%s.h5' % output_root - - # Append _pbh to filenames if add_pbh will be run - if add_pbh_flag: - events_filename = '%s_pbh_events.fits' % output_root - blends_filename = '%s_pbh_blends.fits' % output_root - noevents_filename = '%s_pbh_NOEVENTS.txt' % output_root - else: - events_filename = '%s_events.fits' % output_root - blends_filename = '%s_blends.fits' % output_root - noevents_filename = '%s_NOEVENTS.txt' % output_root + events_filename = '%s_events.fits' % output_root + blends_filename = '%s_blends.fits' % output_root + noevents_filename = '%s_NOEVENTS.txt' % output_root # Add the filenames to a dictionary filename_dict = { @@ -475,10 +468,10 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, longitude, latitude, area, n_cores_calc_events, walltime, jobname='default', + pbh_config_filename=None, seed=None, overwrite=False, submitFlag=True, skip_galaxia=False, skip_perform_pop_syn=False, - skip_calc_events=False, skip_refine_events=False, - pbh_config_filename=None): + skip_calc_events=False, skip_refine_events=False): """ Generates (and possibly submits) the slurm script that executes the PopSyCLE pipeline @@ -530,7 +523,8 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, pbh_config_filename : str Name of pbh_config.yaml file containing the PBH parameters that will be passed along to the run_on_slurm.py command in the - slurm script. + slurm script. If set to None, `add_pbh` is skipped over. + Default None. seed : int If set to non-None, all random sampling will be seeded with the @@ -615,7 +609,6 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, pbh_config = load_config_file(pbh_config_filename) _check_add_pbh(hdf5_file='test.h5', ebf_file='test.ebf', - output_root2=output_root, fdm=pbh_config['fdm'], pbh_mass=pbh_config['pbh_mass'], r_max=pbh_config['r_max'], @@ -624,7 +617,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], - overwrite=overwrite, + new_output_root=None, seed=seed) if not skip_calc_events: _check_calc_events(hdf5_file='test.h5', @@ -864,6 +857,16 @@ def run(): Exiting...""".format(args.popsycle_config_filename)) sys.exit(1) + # Check for pbh config file, if provided. Exit if not present. + if args.pbh_config_filename is not None: + if not os.path.exists(args.pbh_config_filename): + print("""Error: PBH configuration file {0} missing, + cannot continue. In order to execute run.py with 'add_pbh', + generate a PBH configuration file using + popsycle.run.generate_pbh_config_file. + Exiting...""".format(args.pbh_config_filename)) + sys.exit(1) + # Load the config files for field parameters field_config = load_config_file(args.field_config_filename) @@ -913,7 +916,6 @@ def run(): pbh_config = load_config_file(args.pbh_config_filename) _check_add_pbh(hdf5_file=filename_dict['hdf5_filename'], ebf_file=filename_dict['ebf_filename'], - output_root2=args.output_root, fdm=pbh_config['fdm'], pbh_mass=pbh_config['pbh_mass'], r_max=pbh_config['r_max'], @@ -922,7 +924,7 @@ def run(): v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], - overwrite=args.overwrite, + new_output_root=None, seed=args.seed) if not args.skip_calc_events: _check_calc_events(hdf5_file=filename_dict['hdf5_filename'], @@ -975,34 +977,18 @@ def run(): overwrite=args.overwrite, seed=args.seed) - # Append '_pbh' to output_root if add_pbh will be run + # If optional pbh_config_filename is provided: if args.pbh_config_filename is not None: - output_root = '%s_pbh' % args.output_root - else: - output_root = args.output_root - - # only do stuff if optional config file for pbhs was provided - if args.pbh_config_filename is not None: - # Check for pbh config file. Exit if not present. - if not os.path.exists(args.pbh_config_filename): - print("""Error: PBH configuration file {0} missing, - cannot continue. In order to execute run.py, generate a - PBH configuration file using - popsycle.synthetic.generate_pbh_config_file. - Exiting...""".format(args.pbh_config_filename)) - sys.exit(1) - - pbh_config = load_config_file(args.pbh_config_filename) - - # Check if .h5 file exists from perform popsyn, use as input for following function + # Check if .h5 file exists from perform popsyn, + # use as input for following function if not os.path.exists(filename_dict['hdf5_filename']): - print("""Error: H5 file was not created properly by - synthetic.perform_pop_syn""") + print("""Error: hdf5 file was not created properly by + synthetic.perform_pop_syn. + Exiting....""") sys.exit(1) synthetic.add_pbh(hdf5_file=filename_dict['hdf5_filename'], ebf_file=filename_dict['ebf_filename'], - output_root2=output_root, fdm=pbh_config['fdm'], pbh_mass=pbh_config['pbh_mass'], r_max=pbh_config['r_max'], @@ -1011,7 +997,7 @@ def run(): v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], - overwrite=args.overwrite, + new_output_root=None, seed=args.seed) if not args.skip_calc_events: @@ -1026,7 +1012,7 @@ def run(): # Run calc_events print('-- Executing calc_events') synthetic.calc_events(hdf5_file=filename_dict['hdf5_filename'], - output_root2=output_root, + output_root2=args.output_root, radius_cut=popsycle_config['radius_cut'], obs_time=popsycle_config['obs_time'], n_obs=popsycle_config['n_obs'], @@ -1045,7 +1031,7 @@ def run(): if not args.skip_refine_events: # Remove refine_events output if already exists and overwrite=True filename = '{0:s}_refined_events_{1:s}_{2:s}.' \ - 'fits'.format(output_root, + 'fits'.format(args.output_root, popsycle_config['filter_name'], popsycle_config['red_law']) if _check_for_output(filename, args.overwrite): @@ -1053,7 +1039,7 @@ def run(): # Run refine_events print('-- Executing refine_events') - synthetic.refine_events(input_root=output_root, + synthetic.refine_events(input_root=args.output_root, filter_name=popsycle_config['filter_name'], photometric_system=popsycle_config['photometric_system'], red_law=popsycle_config['red_law'], diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 07732cf9..d6b05bb7 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1406,10 +1406,10 @@ def rho_dmhalo(r, rho_0=.0093, r_s=18.6, gamma=1): return rho -def _check_add_pbh(hdf5_file, ebf_file, output_root2, +def _check_add_pbh(hdf5_file, ebf_file, fdm, pbh_mass, r_max, r_s, gamma, v_esc, - rho_0, n_lin, overwrite, seed): + rho_0, n_lin, new_output_root, seed): """ Checks that the inputs of add_pbj are valid @@ -1422,13 +1422,6 @@ def _check_add_pbh(hdf5_file, ebf_file, output_root2, str : name of the ebf file from Galaxia ebf file : actually the ebf file from Galaxia - output_root2 : str - The thing you want the output files to be named - Examples: - 'myout' - '/some/path/to/myout' - '../back/to/some/path/myout' - fdm : float Fraction of dark matter. The fraction of dark matter that you want to consist of PBHs. @@ -1464,10 +1457,11 @@ def _check_add_pbh(hdf5_file, ebf_file, output_root2, v_esc is used in calculating the velocities. Default is 550 km/s. Most papers cite values of 515-575. - overwrite : bool - If set to True, overwrites output files. If set to False, exists the - function if output files are already on disk. - Default is False. + new_output_root : str + If set to None, 'add_pbh' overwrites the original hdf5 file with a + new hdf5 file of the same name. If set to a string, this string is the + prefix of the new hdf5 file. + Default None. seed : int If set to non-None, all random sampling will be seeded with the @@ -1480,58 +1474,49 @@ def _check_add_pbh(hdf5_file, ebf_file, output_root2, if ebf_file[-4:] != '.ebf': raise Exception('ebf_file (%s) must be an ebf file.' % str(ebf_file)) - if type(output_root2) != str: - raise Exception( - 'output_root2 (%s) must be a string.' % str(output_root)) - if type(fdm) != int: if type(fdm) != float: - raise Exception( - 'fdm (%s) must be an integer or a float.' % str(fdm)) + raise Exception('fdm (%s) must be an integer or a float.' % str(fdm)) if type(pbh_mass) != int: if type(pbh_mass) != float: - raise Exception( - 'pbh_mass (%s) must be an integer or a float.' % str(pbh_mass)) + raise Exception('pbh_mass (%s) must be an integer or a float.' % str(pbh_mass)) if type(r_max) != int: if type(r_max) != float: - raise Exception( - 'r_max (%s) must be an integer or a float.' % str(r_max)) + raise Exception('r_max (%s) must be an integer or a float.' % str(r_max)) if type(r_s) != int: if type(r_s) != float: - raise Exception( - 'r_s (%s) must be an integer or a float.' % str(r_s)) + raise Exception('r_s (%s) must be an integer or a float.' % str(r_s)) if gamma not in [.25, .5, 1]: raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) if type(v_esc) != int: if type(v_esc) != float: - raise Exception( - 'v_esc (%s) must be an integer or a float.' % str(v_esc)) + raise Exception('v_esc (%s) must be an integer or a float.' % str(v_esc)) if type(rho_0) != int: if type(rho_0) != float: - raise Exception( - 'rho_0 (%s) must be an integer or a float.' % str(rho_0)) + raise Exception('rho_0 (%s) must be an integer or a float.' % str(rho_0)) if type(n_lin) != int: raise Exception('n_lin (%s) must be an integerr.' % str(n_lin)) - if type(overwrite) != bool: - raise Exception('overwrite (%s) must be a boolean.' % str(overwrite)) + if new_output_root is not None: + if type(new_output_root) != str: + raise Exception('new_output_root (%s) must be None or a string.' % str(new_output_root)) if seed is not None: if type(seed) != int: - raise Exception( - 'seed (%s) must be None or an integer.' % str(seed)) + raise Exception('seed (%s) must be None or an integer.' % str(seed)) -def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, +def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, r_max=16.6, r_s=18.6, gamma=1, v_esc=550, - rho_0=0.0093, n_lin=1000, overwrite=False, seed=None): + rho_0=0.0093, n_lin=1000, + new_output_root=None, seed=None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, and saves them in a new HDF5 file with the PBHs added. @@ -1545,13 +1530,6 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, str : name of the ebf file from Galaxia ebf file : actually the ebf file from Galaxia - output_root2 : str - The thing you want the output files to be named - Examples: - 'myout' - '/some/path/to/myout' - '../back/to/some/path/myout' - fdm : float Fraction of dark matter. The fraction of dark matter that you want to consist of PBHs. @@ -1589,10 +1567,11 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, Optional Parameters ------------------- - overwrite : bool - If set to True, overwrites output files. If set to False, exists the - function if output files are already on disk. - Default is False. + new_output_root : str + If set to None, 'add_pbh' overwrites the original hdf5 file with a + new hdf5 file of the same name. If set to a string, this string is the + prefix of the new hdf5 file. + Default None. seed : int If set to non-None, all random sampling will be seeded with the @@ -1601,7 +1580,7 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, Outputs ------- - .h5 : hdf5 file + .h5 : hdf5 file The new .h5 file with PBHs injected in. """ ########## @@ -1610,25 +1589,19 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, ########## _check_add_pbh(hdf5_file=hdf5_file, ebf_file=ebf_file, - output_root2=output_root2, fdm=fdm, pbh_mass=pbh_mass, + fdm=fdm, pbh_mass=pbh_mass, r_max=r_max, r_s=r_s, gamma=gamma, v_esc=v_esc, rho_0=rho_0, n_lin=n_lin, - overwrite=overwrite, seed=seed) - - if not overwrite: - # Check if HDF5 file exists already. If it does, throw an error message - # to complain and exit. - if os.path.isfile(output_root2 + '.h5'): - raise Exception( - 'That .h5 file name is taken! Either delete the .h5 file, ' - 'or pick a new name.') - - # Check to make sure that the output hdf5 file - # will not overwrite the input hdf5 file - output_hdf5_file = '%s.h5' % output_root2 - if hdf5_file == output_hdf5_file: - raise Exception('Output hdf5 file %s cannot be equal to ' - 'input hdf5 file %s' % (output_hdf5_file, hdf5_file)) + new_output_root=new_output_root, + seed=seed) + + if new_output_root is None: + output_hdf5_file = hdf5_file.replace('.h5', '_pbh_tmp.h5') + print('** WARNING **') + print(" 'add_pbh' will overwrite %s. PBHs are appended to each key." % hdf5_file) + print(" To generate a new hdf5 file instead, rerun 'add_pbh' with the 'new_output_root' argument.") + else: + output_hdf5_file = '%s.h5' % new_output_root ########## # Start of code @@ -1933,6 +1906,11 @@ def add_pbh(hdf5_file, ebf_file, output_root2, fdm=1, pbh_mass=40, else: print('** MISSING PBHs!! **') + # If 'new_output_root' is None, replace temporary file with original + if new_output_root is None: + os.remove(hdf5_file) + os.rename(output_hdf5_file, hdf5_file) + t1 = time.time() print('Total runtime: {0:f} s'.format(t1 - t0)) From 7134bdd1788c0486efb1039377f327431546db48 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 12:33:50 -0700 Subject: [PATCH 071/125] fix run.py to work with new add_pbh new_output_root scheme --- popsycle/run.py | 19 +++++++++---------- popsycle/synthetic.py | 15 +++++++++------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index cf865034..dc77d4ba 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -832,8 +832,8 @@ def run(): action='store_true') optional.add_argument('--pbh-config-filename', type=str, help='Name of configuration file containing ' - 'pbh inputs. Default if needed is: ' - 'pbh_config.yaml') + 'pbh inputs. Default None.', + default=None) args = parser.parse_args() @@ -882,13 +882,7 @@ def run(): sys.exit(1) # Return the dictionary containing PopSyCLE output filenames - # Append '_pbh' to filenames if add_pbh will be run - if args.pbh_config_filename is not None: - add_pbh_flag = True - else: - add_pbh_flag = False - filename_dict = _return_filename_dict(args.output_root, - add_pbh_flag=add_pbh_flag) + filename_dict = _return_filename_dict(args.output_root) # Prepare additional_photometric_systems additional_photometric_systems = None @@ -912,7 +906,7 @@ def run(): additional_photometric_systems=additional_photometric_systems, overwrite=args.overwrite, seed=args.seed) - if add_pbh_flag: + if args.pbh_config_filename is not None: pbh_config = load_config_file(args.pbh_config_filename) _check_add_pbh(hdf5_file=filename_dict['hdf5_filename'], ebf_file=filename_dict['ebf_filename'], @@ -987,6 +981,11 @@ def run(): Exiting....""") sys.exit(1) + print("** COMMENT ON WARNING **") + print(" run.py executes 'add_pbh' without ") + print(" using the 'new_output_root' argument") + print(" and instead replaces %s." % filename_dict['hdf5_filename']) + print(" Therefore ignore the following warning:") synthetic.add_pbh(hdf5_file=filename_dict['hdf5_filename'], ebf_file=filename_dict['ebf_filename'], fdm=pbh_config['fdm'], diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index d6b05bb7..e5b030c9 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1598,8 +1598,8 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, if new_output_root is None: output_hdf5_file = hdf5_file.replace('.h5', '_pbh_tmp.h5') print('** WARNING **') - print(" 'add_pbh' will overwrite %s. PBHs are appended to each key." % hdf5_file) - print(" To generate a new hdf5 file instead, rerun 'add_pbh' with the 'new_output_root' argument.") + print(" 'add_pbh' will overwrite %s, with PBHs appended to each key." % hdf5_file) + print(" To generate a new hdf5 file, rerun 'add_pbh' with the 'new_output_root' argument.") else: output_hdf5_file = '%s.h5' % new_output_root @@ -1860,12 +1860,14 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # Appending the PBH data to the no PBH data and writing to the new .h5 file. N_objs_no_pbh = 0 N_objs_pbh = 0 + N_pbhs_removed = 0 for idx, key in enumerate(key_list): key_data = no_pbh_hdf5_file[key][:] # Remove any PBHs that are already in the h5 file cond = key_data['rem_id'] != 104 key_data = key_data[cond] + N_pbhs_removed += np.sum(~cond) # Count the number of objects in the key N_objs_no_pbh += key_data.shape[0] @@ -1897,12 +1899,13 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, pbh_hdf5_file.close() print('Checking totals') - print('-- %i original objects' % N_objs_no_pbh) - print('-- %i PBHs in the field' % N_PBHs_in_field) - print('-- %i new total objects' % N_objs_pbh) + print('- %i PBHS removed from %s' % (N_pbhs_removed, hdf5_file)) + print('--- %i original objects' % N_objs_no_pbh) + print('--- %i PBHs in the field' % N_PBHs_in_field) + print('--- %i new total objects' % N_objs_pbh) if N_objs_pbh == N_objs_no_pbh + N_PBHs_in_field: - print('-- Totals match!') + print('Binned PBHs equals total PBHs') else: print('** MISSING PBHs!! **') From aa6e548797bfd99d818dcee9583defc33ad57679 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 12:44:36 -0700 Subject: [PATCH 072/125] include runtime of galaxia --- popsycle/synthetic.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index e5b030c9..b6cabc5d 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -273,9 +273,13 @@ def run_galaxia(output_root, longitude, latitude, area, # Execute Galaxia cmd = 'galaxia -r galaxia_params.%s.txt' % output_root - print('** Executing Galaxia with galaxia_params.%s.txt **' % output_root) + print('** Galaxia **') + print('Executing with galaxia_params.%s.txt' % output_root) + t0 = time.time() _ = utils.execute(cmd) - print('** Galaxia complete **') + t1 = time.time() + print('Galaxia complete') + print('total runtime {0:f} s'.format(t1 - t0)) def _check_perform_pop_syn(ebf_file, output_root, iso_dir, From 61d5fd8b421829782edcd143786e056487608411 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 12:45:34 -0700 Subject: [PATCH 073/125] remove redundant title --- popsycle/synthetic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index b6cabc5d..ef99f9ad 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -273,7 +273,6 @@ def run_galaxia(output_root, longitude, latitude, area, # Execute Galaxia cmd = 'galaxia -r galaxia_params.%s.txt' % output_root - print('** Galaxia **') print('Executing with galaxia_params.%s.txt' % output_root) t0 = time.time() _ = utils.execute(cmd) From 07e2dce1468f8d399f37ddc44db637a331263d0e Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 12:45:47 -0700 Subject: [PATCH 074/125] fix typo --- popsycle/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index ef99f9ad..b94a1ba5 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -278,7 +278,7 @@ def run_galaxia(output_root, longitude, latitude, area, _ = utils.execute(cmd) t1 = time.time() print('Galaxia complete') - print('total runtime {0:f} s'.format(t1 - t0)) + print('Total runtime {0:f} s'.format(t1 - t0)) def _check_perform_pop_syn(ebf_file, output_root, iso_dir, From 042406564020bbf109c5446c778d88da1f5ec34c Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 12:48:55 -0700 Subject: [PATCH 075/125] change run time to runtime --- docs/PopSyCLE_example.ipynb | 2 +- popsycle/run.py | 2 +- popsycle/synthetic.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/PopSyCLE_example.ipynb b/docs/PopSyCLE_example.ipynb index 6fa3bae8..f7e65e3c 100755 --- a/docs/PopSyCLE_example.ipynb +++ b/docs/PopSyCLE_example.ipynb @@ -349,7 +349,7 @@ "Starting age bin 9.9\n", "Starting sub-bin 0\n", "Found 17651 stars out of mass range\n", - "Total run time is 52.840643 s\n", + "Total runtime is 52.840643 s\n", "******************** INFO **********************\n", "Total number of stars from Galaxia: 665179\n", "Total number of compact objects made: 43596\n", diff --git a/popsycle/run.py b/popsycle/run.py index dc77d4ba..0e6270ef 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -1046,7 +1046,7 @@ def run(): output_file='default') t1 = time.time() - print('run.py complete : total run time is {0:f} s'.format(t1 - t0)) + print('run.py complete : total runtime is {0:f} s'.format(t1 - t0)) if __name__ == '__main__': diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index b94a1ba5..fc1d4335 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -759,7 +759,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, del kdt_star_p, kdt_star_exbv t1 = time.time() - print('Total run time is {0:f} s'.format(t1 - t0)) + print('Total runtime is {0:f} s'.format(t1 - t0)) ########## # Figure out how much stuff got binned. From 9bad7e00003a93f5457200dada096f3cef303077 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 13:13:36 -0700 Subject: [PATCH 076/125] propagate add_pbh data in hdf5 --- popsycle/converter.py | 1 + popsycle/synthetic.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/popsycle/converter.py b/popsycle/converter.py index 4d63fcdf..9684846b 100644 --- a/popsycle/converter.py +++ b/popsycle/converter.py @@ -47,6 +47,7 @@ def convert_h5_array_dtype_to_compound_dtype(hdf5_file): hdf5_file_new = hdf5_file.replace('.h5', '.compound_dtype.h5') f_in = h5py.File(hdf5_file, 'r') f_out = h5py.File(hdf5_file_new, 'w') + f_out['add_pbh'] = False # Looping over all of the datasets for key in f_in: diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index fc1d4335..eed4a682 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -540,11 +540,13 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, long_bin_edges[wrap_id] -= 360 ########## - # Create h5py file to store lat/long binned output + # Create h5py file to store lat/long binned output and + # that this hdf5 file was created without / before add_pbh ########## h5file = h5py.File(output_root + '.h5', 'w') - dataset = h5file.create_dataset('lat_bin_edges', data=lat_bin_edges) - dataset = h5file.create_dataset('long_bin_edges', data=long_bin_edges) + h5file['lat_bin_edges'] = lat_bin_edges + h5file['long_bin_edges'] = long_bin_edges + h5file['add_pbh'] = False h5file.close() ########## @@ -767,7 +769,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, binned_counter = 0 hf = h5py.File(output_root + '.h5', 'r') for key in hf: - if 'bin_edges' not in key: + if 'bin_edges' not in key and 'add_pbh' not in key: binned_counter += len(hf[key]) ########## @@ -1618,8 +1620,9 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # Read in the hdf5 file that doesn't have PBHs. Product of perform_pop_syn. no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') key_list = list(no_pbh_hdf5_file) - # Delete lat_bin_edges and long_bin_edges from key_list. + # Delete lat_bin_edges, long_bin_edges, add_pbh from key_list. key_list = [key for key in key_list if 'bin_edges' not in key] + key_list = [key for key in key_list if 'add_pbh' not in key] # Get data from lat_bin_edges and long_bin_edges lat_bin = no_pbh_hdf5_file['lat_bin_edges'][:] @@ -1859,6 +1862,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # Opening the file with no PBHs and creating a new file for the PBHs added. no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') pbh_hdf5_file = h5py.File(output_hdf5_file, 'w') + pbh_hdf5_file['add_pbh'] = True # Appending the PBH data to the no PBH data and writing to the new .h5 file. N_objs_no_pbh = 0 From aed4d2f68776a254308d7faded9195e52d3eacfd Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 13:28:06 -0700 Subject: [PATCH 077/125] add ztf-i, check for existence of objects before appended to max_id_no_pbh --- popsycle/synthetic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index eed4a682..7b298813 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1633,7 +1633,8 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # Later used to set the IDs of the PBHs. max_id_no_pbh = [] for key in key_list: - # max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key][20])) + if len(no_pbh_hdf5_file[key]) == 0: + continue max_id_no_pbh.append(np.max(no_pbh_hdf5_file[key]['obj_id'])) max_id = np.amax(max_id_no_pbh) @@ -1847,6 +1848,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, if any(['ztf' in n for n in hdf5_dset_names]): pbh_data['ztf_g'] = np.full(len(d_galac), np.nan) pbh_data['ztf_r'] = np.full(len(d_galac), np.nan) + pbh_data['ztf_i'] = np.full(len(d_galac), np.nan) # Calculate the maximum and minimum l and b values for each dataset in the no PBH file, # so that we can determine which datasets to correctly add the PBHs. From 321571b7b6b350f44a7364c3fb1c8f8bb8113e21 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 14:17:21 -0700 Subject: [PATCH 078/125] check to see if certain fields are in hdf5_dset_names before appending to pbh_data --- popsycle/synthetic.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 7b298813..67b0537e 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1841,14 +1841,14 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, pbh_data['ubv_B'] = np.full(len(d_galac), np.nan) pbh_data['ubv_H'] = np.full(len(d_galac), np.nan) pbh_data['ubv_V'] = np.full(len(d_galac), np.nan) - pbh_data['teff'] = np.full(len(d_galac), np.nan) - pbh_data['grav'] = np.full(len(d_galac), np.nan) - pbh_data['mbol'] = np.full(len(d_galac), np.nan) - pbh_data['feh'] = np.full(len(d_galac), np.nan) - if any(['ztf' in n for n in hdf5_dset_names]): - pbh_data['ztf_g'] = np.full(len(d_galac), np.nan) - pbh_data['ztf_r'] = np.full(len(d_galac), np.nan) - pbh_data['ztf_i'] = np.full(len(d_galac), np.nan) + + # The first four elements may not be in the compound datatype + # due to legacy files. The ztf filters will only be present if + # the hdf5 file was created with additional_photometric_systems = ['ztf'] + for field in ['teff', 'grav', 'mbol', 'feh', + 'ztf_g', 'ztf_r', 'ztf_i']: + if field in hdf5_dset_names: + pbh_data[field] = np.full(len(d_galac), np.nan) # Calculate the maximum and minimum l and b values for each dataset in the no PBH file, # so that we can determine which datasets to correctly add the PBHs. From d50144e3835dd04a7c83834b3164f6858d77fa55 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 18:27:43 -0700 Subject: [PATCH 079/125] correct construction of lat_long_list --- popsycle/synthetic.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 67b0537e..8f7ef523 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1850,16 +1850,20 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, if field in hdf5_dset_names: pbh_data[field] = np.full(len(d_galac), np.nan) - # Calculate the maximum and minimum l and b values for each dataset in the no PBH file, - # so that we can determine which datasets to correctly add the PBHs. + # Calculate the maximum and minimum l and b values for each dataset in + # the no PBH file, so that we can determine which datasets to + # correctly append the PBHs. lat_long_list = [] - for idx in range(len(long_bin) - 1): - max_l = long_bin[idx + 1] - min_l = long_bin[idx] - for idx2 in range(len(lat_bin) - 1): - max_b = lat_bin[idx2 + 1] - min_b = lat_bin[idx2] - lat_long_list.append((min_l, max_l, min_b, max_b)) + for key in key_list: + idx_l = int(key.split('b')[0].replace('l', '')) + max_l = long_bin[idx_l + 1] + min_l = long_bin[idx_l] + + idx_b = int(key.split('b')[1]) + max_b = lat_bin[idx_b + 1] + min_b = lat_bin[idx_b] + + lat_long_list.append((min_l, max_l, min_b, max_b)) # Opening the file with no PBHs and creating a new file for the PBHs added. no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') From 5419810a420438f5b71478c48b26ed41ae3c14a7 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 18:45:40 -0700 Subject: [PATCH 080/125] uniform runtime statements, generate add_pbh log --- popsycle/run.py | 2 +- popsycle/synthetic.py | 90 +++++++++++++++++++++++++++++++++---------- 2 files changed, 70 insertions(+), 22 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index 0e6270ef..92f69141 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -1046,7 +1046,7 @@ def run(): output_file='default') t1 = time.time() - print('run.py complete : total runtime is {0:f} s'.format(t1 - t0)) + print('run.py runtime : {0:f} s'.format(t1 - t0)) if __name__ == '__main__': diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 8f7ef523..3ed76a8e 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -278,7 +278,7 @@ def run_galaxia(output_root, longitude, latitude, area, _ = utils.execute(cmd) t1 = time.time() print('Galaxia complete') - print('Total runtime {0:f} s'.format(t1 - t0)) + print('galaxia runtime : {0:f} s'.format(t1 - t0)) def _check_perform_pop_syn(ebf_file, output_root, iso_dir, @@ -761,7 +761,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, del kdt_star_p, kdt_star_exbv t1 = time.time() - print('Total runtime is {0:f} s'.format(t1 - t0)) + print('perform_pop_syn runtime : {0:f} s'.format(t1 - t0)) ########## # Figure out how much stuff got binned. @@ -1552,14 +1552,6 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, The scale radius of the Milky Way (in kpc). r_s = r_vir / c (virial radius / concentration index) Defaults to 18.6 kpc. The median value given in McMillan 2017. - rho_0: float - The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). - Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2017. - - n_lin: int - The number of times you want the density determined along the line of sight when calculating PBH positions - Defaults to 1000. Will need to make large if you are closer to the galactic center. - gamma: float The inner slope of the MW dark matter halo as described in LaCroix 2018. Gamma goes into the determination of the velocities and each value returns a slightly different distribution. @@ -1570,6 +1562,14 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, v_esc is used in calculating the velocities. Default is 550 km/s. Most papers cite values of 515-575. + rho_0: float + The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). + Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2017. + + n_lin: int + The number of times you want the density determined along the line of sight when calculating PBH positions + Defaults to 1000. Will need to make large if you are closer to the galactic center. + Optional Parameters ------------------- new_output_root : str @@ -1601,11 +1601,13 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, seed=seed) if new_output_root is None: + output_root = hdf5_file.replace('.h5', '') output_hdf5_file = hdf5_file.replace('.h5', '_pbh_tmp.h5') print('** WARNING **') print(" 'add_pbh' will overwrite %s, with PBHs appended to each key." % hdf5_file) print(" To generate a new hdf5 file, rerun 'add_pbh' with the 'new_output_root' argument.") else: + output_root = new_output_root output_hdf5_file = '%s.h5' % new_output_root ########## @@ -1911,7 +1913,61 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, no_pbh_hdf5_file.close() pbh_hdf5_file.close() - print('Checking totals') + # If 'new_output_root' is None, replace temporary file with original + if new_output_root is None: + os.remove(hdf5_file) + os.rename(output_hdf5_file, hdf5_file) + + t1 = time.time() + print('add_pbh runtime: {0:f} s'.format(t1 - t0)) + + ########## + # Make log file + ########## + now = datetime.datetime.now() + microlens_path = os.path.dirname(inspect.getfile(perform_pop_syn)) + microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=microlens_path).decode('ascii').strip() + dash_line = '-----------------------------' + '\n' + empty_line = '\n' + + line0 = 'FUNCTION INPUT PARAMETERS' + '\n' + line1 = 'hdf5_file , ' + hdf5_file + '\n' + line2 = 'ebf_file , ' + ebf_file + '\n' + line3 = 'fdm , ' + str(fdm) + '\n' + line4 = 'pbh_mass , ' + str(pbh_mass) + ' , (Msun)' + '\n' + line5 = 'r_max , ' + str(r_max) + ' , (kpc)' + '\n' + line6 = 'r_s , ' + str(r_s) + ' , (kpc)' + '\n' + line7 = 'gamma , ' + str(gamma) + '\n' + line8 = 'v_esc , ' + str(v_esc) + ' , (km/s)' + '\n' + line9 = 'rho_0 , ' + str(rho_0) + ' , (Msun / pc^3)' + '\n' + line10 = 'n_lin , ' + str(n_lin) + '\n' + line11 = 'new_output_root , ' + str(new_output_root) + '\n' + line12 = 'seed , ' + str(seed) + '\n' + + line13 = 'VERSION INFORMATION' + '\n' + line14 = str(now) + ' : creation date' + '\n' + line15 = microlens_hash + ' : microlens commit' + '\n' + + line16 = 'OTHER INFORMATION' + '\n' + line17 = str(t1 - t0) + ' : total runtime (s)' + '\n' + line18 = str(N_objs_no_pbh) + ' : original objects' + '\n' + line19 = str(N_PBHs_in_field) + ' : PBHs in the field' + '\n' + line20 = str(N_objs_pbh) + ' : new total objects' + '\n' + + line21 = 'FILES CREATED' + '\n' + line22 = output_hdf5_file + ' : HDF5 file' + '\n' + + with open(output_root + '_add_pbh.log', 'w') as out: + out.writelines([line0, dash_line, line1, line2, line3, line4, line5, + line6, line7, line8, line9, line10, line11, line12, + empty_line, line13, dash_line, line14, line15, + empty_line, line16, dash_line, line17, line18, line19, + line20, empty_line, line21, dash_line, line22]) + + ########## + # Informative print statements. + ########## print('- %i PBHS removed from %s' % (N_pbhs_removed, hdf5_file)) print('--- %i original objects' % N_objs_no_pbh) print('--- %i PBHs in the field' % N_PBHs_in_field) @@ -1922,14 +1978,6 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, else: print('** MISSING PBHs!! **') - # If 'new_output_root' is None, replace temporary file with original - if new_output_root is None: - os.remove(hdf5_file) - os.rename(output_hdf5_file, hdf5_file) - - t1 = time.time() - print('Total runtime: {0:f} s'.format(t1 - t0)) - return @@ -2212,7 +2260,7 @@ def calc_events(hdf5_file, output_root2, line12, dash_line, line13, line14, empty_line, line15, dash_line, line16, line17]) - print('Total runtime: {0:f} s'.format(t1 - t0)) + print('calc_events runtime : {0:f} s'.format(t1 - t0)) return @@ -3096,7 +3144,7 @@ def refine_events(input_root, filter_name, photometric_system, red_law, line8, dash_line, line9, line10, line11, empty_line, line12, dash_line, line13]) - print('Total runtime: {0:f} s'.format(t_1 - t_0)) + print('refined_events runtime : {0:f} s'.format(t_1 - t_0)) return From 71d401fd532baad90540e055fcb2a4fa6ca2cceb Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 18:47:21 -0700 Subject: [PATCH 081/125] add seed to perform_pop_syn log --- popsycle/synthetic.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 3ed76a8e..3397d7d8 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -797,28 +797,29 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, line4 = 'BH_kick_speed_mean , ' + str(BH_kick_speed_mean) + ' , (km/s)' + '\n' line5 = 'NS_kick_speed_mean , ' + str(NS_kick_speed_mean) + ' , (km/s)' + '\n' line6 = 'iso_dir , ' + iso_dir + '\n' + line7 = 'seed , ' + str(seed) + '\n' - line7 = 'VERSION INFORMATION' + '\n' - line8 = str(now) + ' : creation date' + '\n' - line9 = popstar_hash + ' : PopStar commit' + '\n' - line10 = microlens_hash + ' : microlens commit' + '\n' + line8 = 'VERSION INFORMATION' + '\n' + line9 = str(now) + ' : creation date' + '\n' + line10 = popstar_hash + ' : PopStar commit' + '\n' + line11 = microlens_hash + ' : microlens commit' + '\n' - line11 = 'OTHER INFORMATION' + '\n' - line12 = str(t1 - t0) + ' : total runtime (s)' + '\n' - line13 = str(n_stars) + ' : total stars from Galaxia' + '\n' - line14 = str(comp_counter) + ' : total compact objects made' + '\n' - line15 = str(binned_counter) + ' : total things binned' + '\n' + line12 = 'OTHER INFORMATION' + '\n' + line13 = str(t1 - t0) + ' : total runtime (s)' + '\n' + line14 = str(n_stars) + ' : total stars from Galaxia' + '\n' + line15 = str(comp_counter) + ' : total compact objects made' + '\n' + line16 = str(binned_counter) + ' : total things binned' + '\n' - line16 = 'FILES CREATED' + '\n' - line17 = output_root + '.h5 : HDF5 file' + '\n' - line18 = output_root + '_label.fits : label file' + '\n' + line17 = 'FILES CREATED' + '\n' + line18 = output_root + '.h5 : HDF5 file' + '\n' + line19 = output_root + '_label.fits : label file' + '\n' with open(output_root + '_perform_pop_syn.log', 'w') as out: out.writelines([line0, dash_line, line1, line2, line3, line4, line5, - line6, empty_line, line7, dash_line, line8, line9, - line10, empty_line, line11, dash_line, line12, line13, - line14, line15, empty_line, line16, dash_line, line17, - line18]) + line6, line7, empty_line, line8, dash_line, line9, + line10, line11, empty_line, line12, dash_line, line13, + line14, line15, line16, empty_line, line17, dash_line, + line18, line19]) ########## # Informative print statements. From 639d6a9d758fd02eba57593b6f97e614ff41f9e4 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 18:54:17 -0700 Subject: [PATCH 082/125] change microlens hash in logs to popsycle hash --- popsycle/synthetic.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 3397d7d8..17c9d889 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -781,10 +781,10 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, # Make log file ########## now = datetime.datetime.now() - microlens_path = os.path.dirname(inspect.getfile(perform_pop_syn)) + popsycle_path = os.path.dirname(inspect.getfile(perform_pop_syn)) popstar_path = os.path.dirname(inspect.getfile(imf)) - microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=microlens_path).decode('ascii').strip() + popsycle_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=popsycle_path).decode('ascii').strip() popstar_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=popstar_path).decode('ascii').strip() dash_line = '-----------------------------' + '\n' @@ -802,7 +802,7 @@ def perform_pop_syn(ebf_file, output_root, iso_dir, line8 = 'VERSION INFORMATION' + '\n' line9 = str(now) + ' : creation date' + '\n' line10 = popstar_hash + ' : PopStar commit' + '\n' - line11 = microlens_hash + ' : microlens commit' + '\n' + line11 = popsycle_hash + ' : PopSyCLE commit' + '\n' line12 = 'OTHER INFORMATION' + '\n' line13 = str(t1 - t0) + ' : total runtime (s)' + '\n' @@ -1926,9 +1926,9 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # Make log file ########## now = datetime.datetime.now() - microlens_path = os.path.dirname(inspect.getfile(perform_pop_syn)) - microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=microlens_path).decode('ascii').strip() + popsycle_path = os.path.dirname(inspect.getfile(perform_pop_syn)) + popsycle_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=popsycle_path).decode('ascii').strip() dash_line = '-----------------------------' + '\n' empty_line = '\n' @@ -1948,7 +1948,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, line13 = 'VERSION INFORMATION' + '\n' line14 = str(now) + ' : creation date' + '\n' - line15 = microlens_hash + ' : microlens commit' + '\n' + line15 = popsycle_hash + ' : PopSyCLE commit' + '\n' line16 = 'OTHER INFORMATION' + '\n' line17 = str(t1 - t0) + ' : total runtime (s)' + '\n' @@ -2228,9 +2228,9 @@ def calc_events(hdf5_file, output_root2, ########## now = datetime.datetime.now() radius_cut = radius_cut / 1000.0 # back to arcsec - microlens_path = os.path.dirname(inspect.getfile(perform_pop_syn)) - microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=microlens_path).decode('ascii').strip() + popsycle_path = os.path.dirname(inspect.getfile(perform_pop_syn)) + popsycle_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=popsycle_path).decode('ascii').strip() dash_line = '-----------------------------' + '\n' empty_line = '\n' line0 = 'FUNCTION INPUT PARAMETERS' + '\n' @@ -2244,7 +2244,7 @@ def calc_events(hdf5_file, output_root2, line8 = 'n_proc , ' + str(n_proc) + '\n' line9 = 'VERSION INFORMATION' + '\n' line10 = str(now) + ' : creation date' + '\n' - line11 = microlens_hash + ' : microlens commit' + '\n' + line11 = popsycle_hash + ' : PopSyCLE commit' + '\n' line12 = 'OTHER INFORMATION' + '\n' line13 = str(t1 - t0) + ' : total runtime (s)' + '\n' @@ -3112,10 +3112,10 @@ def refine_events(input_root, filter_name, photometric_system, red_law, # Make log file ########## now = datetime.datetime.now() - microlens_path = os.path.dirname(inspect.getfile(perform_pop_syn)) + popsycle_path = os.path.dirname(inspect.getfile(perform_pop_syn)) popstar_path = os.path.dirname(inspect.getfile(imf)) - microlens_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=microlens_path).decode('ascii').strip() + popsycle_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=popsycle_path).decode('ascii').strip() popstar_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=popstar_path).decode('ascii').strip() dash_line = '-----------------------------' + '\n' @@ -3129,7 +3129,7 @@ def refine_events(input_root, filter_name, photometric_system, red_law, line4 = 'VERSION INFORMATION' + '\n' line5 = str(now) + ' : creation date' + '\n' line6 = popstar_hash + ' : PopStar commit' + '\n' - line7 = microlens_hash + ' : microlens commit' + '\n' + line7 = popsycle_hash + ' : PopSyCLE commit' + '\n' line8 = 'OTHER INFORMATION' + '\n' line9 = str(t_1 - t_0) + ' : total runtime (s)' + '\n' From 6c85287d6bc875dbe5a77984c2ea45d032867454 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 19:16:45 -0700 Subject: [PATCH 083/125] change example notebook back --- docs/PopSyCLE_example.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/PopSyCLE_example.ipynb b/docs/PopSyCLE_example.ipynb index f7e65e3c..6fa3bae8 100755 --- a/docs/PopSyCLE_example.ipynb +++ b/docs/PopSyCLE_example.ipynb @@ -349,7 +349,7 @@ "Starting age bin 9.9\n", "Starting sub-bin 0\n", "Found 17651 stars out of mass range\n", - "Total runtime is 52.840643 s\n", + "Total run time is 52.840643 s\n", "******************** INFO **********************\n", "Total number of stars from Galaxia: 665179\n", "Total number of compact objects made: 43596\n", From 5c7e6cf5cc82c3d2c7a598b66742a588a984ab12 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 19:21:02 -0700 Subject: [PATCH 084/125] move bool on slurm_config to below check --- popsycle/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index 92f69141..d22090cf 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -583,10 +583,10 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, # Load the slurm configuration file slurm_config = load_config_file(slurm_config_filename) - slurm_config['include_constraint'] = bool(slurm_config['include_constraint']) # Check pipeline stages for valid inputs _check_slurm_config(slurm_config, walltime) + slurm_config['include_constraint'] = bool(slurm_config['include_constraint']) if not skip_galaxia: _check_run_galaxia(output_root=output_root, longitude=longitude, From 66a1ea61511166657bdb50f785d3d541f262a601 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 23 Mar 2020 19:28:49 -0700 Subject: [PATCH 085/125] remove pandas dependency --- popsycle/synthetic.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 17c9d889..30d9b6a6 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -11,7 +11,6 @@ """ import numpy as np -import pandas as pd import h5py import math import astropy @@ -1766,11 +1765,14 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # From Lacroix et al 2018, Figure 11 (top left panel) data_dir = '%s/data' % os.path.dirname(inspect.getfile(add_pbh)) if gamma == 1: - vel_data = pd.read_csv('%s/radial_velocity_profile_steep.csv' % data_dir) + vel_data = np.genfromtxt('%s/radial_velocity_profile_steep.csv' % data_dir, + names=True, delimiter=',') elif gamma == .25: - vel_data = pd.read_csv('%s/radial_velocity_profile_shallow.csv' % data_dir) + vel_data = np.genfromtxt('%s/radial_velocity_profile_shallow.csv' % data_dir, + names=True, delimiter=',') elif gamma == .5: - vel_data = pd.read_csv('%s/radial_velocity_profile_middle.csv' % data_dir) + vel_data = np.genfromtxt('%s/radial_velocity_profile_middle.csv' % data_dir, + names=True, delimiter=',') else: raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) From cabbbeef26685f13e4946be42802abab459f5987 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Tue, 24 Mar 2020 09:24:28 -0700 Subject: [PATCH 086/125] remove extraneous imports --- popsycle/synthetic.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 29e559ee..78f1c5df 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -12,7 +12,6 @@ import numpy as np import h5py import math -import astropy from astropy import units from scipy.stats import maxwell import astropy.coordinates as coord @@ -23,7 +22,6 @@ from astropy.table import vstack from popstar.imf import imf from popstar import synthetic, evolution, reddening, ifmr -import scipy from scipy.interpolate import interp1d from scipy.spatial import cKDTree from scipy import special, integrate, interpolate @@ -42,7 +40,6 @@ from popsycle.filters import transform_ubv_to_ztf import shutil from popsycle import utils -import matplotlib.pyplot as plt ########## @@ -1756,9 +1753,9 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, distance=d_galac * units.kpc) galacto_pbh = galactic_pbh.transform_to( coord.Galactocentric(representation_type='spherical')) - cart_pbh = astropy.coordinates.cartesian_to_spherical(galacto_pbh.x, - galacto_pbh.y, - galacto_pbh.z) + cart_pbh = coord.cartesian_to_spherical(galacto_pbh.x, + galacto_pbh.y, + galacto_pbh.z) pbh_r_galacto = cart_pbh[0] # Inner slope of the MW halo @@ -1785,9 +1782,9 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, rand_cdf = np.array([]) for a_val in a: - cdf = scipy.special.erf(v_vals / (a_val * 2 ** (1 / 2))) - ( - ((2 / np.pi) ** (1 / 2)) * ((v_vals * np.exp( - -v_vals ** 2 / 2 * a_val ** 2)) / a_val)) + cdf = special.erf(v_vals / (a_val * 2 ** (1 / 2))) - ( + ((2 / np.pi) ** (1 / 2)) * ((v_vals * np.exp( + -v_vals ** 2 / 2 * a_val ** 2)) / a_val)) rand_cdf = np.append(rand_cdf, np.random.uniform(0, np.amax(cdf))) interpreted_rms_velocities = np.interp(rand_cdf, cdf, v_vals) @@ -1797,8 +1794,8 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, long_vel = np.random.uniform(0, 2 * np.pi, len(d_galac)) # Transforming velocities to cartesian to get vx, vy, and vz. - cart_vel = astropy.coordinates.spherical_to_cartesian( - interpreted_rms_velocities, lat_vel, long_vel) + cart_vel = coord.spherical_to_cartesian(interpreted_rms_velocities, + lat_vel, long_vel) # Load up a numpy array comp_dtype = _generate_comp_dtype(hdf5_dset_names) @@ -1822,8 +1819,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, pbh_data['popid'] = np.full(len(d_galac), 10) pbh_data['rem_id'] = np.full(len(d_galac), 104) - cart_helio = astropy.coordinates.spherical_to_cartesian(d_galac, b_galac, - l_galac) + cart_helio = coord.spherical_to_cartesian(d_galac, b_galac, l_galac) pbh_data['px'] = cart_helio[0] pbh_data['py'] = cart_helio[1] pbh_data['pz'] = cart_helio[2] From 1c81ebe94993c193f69e11b563bea58303d7ff78 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Tue, 24 Mar 2020 13:58:21 -0700 Subject: [PATCH 087/125] enforce pbh_config_filename as absolute path --- popsycle/run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/popsycle/run.py b/popsycle/run.py index d22090cf..1ae3e53b 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -606,6 +606,8 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, overwrite=overwrite, seed=seed) if pbh_config_filename is not None: + # Enforce pbh_config_filename is an absolute path + pbh_config_filename = os.path.abspath(pbh_config_filename) pbh_config = load_config_file(pbh_config_filename) _check_add_pbh(hdf5_file='test.h5', ebf_file='test.ebf', From 23ca39797980930852233db0371e360b1a466b48 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Tue, 24 Mar 2020 14:30:54 -0700 Subject: [PATCH 088/125] enforce pbh_config_filename as absolute path --- popsycle/run.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index 1ae3e53b..e56c33d0 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -566,27 +566,48 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, """ # Check for files + # Enforce slurm_config_filename is an absolute path + slurm_config_filename = os.path.abspath(slurm_config_filename) + # Check for slurm config file. Exit if not present. if not os.path.exists(slurm_config_filename): raise Exception('Slurm configuration file {0} does not exist. ' 'Write out file using ' 'run.generate_slurm_config_file ' 'before proceeding.'.format(slurm_config_filename)) + + # Enforce popsycle_config_filename is an absolute path + popsycle_config_filename = os.path.abspath(popsycle_config_filename) + # Check for popsycle config file. Exit if not present. if not os.path.exists(popsycle_config_filename): raise Exception('PopSyCLE configuration file {0} does not exist. ' 'Write out file using ' 'run.generate_popsycle_config_file ' 'before proceeding.'.format(popsycle_config_filename)) - # Enforce popsycle_config_filename is an absolute path - popsycle_config_filename = os.path.abspath(popsycle_config_filename) - popsycle_config = load_config_file(popsycle_config_filename) - - # Load the slurm configuration file + if pbh_config_filename is not None: + # Enforce pbh_config_filename is an absolute path + pbh_config_filename = os.path.abspath(pbh_config_filename) + # Check for pbh config file, if provided. Exit if not present. + if not os.path.exists(pbh_config_filename): + raise Exception('PBH configuration file {0} does not exist. ' + 'Write out file using ' + 'run.generate_pbh_config_file ' + 'before proceeding.'.format(pbh_config_filename)) + + # Load the configuration files + # Load slurm config slurm_config = load_config_file(slurm_config_filename) + slurm_config['include_constraint'] = bool(slurm_config['include_constraint']) # Enforce boolean for 'include_constraint' + # Load popsycle config + popsycle_config = load_config_file(popsycle_config_filename) + if popsycle_config['bin_edges_number'] == 'None': # Enforce None for 'bin_edges_number' + popsycle_config['bin_edges_number'] = None + # Load pbh config, if provided. + if pbh_config_filename is not None: + pbh_config = load_config_file(pbh_config_filename) # Check pipeline stages for valid inputs _check_slurm_config(slurm_config, walltime) - slurm_config['include_constraint'] = bool(slurm_config['include_constraint']) if not skip_galaxia: _check_run_galaxia(output_root=output_root, longitude=longitude, @@ -594,8 +615,6 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, area=area, seed=seed) if not skip_perform_pop_syn: - if popsycle_config['bin_edges_number'] == 'None': - popsycle_config['bin_edges_number'] = None _check_perform_pop_syn(ebf_file='test.ebf', output_root=output_root, iso_dir=popsycle_config['isochrones_dir'], @@ -606,9 +625,6 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, overwrite=overwrite, seed=seed) if pbh_config_filename is not None: - # Enforce pbh_config_filename is an absolute path - pbh_config_filename = os.path.abspath(pbh_config_filename) - pbh_config = load_config_file(pbh_config_filename) _check_add_pbh(hdf5_file='test.h5', ebf_file='test.ebf', fdm=pbh_config['fdm'], @@ -639,7 +655,6 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, overwrite=overwrite, output_file='default') - # Make a run directory for the PopSyCLE output path_run = os.path.abspath(path_run) if not os.path.exists(path_run): From bc79812efee6b33a8fdad6ba4da5c07b8fc72e18 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 25 Mar 2020 13:29:47 -0700 Subject: [PATCH 089/125] force pbh_r_galacto.value --- popsycle/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 78f1c5df..fd43e020 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1774,7 +1774,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) # Interpolating v values from the above data, given the PBH r values. - pbh_vrms = np.interp(pbh_r_galacto, vel_data['r'], vel_data['v']) + pbh_vrms = np.interp(pbh_r_galacto.value, vel_data['r'], vel_data['v']) v_vals = np.arange(0, v_esc) # Goes from v to v_esc a = (1 / 2) * pbh_vrms * ((np.pi / 2) ** (1 / 2)) From 8584a55af681e75a136a1198655ccde532aab24d Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Wed, 25 Mar 2020 20:59:32 -0700 Subject: [PATCH 090/125] Added a flag for diagnostic plots --- popsycle/add_pbh_plots.py | 183 ++++++++++++++++++++++++++++++++++++++ popsycle/run.py | 3 + popsycle/synthetic.py | 33 ++++++- 3 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 popsycle/add_pbh_plots.py diff --git a/popsycle/add_pbh_plots.py b/popsycle/add_pbh_plots.py new file mode 100644 index 00000000..509c9e3d --- /dev/null +++ b/popsycle/add_pbh_plots.py @@ -0,0 +1,183 @@ +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.backends.backend_pdf import PdfPages + +def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin_l, galactocen_lin_spherical_distance, + galactocen_lin_spherical_b, galactocen_lin_spherical_l, rho_lin, r_max, n_lin, cdf_los, + x_cyl, y_cyl, r_cyl, r_proj_los_cyl, n_pbh, d_galac, b_galac, l_galac, area_proj_los_cyl, + mask_obs_cone, field_of_view_radius, l_radian, b_radian, f_cdf_d, pbh_mass): + + ################################################################################## + #Line of Sight - Galactic Coordinates + ################################################################################## + fig1, axs = plt.subplots(3, sharex=True, gridspec_kw={'hspace': 0}, figsize=(8,8)) + axs[0].plot(galactic_lin_distance, '.', label='Distance', c='C0') + axs[1].plot(galactic_lin_b, '.', label='Latitude',c='C1') + axs[2].plot(galactic_lin_l, '.', label='Longitude', c='C2') + + + plt.legend(loc='best') + # Hide x labels and tick labels for all but bottom plot. + for ax in axs: + ax.label_outer() + ax.legend(loc='upper left') + + axs[0].set_title('Line of Sight Linear Space Galactic Coordinates') + axs[0].set_ylabel('[kpc]') + + axs[1].set_ylabel('[deg]') + + axs[2].set_ylabel('[deg]') + axs[2].set_xlabel('Line-of-Sight Linear Space Coordinate Index') + + axs[0].set_ylim(0, axs[0].set_ylim()[1]) + + ################################################################################## + #Line of Sight - Galactocentric Coordinates + ################################################################################## + fig2, axs = plt.subplots(3, sharex=True, gridspec_kw={'hspace': 0}, figsize=(8,8)) + axs[0].plot(galactic_lin_distance, + galactocen_lin_spherical_distance, '.', label='Distance', c='C0') + axs[1].plot(galactic_lin_distance, + galactocen_lin_spherical_b, '.', label='Latitude', c='C1') + axs[2].plot(galactic_lin_distance, + galactocen_lin_spherical_l, '.', label='Longitude',c='C2') + + plt.legend(loc='best') + # Hide x labels and tick labels for all but bottom plot. + for ax in axs: + ax.label_outer() + ax.legend(loc='upper left') + + axs[0].set_title('Line of Sight Linear Space\nGalactocentric Coordinates') + axs[0].set_ylabel('[kpc]') + + axs[1].set_ylabel('[deg]') + + axs[2].set_ylabel('[deg]') + axs[2].set_xlabel('Galactic Coordinate Distance [kpc]') + + axs[0].set_ylim(0, axs[0].set_ylim()[1]) + + ################################################################################## + #Denisty along Line of Sight + ################################################################################## + fig3, (ax1, ax2) = plt.subplots(1, 2, figsize=(9,4), + sharey=True, gridspec_kw={'wspace': 0}) + # + ax1.semilogy(galactic_lin_distance, + rho_lin, '.',c='k') + ax1.set_ylabel('DENSITY Msun*pc^-3') + ax1.set_xlabel('Galactic Coordinate Distance [kpc]') + ax1.set_xlim(0, ax1.set_xlim()[1]) + ax2.semilogy(galactocen_lin_spherical_distance, + rho_lin, '.',c='k') + ax2.set_xlabel('Galactocentric Distance [kpc]') + ax2.set_xlim(0, ax2.set_xlim()[1]) + + plt.suptitle('Milky Way Density along Line-of-Sight') + + ################################################################################## + #Cumulative Projected Desnity + ################################################################################## + fig4 = plt.figure(figsize=(10,7)) + + plt.plot(galactic_lin_distance, + np.cumsum(rho_lin) * r_max / n_lin, + '.', label='Density',c='k') + plt.legend(loc='best') + plt.ylabel('Cumulative Projected Density} [M_sun*pc^{-2}]$') + plt.xlabel('Galactic Distance [kpc]') + plt.xlim(0, plt.xlim()[1]) + + ################################################################################## + #Discrete CDF + ################################################################################## + fig5 = plt.figure(figsize=(10,7)) + plt.plot(cdf_los, lw=3) + plt.ylabel('CDF') + plt.xlabel('Line-of-Sight Linear Space Coordinate Index') + + ################################################################################## + #Interpolated CDF + ################################################################################## + fig6 = plt.figure(figsize=(10,7)) + x = np.linspace(0,1,100) + y = f_cdf_d(x) + plt.plot(y, x, lw=3) + plt.ylabel('CDF') + plt.xlabel('Galactic Coordinate Distance along LOS [kpc]') + + ################################################################################## + #Cylindrical distribution of PBHs + #Infered Radii Distribution + #Galactic Distance Distribution + ################################################################################## + fig7, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(14,4)) + # Create a plot to look at the projected cylindrical distribution of PBH + ax1.plot(x_cyl * 1000, y_cyl * 1000, ',') + ax1.set_aspect('equal', 'datalim') + ax1.set_ylabel('Cylindrical LOS Samples l-axis [pc]') + ax1.set_xlabel('Cylindrical LOS Samples b-axis [pc]') + # Create a histogram of the infered radii to make sure that it shows the expected linear trend + bins = 20 + ax2.hist(r_cyl * 1000, bins=bins, label='Sampled Number') + ax2.plot((0, r_proj_los_cyl * 1000), (0, 2 * n_pbh / bins), c='C2', label='Expected Number') + ax2.legend(loc='upper left') + ax2.set_ylabel('Number of PBH per LOS Cylindrical Annuli') + ax2.set_xlabel('Cylindrical Radius [pc]') + + # Create a histogram of the galactic distance distribution + bins=100 + n_per_bin, _, _ = ax3.hist(d_galac, bins=bins, label='Sampled Number') + ax3.plot(galactic_lin_distance, + rho_lin * area_proj_los_cyl / pbh_mass * r_max / bins * 1000**3,c='C2', label='Approximate Expectation') + ax3.legend(loc='best') + ax3.set_ylabel('Number of PBH') + ax3.set_xlabel('Galactic Distance [kpc]') + + fig7.tight_layout(pad=2.0) + + plt.suptitle('Distribution of PBHs in Cylindrical Line-of-Sight Tube (i.e. not light cone)') + + ################################################################################## + #Checking Light Cone Boundaries + ################################################################################## + fig8, (ax1, ax2) = plt.subplots(2, 1, figsize=(5,8), + sharex=True, gridspec_kw={'hspace': 0}) + ax1.plot(d_galac[~mask_obs_cone], r_cyl[~mask_obs_cone] * 1000, + '.', alpha=0.1, label='Outside Light Cone') + ax1.plot(d_galac[mask_obs_cone], r_cyl[mask_obs_cone] * 1000, + '.', alpha=0.1, label='Inside Light Cone') + ax1.plot((0,r_max), + (0, field_of_view_radius * np.pi / 180 * r_max *1000), + label='Light Cone Boundry') + ax1.legend(loc='upper left') + ax1.set_ylabel('Radius on LOS Cylinder [pc]') + + ax2.plot(d_galac[~mask_obs_cone], np.sqrt((l_galac[~mask_obs_cone] - l_radian)**2 + + (b_galac[~mask_obs_cone] - b_radian)**2) * 180/np.pi, + '.', alpha=0.1, label='Outside Light Cone' ) + ax2.plot(d_galac[mask_obs_cone], np.sqrt((l_galac[mask_obs_cone] - l_radian)**2 + + (b_galac[mask_obs_cone] - b_radian)**2) * 180/np.pi, + '.', alpha=0.1, label='Inside Light Cone') + ax2.plot((0,r_max), + (field_of_view_radius, field_of_view_radius), label='Light Cone Boundry') + ax2.set_ylim(0,5*field_of_view_radius) + ax2.legend(loc='best') + ax2.set_ylabel('Field of View Radius [deg]') + ax2.set_xlabel('Galactic Coordinate Distance [kpc]') + + ################################################################################## + #Saving all plots to PDF + ################################################################################## + pp = PdfPages('diagnostic_plots_'+output_root+'.pdf') + pp.savefig(fig1) + pp.savefig(fig2) + pp.savefig(fig3) + pp.savefig(fig4) + pp.savefig(fig5) + pp.savefig(fig6) + pp.savefig(fig7) + pp.savefig(fig8) + pp.close() \ No newline at end of file diff --git a/popsycle/run.py b/popsycle/run.py index e56c33d0..d9694aa3 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -635,6 +635,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], + diagnostic_plots=False, new_output_root=None, seed=seed) if not skip_calc_events: @@ -935,6 +936,7 @@ def run(): v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], + diagnostic_plots=False, new_output_root=None, seed=args.seed) if not args.skip_calc_events: @@ -1013,6 +1015,7 @@ def run(): v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], + diagnostic_plots=False, new_output_root=None, seed=args.seed) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index fd43e020..258e2cde 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1410,8 +1410,8 @@ def rho_dmhalo(r, rho_0=.0093, r_s=18.6, gamma=1): def _check_add_pbh(hdf5_file, ebf_file, fdm, pbh_mass, - r_max, r_s, gamma, v_esc, - rho_0, n_lin, new_output_root, seed): + r_max, r_s, gamma, v_esc, rho_0, + n_lin, diagnostic_plots, new_output_root, seed): """ Checks that the inputs of add_pbj are valid @@ -1459,6 +1459,10 @@ def _check_add_pbh(hdf5_file, ebf_file, v_esc is used in calculating the velocities. Default is 550 km/s. Most papers cite values of 515-575. + diagnostic_plots: bool + If set to True, pbh_diagnostic_plots.py is run, and diagnostic plots are saved into a png file. + Default False. + new_output_root : str If set to None, 'add_pbh' overwrites the original hdf5 file with a new hdf5 file of the same name. If set to a string, this string is the @@ -1506,6 +1510,10 @@ def _check_add_pbh(hdf5_file, ebf_file, if type(n_lin) != int: raise Exception('n_lin (%s) must be an integerr.' % str(n_lin)) + if diagnostic_plots is not False: + if type(diagnostic_plots) != bool: + raise Exception('diagnostic plot (%s) must be False or a boolean.' % str(diagnostic_plots)) + if new_output_root is not None: if type(new_output_root) != str: raise Exception('new_output_root (%s) must be None or a string.' % str(new_output_root)) @@ -1517,7 +1525,7 @@ def _check_add_pbh(hdf5_file, ebf_file, def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, r_max=16.6, r_s=18.6, gamma=1, v_esc=550, - rho_0=0.0093, n_lin=1000, + rho_0=0.0093, n_lin=1000, diagnostic_plots=False, new_output_root=None, seed=None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, @@ -1569,6 +1577,10 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, Optional Parameters ------------------- + diagnostic_plots: bool + If set to True, pbh_diagnostic_plots.py is run, and diagnostic plots are saved into a png file. + Default False. + new_output_root : str If set to None, 'add_pbh' overwrites the original hdf5 file with a new hdf5 file of the same name. If set to a string, this string is the @@ -1594,6 +1606,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, fdm=fdm, pbh_mass=pbh_mass, r_max=r_max, r_s=r_s, gamma=gamma, v_esc=v_esc, rho_0=rho_0, n_lin=n_lin, + diagnostic_plots=diagnostic_plots, new_output_root=new_output_root, seed=seed) @@ -1731,6 +1744,20 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, l_galac = r_cyl * np.cos(theta) / np.cos( b_radian) / d_galac + l_radian # rad + if diagnostic_plots: + print('Saving diagnostic plots') + from popsycle.add_pbh_plots import print_plots + print_plots(output_root=output_root, galactic_lin_distance=galactic_lin.distance.kpc, + galactic_lin_b=galactic_lin.b.deg, galactic_lin_l=galactic_lin.l.deg, + galactocen_lin_spherical_distance=galactocen_lin.spherical.distance.kpc, + galactocen_lin_spherical_b=galactocen_lin.spherical.lat.deg, + galactocen_lin_spherical_l=galactocen_lin.spherical.lon.deg, rho_lin=rho_lin, + r_max=r_max, n_lin=n_lin, cdf_los=cdf_los, + x_cyl=x_cyl, y_cyl=y_cyl, r_cyl=r_cyl, r_proj_los_cyl=r_proj_los_cyl, n_pbh=n_pbh, + d_galac=d_galac, b_galac=b_galac, l_galac=l_galac, area_proj_los_cyl=area_proj_los_cyl, + mask_obs_cone=mask_obs_cone, field_of_view_radius=field_of_view_radius, l_radian=l_radian, + b_radian=b_radian, f_cdf_d=f_cdf_d, pbh_mass=pbh_mass) + d_galac = d_galac[mask_obs_cone] b_galac = b_galac[mask_obs_cone] l_galac = l_galac[mask_obs_cone] From e8dfdf81168e2f87114b8b694cc0cc2d9519a669 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Wed, 25 Mar 2020 21:09:16 -0700 Subject: [PATCH 091/125] removed diagnostic_plots --- popsycle/run.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index d9694aa3..e56c33d0 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -635,7 +635,6 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], - diagnostic_plots=False, new_output_root=None, seed=seed) if not skip_calc_events: @@ -936,7 +935,6 @@ def run(): v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], - diagnostic_plots=False, new_output_root=None, seed=args.seed) if not args.skip_calc_events: @@ -1015,7 +1013,6 @@ def run(): v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], - diagnostic_plots=False, new_output_root=None, seed=args.seed) From da1edb8e12979cd7d4492bcfc479a8e440ba0e00 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Wed, 25 Mar 2020 21:10:20 -0700 Subject: [PATCH 092/125] fixed errors/typos with diagnostic_plots --- popsycle/synthetic.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 258e2cde..4b04d6a4 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1460,7 +1460,7 @@ def _check_add_pbh(hdf5_file, ebf_file, Default is 550 km/s. Most papers cite values of 515-575. diagnostic_plots: bool - If set to True, pbh_diagnostic_plots.py is run, and diagnostic plots are saved into a png file. + If set to True, pbh_diagnostic_plots.py is run, and diagnostic plots are saved into a pdf file. Default False. new_output_root : str @@ -1510,9 +1510,8 @@ def _check_add_pbh(hdf5_file, ebf_file, if type(n_lin) != int: raise Exception('n_lin (%s) must be an integerr.' % str(n_lin)) - if diagnostic_plots is not False: - if type(diagnostic_plots) != bool: - raise Exception('diagnostic plot (%s) must be False or a boolean.' % str(diagnostic_plots)) + if type(diagnostic_plots) != bool: + raise Exception('diagnostic_plots (%s) must be a boolean.' % str(diagnostic_plots)) if new_output_root is not None: if type(new_output_root) != str: @@ -1578,7 +1577,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, Optional Parameters ------------------- diagnostic_plots: bool - If set to True, pbh_diagnostic_plots.py is run, and diagnostic plots are saved into a png file. + If set to True, pbh_diagnostic_plots.py is run, and diagnostic plots are saved into a pdf file. Default False. new_output_root : str From a3afc224eedcfc5236e44ee95fa05ea90b46895a Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 25 Mar 2020 22:12:37 -0700 Subject: [PATCH 093/125] close all figs --- popsycle/add_pbh_plots.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/popsycle/add_pbh_plots.py b/popsycle/add_pbh_plots.py index 509c9e3d..054310c1 100644 --- a/popsycle/add_pbh_plots.py +++ b/popsycle/add_pbh_plots.py @@ -10,7 +10,9 @@ def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin ################################################################################## #Line of Sight - Galactic Coordinates ################################################################################## + fig_arr = [] fig1, axs = plt.subplots(3, sharex=True, gridspec_kw={'hspace': 0}, figsize=(8,8)) + fig_arr.append(fig1) axs[0].plot(galactic_lin_distance, '.', label='Distance', c='C0') axs[1].plot(galactic_lin_b, '.', label='Latitude',c='C1') axs[2].plot(galactic_lin_l, '.', label='Longitude', c='C2') @@ -36,6 +38,7 @@ def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin #Line of Sight - Galactocentric Coordinates ################################################################################## fig2, axs = plt.subplots(3, sharex=True, gridspec_kw={'hspace': 0}, figsize=(8,8)) + fig_arr.append(fig2) axs[0].plot(galactic_lin_distance, galactocen_lin_spherical_distance, '.', label='Distance', c='C0') axs[1].plot(galactic_lin_distance, @@ -64,6 +67,7 @@ def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin ################################################################################## fig3, (ax1, ax2) = plt.subplots(1, 2, figsize=(9,4), sharey=True, gridspec_kw={'wspace': 0}) + fig_arr.append(fig3) # ax1.semilogy(galactic_lin_distance, rho_lin, '.',c='k') @@ -81,6 +85,7 @@ def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin #Cumulative Projected Desnity ################################################################################## fig4 = plt.figure(figsize=(10,7)) + fig_arr.append(fig4) plt.plot(galactic_lin_distance, np.cumsum(rho_lin) * r_max / n_lin, @@ -94,6 +99,7 @@ def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin #Discrete CDF ################################################################################## fig5 = plt.figure(figsize=(10,7)) + fig_arr.append(fig5) plt.plot(cdf_los, lw=3) plt.ylabel('CDF') plt.xlabel('Line-of-Sight Linear Space Coordinate Index') @@ -102,6 +108,7 @@ def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin #Interpolated CDF ################################################################################## fig6 = plt.figure(figsize=(10,7)) + fig_arr.append(fig6) x = np.linspace(0,1,100) y = f_cdf_d(x) plt.plot(y, x, lw=3) @@ -114,6 +121,7 @@ def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin #Galactic Distance Distribution ################################################################################## fig7, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(14,4)) + fig_arr.append(fig7) # Create a plot to look at the projected cylindrical distribution of PBH ax1.plot(x_cyl * 1000, y_cyl * 1000, ',') ax1.set_aspect('equal', 'datalim') @@ -145,6 +153,7 @@ def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin ################################################################################## fig8, (ax1, ax2) = plt.subplots(2, 1, figsize=(5,8), sharex=True, gridspec_kw={'hspace': 0}) + fig_arr.append(fig8) ax1.plot(d_galac[~mask_obs_cone], r_cyl[~mask_obs_cone] * 1000, '.', alpha=0.1, label='Outside Light Cone') ax1.plot(d_galac[mask_obs_cone], r_cyl[mask_obs_cone] * 1000, @@ -171,13 +180,8 @@ def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin ################################################################################## #Saving all plots to PDF ################################################################################## - pp = PdfPages('diagnostic_plots_'+output_root+'.pdf') - pp.savefig(fig1) - pp.savefig(fig2) - pp.savefig(fig3) - pp.savefig(fig4) - pp.savefig(fig5) - pp.savefig(fig6) - pp.savefig(fig7) - pp.savefig(fig8) - pp.close() \ No newline at end of file + pp = PdfPages(output_root+'_pbh_diagnostic_plots.pdf') + for fig in fig_arr: + pp.savefig(fig) + fig.close() + pp.close() From 3b2f6889cee9249e432d8aa02beb333c0785c9fa Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 25 Mar 2020 22:13:11 -0700 Subject: [PATCH 094/125] pass diagnostic_plots=True to _check_add_pbh --- popsycle/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index e56c33d0..451c22be 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -635,6 +635,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], + diagnostic_plots=True, new_output_root=None, seed=seed) if not skip_calc_events: @@ -935,6 +936,7 @@ def run(): v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], + diagnostic_plots=True, new_output_root=None, seed=args.seed) if not args.skip_calc_events: @@ -1013,7 +1015,6 @@ def run(): v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], - new_output_root=None, seed=args.seed) if not args.skip_calc_events: From 3910c6df6104d31abbb3ec5e5947ba0e4e0db4a3 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 3 Apr 2020 09:55:36 -0700 Subject: [PATCH 095/125] select galaxia_exe in run_galaxia --- popsycle/run.py | 2 ++ popsycle/synthetic.py | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index c59dab5e..ed102824 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -503,6 +503,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, longitude=longitude, latitude=latitude, area=area, + galaxia_exe='galaxia', seed=seed) if not skip_perform_pop_syn: if popsycle_config['bin_edges_number'] == 'None': @@ -772,6 +773,7 @@ def run(): longitude=field_config['longitude'], latitude=field_config['latitude'], area=field_config['area'], + galaxia_exe='galaxia', seed=args.seed) if not args.skip_perform_pop_syn: _check_perform_pop_syn(ebf_file=filename_dict['ebf_filename'], diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 802f9b11..f6f09607 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -34,6 +34,7 @@ import inspect import numpy.lib.recfunctions as rfn import copy +from distutils import spawn from popsycle import ebf from popsycle.filters import transform_ubv_to_ztf from popsycle import utils @@ -172,7 +173,7 @@ def write_galaxia_params(output_root, def _check_run_galaxia(output_root, longitude, latitude, area, - seed): + galaxia_exe, seed): """ Check that the inputs to run_galaxia are valid @@ -194,6 +195,9 @@ def _check_run_galaxia(output_root, longitude, latitude, area, area : float Area of the sky that will be generated, in square degrees + galaxia_exe : str + Name of the galaxia executable + seed : int Seed Galaxia will use to generate objects. If not set, script will generate a seed from the current time. Setting this seed guarantees @@ -215,12 +219,20 @@ def _check_run_galaxia(output_root, longitude, latitude, area, if type(area) != float: raise Exception('area (%s) must be an integer or a float.' % str(area)) + if type(galaxia_exe) != str: + raise Exception('galaxia_exe (%s) must be a string.' % str(galaxia_exe)) + + if spawn.find_executable(galaxia_exe) is None: + raise Exception('galaxia_exe (%s) is not an ' + 'executable currently in $PATH' % str(galaxia_exe)) + if seed is not None: if type(seed) != int: raise Exception('seed (%s) must be None or an integer.' % str(seed)) def run_galaxia(output_root, longitude, latitude, area, + galaxia_exe='galaxia', seed=None): """ Given an object root, sky location and area, creates the parameter @@ -247,6 +259,9 @@ def run_galaxia(output_root, longitude, latitude, area, Optional Parameters ------------------- + galaxia_exe : str + Name of the galaxia executable + seed : int Seed Galaxia will use to generate objects. If not set, script will generate a seed from the current time. Setting this seed guarantees @@ -255,7 +270,7 @@ def run_galaxia(output_root, longitude, latitude, area, # Error handling/complaining if input types are not right. _check_run_galaxia(output_root, longitude, latitude, area, - seed) + galaxia_exe, seed) # Writes out galaxia params to disk write_galaxia_params(output_root=output_root, @@ -265,7 +280,7 @@ def run_galaxia(output_root, longitude, latitude, area, seed=seed) # Execute Galaxia - cmd = 'galaxia -r galaxia_params.%s.txt' % output_root + cmd = '%s -r galaxia_params.%s.txt' % (galaxia_exe, output_root) print('** Executing Galaxia with galaxia_params.%s.txt **' % output_root) _ = utils.execute(cmd) print('** Galaxia complete **') From 2ed41e053c75a9952381a6a8450bf898660c4e0f Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 3 Apr 2020 09:59:08 -0700 Subject: [PATCH 096/125] attempt setting galaxia_exe in slurm scripts --- popsycle/run.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index ed102824..250d2cd7 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -386,7 +386,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, path_run, output_root, longitude, latitude, area, n_cores_calc_events, - walltime, jobname='default', + walltime, jobname='default', galaxia_exe='galaxia', seed=None, overwrite=False, submitFlag=True, skip_galaxia=False, skip_perform_pop_syn=False, skip_calc_events=False, skip_refine_events=False): @@ -616,7 +616,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, slurm_template += '%s\n' % line slurm_template += """ cd {path_run} -srun -N 1 -n 1 {path_python} {run_filepath}/run.py --output-root={output_root} --field-config-filename={field_config_filename} --popsycle-config-filename={popsycle_config_filename} --n-cores-calc-events={n_cores_calc_events} {optional_cmds} +srun -N 1 -n 1 {path_python} {run_filepath}/run.py --output-root={output_root} --field-config-filename={field_config_filename} --popsycle-config-filename={popsycle_config_filename} --n-cores-calc-events={n_cores_calc_events} --galaxia-exe={galaxia_exe} {optional_cmds} date echo "All done!" """ @@ -700,6 +700,9 @@ def run(): 'PopSyCLE pipeline that uses multiprocessing). ' 'Default is --n-cores=1 or serial processing.', default=1) + required.add_argument('--galaxia-exe', type=str, + help='galaxia_exe', + default='galaxia') optional = parser.add_argument_group(title='Optional') optional.add_argument('--seed', type=int, @@ -773,7 +776,7 @@ def run(): longitude=field_config['longitude'], latitude=field_config['latitude'], area=field_config['area'], - galaxia_exe='galaxia', + galaxia_exe=args.galaxia_exe, seed=args.seed) if not args.skip_perform_pop_syn: _check_perform_pop_syn(ebf_file=filename_dict['ebf_filename'], @@ -815,6 +818,7 @@ def run(): longitude=field_config['longitude'], latitude=field_config['latitude'], area=field_config['area'], + galaxia_exe=args.galaxia_exe, seed=args.seed) if not args.skip_perform_pop_syn: From 88a4f0e611103cf641c8a90ac9a9c6c17641210f Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 3 Apr 2020 10:05:15 -0700 Subject: [PATCH 097/125] check galaxia_exe in generate_slurm_script --- popsycle/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index 250d2cd7..42f9cab4 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -503,7 +503,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, longitude=longitude, latitude=latitude, area=area, - galaxia_exe='galaxia', + galaxia_exe=galaxia_exe, seed=seed) if not skip_perform_pop_syn: if popsycle_config['bin_edges_number'] == 'None': From f3693f964a1df7fcc0883899416ec77807cabf8a Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 3 Apr 2020 18:17:13 -0700 Subject: [PATCH 098/125] add in t0 --- popsycle/synthetic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index eee4e954..46825a10 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -282,6 +282,7 @@ def run_galaxia(output_root, longitude, latitude, area, # Execute Galaxia cmd = '%s -r galaxia_params.%s.txt' % (galaxia_exe, output_root) print('** Executing Galaxia with galaxia_params.%s.txt **' % output_root) + t0 = time.time() _ = utils.execute(cmd) t1 = time.time() print('Galaxia complete') From 97b2a7efc3748649e94d8f510ad2ca1971c6b7c3 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Fri, 3 Apr 2020 22:34:02 -0700 Subject: [PATCH 099/125] run which on galaxia_exe instead of galaxia --- popsycle/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 46825a10..13a96c58 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -292,7 +292,7 @@ def run_galaxia(output_root, longitude, latitude, area, # Make log file ########## - stdout, _ = utils.execute('which galaxia') + stdout, _ = utils.execute('which %s' % galaxia_exe) galaxia_path = stdout.replace('\n', '') now = datetime.datetime.now() From c3ca709bac65ab3b7efe2edbc8955219053a716e Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Tue, 7 Apr 2020 14:48:59 -0700 Subject: [PATCH 100/125] switch from galaxia_exe to galaxia_galaxy_model_filename --- README.rst | 5 ++- popsycle/data/popsycle_config.yaml | 1 + popsycle/run.py | 26 ++++++++------ popsycle/synthetic.py | 57 +++++++++++++++++++++--------- 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/README.rst b/README.rst index a7134f83..47b71785 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,10 @@ PopSyCLE Dependencies ------------ -`Galaxia `_ +`galaxia `_ +PopSyCLE requires a custom version of galaxia in order to support +user select galaxy models. Please follow the installation instructions +found at our galaxia GitHub repo: https://github.com/jluastro/galaxia_. `PyPopStar `_ diff --git a/popsycle/data/popsycle_config.yaml b/popsycle/data/popsycle_config.yaml index 5106aba6..92ed5bd0 100644 --- a/popsycle/data/popsycle_config.yaml +++ b/popsycle/data/popsycle_config.yaml @@ -6,6 +6,7 @@ n_obs: 101 theta_frac: 2 blend_rad: 0.75 isochrones_dir: /Users/myself/popsycle_isochrones +galaxia_galaxy_model_filename: /Users/myself/galaxia_galaxy_model_filename bin_edges_number: None BH_kick_speed_mean: 50 NS_kick_speed_mean: 400 diff --git a/popsycle/run.py b/popsycle/run.py index b7fcb4e1..01bffc76 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -188,6 +188,7 @@ def generate_slurm_config_file(path_python='python', account='ulens', def generate_popsycle_config_file(radius_cut=2, obs_time=1000, n_obs=101, theta_frac=2, blend_rad=0.75, isochrones_dir='/Users/myself/popsycle_isochrones', + galaxia_galaxy_model_filename='/Users/myself/galaxia_galaxy_model_filename', bin_edges_number=None, BH_kick_speed_mean=50, NS_kick_speed_mean=400, @@ -218,6 +219,9 @@ def generate_popsycle_config_file(radius_cut=2, obs_time=1000, isochrones_dir : str Directory for PyPopStar isochrones + galaxia_galaxy_model_filename : str + Name of the galaxia galaxy model, as outlined at https://github.com/jluastro/galaxia + bin_edges_number : int Number of edges for the bins bins = bin_edges_number - 1 @@ -257,9 +261,11 @@ def generate_popsycle_config_file(radius_cut=2, obs_time=1000, if bin_edges_number is None: bin_edges_number = 'None' if isochrones_dir == '/Users/myself/popsycle_isochrones': - print('** WARNING **') - print("'isochrones_dir' must be set by the user. " - "The default value is only an example.") + raise Exception("'isochrones_dir' must be set by the user. " + "The default value is only an example.") + if galaxia_galaxy_model_filename == '/Users/myself/galaxia_galaxy_model_filename': + raise Exception("'galaxia_galaxy_model_filename' must be set by the user. " + "The default value is only an example.") config = {'radius_cut': radius_cut, 'obs_time': obs_time, @@ -267,6 +273,7 @@ def generate_popsycle_config_file(radius_cut=2, obs_time=1000, 'theta_frac': theta_frac, 'blend_rad': blend_rad, 'isochrones_dir': isochrones_dir, + 'galaxia_galaxy_model_filename': galaxia_galaxy_model_filename, 'bin_edges_number': bin_edges_number, 'BH_kick_speed_mean': BH_kick_speed_mean, 'NS_kick_speed_mean': NS_kick_speed_mean, @@ -402,7 +409,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, path_run, output_root, longitude, latitude, area, n_cores_calc_events, - walltime, jobname='default', galaxia_exe='galaxia', + walltime, jobname='default', seed=None, overwrite=False, submitFlag=True, returnJobID=False, dependencyJobID=None, skip_galaxia=False, skip_perform_pop_syn=False, @@ -549,7 +556,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, longitude=longitude, latitude=latitude, area=area, - galaxia_exe=galaxia_exe, + galaxia_galaxy_model_filename=popsycle_config['galaxia_galaxy_model_filename'], seed=seed) if not skip_perform_pop_syn: if popsycle_config['bin_edges_number'] == 'None': @@ -665,7 +672,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, slurm_template += '%s\n' % line slurm_template += """ cd {path_run} -srun -N 1 -n 1 {path_python} {run_filepath}/run.py --output-root={output_root} --field-config-filename={field_config_filename} --popsycle-config-filename={popsycle_config_filename} --n-cores-calc-events={n_cores_calc_events} --galaxia-exe={galaxia_exe} {optional_cmds} +srun -N 1 -n 1 {path_python} {run_filepath}/run.py --output-root={output_root} --field-config-filename={field_config_filename} --popsycle-config-filename={popsycle_config_filename} --n-cores-calc-events={n_cores_calc_events} {optional_cmds} date echo "All done!" """ @@ -768,9 +775,6 @@ def run(): 'PopSyCLE pipeline that uses multiprocessing). ' 'Default is --n-cores=1 or serial processing.', default=1) - required.add_argument('--galaxia-exe', type=str, - help='galaxia_exe', - default='galaxia') optional = parser.add_argument_group(title='Optional') optional.add_argument('--seed', type=int, @@ -844,7 +848,7 @@ def run(): longitude=field_config['longitude'], latitude=field_config['latitude'], area=field_config['area'], - galaxia_exe=args.galaxia_exe, + galaxia_galaxy_model_filename=popsycle_config['galaxia_galaxy_model_filename'], seed=args.seed) if not args.skip_perform_pop_syn: _check_perform_pop_syn(ebf_file=filename_dict['ebf_filename'], @@ -886,7 +890,7 @@ def run(): longitude=field_config['longitude'], latitude=field_config['latitude'], area=field_config['area'], - galaxia_exe=args.galaxia_exe, + galaxia_galaxy_model_filename=popsycle_config['galaxia_galaxy_model_filename'], seed=args.seed) if not args.skip_perform_pop_syn: diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 13a96c58..f46686f6 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -173,7 +173,7 @@ def write_galaxia_params(output_root, def _check_run_galaxia(output_root, longitude, latitude, area, - galaxia_exe, seed): + galaxia_galaxy_model_filename, seed): """ Check that the inputs to run_galaxia are valid @@ -195,8 +195,8 @@ def _check_run_galaxia(output_root, longitude, latitude, area, area : float Area of the sky that will be generated, in square degrees - galaxia_exe : str - Name of the galaxia executable + galaxia_galaxy_model_filename : str + Name of the galaxia galaxy model, as outlined at https://github.com/jluastro/galaxia seed : int Seed Galaxia will use to generate objects. If not set, script will @@ -219,12 +219,33 @@ def _check_run_galaxia(output_root, longitude, latitude, area, if type(area) != float: raise Exception('area (%s) must be an integer or a float.' % str(area)) - if type(galaxia_exe) != str: - raise Exception('galaxia_exe (%s) must be a string.' % str(galaxia_exe)) + if spawn.find_executable('galaxia') is None: + raise Exception('galaxia is not an executable currently in $PATH') - if spawn.find_executable(galaxia_exe) is None: - raise Exception('galaxia_exe (%s) is not an ' - 'executable currently in $PATH' % str(galaxia_exe)) + stdout, _ = utils.execute('galaxia --version') + galaxia_version = stdout.replace('\n', '').split()[1] + if galaxia_version != '0.7.2.1': + raise Exception('galaxia must be version 0.7.2.1 installed from https://github.com/jluastro/galaxia') + + if type(galaxia_galaxy_model_filename) != str: + raise Exception('galaxia_galaxy_model_filename (%s) must be a string.' % str(galaxia_galaxy_model_filename)) + + if not os.path.exists(galaxia_galaxy_model_filename): + raise Exception('galaxia_galaxy_model_filename (%s) does not exist' % galaxia_galaxy_model_filename) + + GalaxiaData = None + for line in open(galaxia_galaxy_model_filename, 'r'): + if 'GalaxiaData' in line: + GalaxiaData = line.replace('\n','').split()[1] + break + + if GalaxiaData is None: + raise Exception('GalaxiaData missing from ' + 'galaxia_galaxy_model_filename (%s)' % galaxia_galaxy_model_filename) + + if not os.path.exists(GalaxiaData): + raise Exception('GalaxiaData (%s) in galaxia_galaxy_model_filename ' + '(%s) does not exist' % (GalaxiaData, galaxia_galaxy_model_filename)) if seed is not None: if type(seed) != int: @@ -232,7 +253,7 @@ def _check_run_galaxia(output_root, longitude, latitude, area, def run_galaxia(output_root, longitude, latitude, area, - galaxia_exe='galaxia', + galaxia_galaxy_model_filename, seed=None): """ Given an object root, sky location and area, creates the parameter @@ -257,11 +278,11 @@ def run_galaxia(output_root, longitude, latitude, area, area : float Area of the sky that will be generated, in square degrees + galaxia_galaxy_model_filename : str + Name of the galaxia galaxy model, as outlined at https://github.com/jluastro/galaxia + Optional Parameters ------------------- - galaxia_exe : str - Name of the galaxia executable - seed : int Seed Galaxia will use to generate objects. If not set, script will generate a seed from the current time. Setting this seed guarantees @@ -270,7 +291,7 @@ def run_galaxia(output_root, longitude, latitude, area, # Error handling/complaining if input types are not right. _check_run_galaxia(output_root, longitude, latitude, area, - galaxia_exe, seed) + galaxia_galaxy_model_filename, seed) # Writes out galaxia params to disk write_galaxia_params(output_root=output_root, @@ -280,8 +301,9 @@ def run_galaxia(output_root, longitude, latitude, area, seed=seed) # Execute Galaxia - cmd = '%s -r galaxia_params.%s.txt' % (galaxia_exe, output_root) - print('** Executing Galaxia with galaxia_params.%s.txt **' % output_root) + cmd = 'galaxia -r galaxia_params.%s.txt %s' % (output_root, galaxia_galaxy_model_filename) + print('** Executing Galaxia with galaxia_params.%s.txt ' + 'and %s **' % (output_root, galaxia_galaxy_model_filename)) t0 = time.time() _ = utils.execute(cmd) t1 = time.time() @@ -292,7 +314,7 @@ def run_galaxia(output_root, longitude, latitude, area, # Make log file ########## - stdout, _ = utils.execute('which %s' % galaxia_exe) + stdout, _ = utils.execute('which galaxia') galaxia_path = stdout.replace('\n', '') now = datetime.datetime.now() @@ -306,6 +328,7 @@ def run_galaxia(output_root, longitude, latitude, area, line1 = 'longitude , ' + str(longitude) + '\n' line2 = 'latitude , ' + str(latitude) + '\n' line3 = 'area , ' + str(area) + '\n' + line3b = 'galaxia_galaxy_model_filename , ' + galaxia_galaxy_model_filename + '\n' line4 = 'seed , ' + str(seed) + '\n' line8 = 'VERSION INFORMATION' + '\n' @@ -320,7 +343,7 @@ def run_galaxia(output_root, longitude, latitude, area, line18 = output_root + '.ebf : ebf file' + '\n' with open(output_root + '_galaxia.log', 'w') as out: - out.writelines([line0, dash_line, line1, line2, line3, line4, + out.writelines([line0, dash_line, line1, line2, line3, line3b, line4, empty_line, line8, dash_line, line9, line10, line11, empty_line, line12, dash_line, line13, empty_line, line17, dash_line, line18]) From 9a2baa91cdcddb04d5f05298a003af38664b130b Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Tue, 7 Apr 2020 15:08:42 -0700 Subject: [PATCH 101/125] remove redundant cd --- popsycle/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index 01bffc76..309db497 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -671,7 +671,6 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, for line in slurm_config['additional_lines']: slurm_template += '%s\n' % line slurm_template += """ -cd {path_run} srun -N 1 -n 1 {path_python} {run_filepath}/run.py --output-root={output_root} --field-config-filename={field_config_filename} --popsycle-config-filename={popsycle_config_filename} --n-cores-calc-events={n_cores_calc_events} {optional_cmds} date echo "All done!" From 8cf3aacf964e51376ef6bbc23e9cfa3b5cc734a6 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Tue, 7 Apr 2020 16:42:51 -0700 Subject: [PATCH 102/125] fix typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 47b71785..105fe4b1 100644 --- a/README.rst +++ b/README.rst @@ -12,7 +12,7 @@ Dependencies ------------ `galaxia `_ PopSyCLE requires a custom version of galaxia in order to support -user select galaxy models. Please follow the installation instructions +user selected galaxy models. Please follow the installation instructions found at our galaxia GitHub repo: https://github.com/jluastro/galaxia_. `PyPopStar `_ From 264a5cec21ee7ec7e250e477df1382b670b72ead Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Tue, 7 Apr 2020 16:44:43 -0700 Subject: [PATCH 103/125] add comments --- popsycle/synthetic.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index f46686f6..6f519007 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -219,20 +219,24 @@ def _check_run_galaxia(output_root, longitude, latitude, area, if type(area) != float: raise Exception('area (%s) must be an integer or a float.' % str(area)) + # Check that galaxia is in the executable PATH if spawn.find_executable('galaxia') is None: raise Exception('galaxia is not an executable currently in $PATH') + # Confrim that galaxia is the PopSyCLE compliant version stdout, _ = utils.execute('galaxia --version') galaxia_version = stdout.replace('\n', '').split()[1] if galaxia_version != '0.7.2.1': raise Exception('galaxia must be version 0.7.2.1 installed from https://github.com/jluastro/galaxia') + # Check the galaxia_galaxy_model_filename for correct type and existence if type(galaxia_galaxy_model_filename) != str: raise Exception('galaxia_galaxy_model_filename (%s) must be a string.' % str(galaxia_galaxy_model_filename)) if not os.path.exists(galaxia_galaxy_model_filename): raise Exception('galaxia_galaxy_model_filename (%s) does not exist' % galaxia_galaxy_model_filename) + # Check that GalaxiaData is a valid galaxia folder GalaxiaData = None for line in open(galaxia_galaxy_model_filename, 'r'): if 'GalaxiaData' in line: From d09a44745483d7a4a96e37f669acbe0112d85c26 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Thu, 9 Apr 2020 11:08:08 -0700 Subject: [PATCH 104/125] propagate diagnostic_plots from pbh_config.yaml to run.py --- popsycle/run.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index ebe08094..95d469c4 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -289,6 +289,7 @@ def generate_popsycle_config_file(radius_cut=2, obs_time=1000, def generate_pbh_config_file(fdm=1, pbh_mass=40, r_max=16.6, r_s=18.6, gamma=1, v_esc=550, rho_0=0.0093, n_lin=1000, + diagnostic_plots=False, config_filename='pbh_config.yaml'): """ Save PBH configuration parameters into a yaml file @@ -324,6 +325,15 @@ def generate_pbh_config_file(fdm=1, pbh_mass=40, The number of times you want the density determined along the line of sight when calculating PBH positions Defaults to 1000. Will need to make large if you are closer to the galactic center. + diagnostic_plots: bool + Generate diganostic plots when running add_pbh. Default False. + + Optional Parameters + ------------------- + config_filename : str + Name of the configuration file + Default: pbh_config.yaml + Output ------ None @@ -336,6 +346,7 @@ def generate_pbh_config_file(fdm=1, pbh_mass=40, 'gamma': gamma, 'v_esc': v_esc, 'rho_0': rho_0, + 'diagnostic_plots': str(diagnostic_plots), 'n_lin': n_lin} generate_config_file(config_filename, config) @@ -642,6 +653,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, # Load pbh config, if provided. if pbh_config_filename is not None: pbh_config = load_config_file(pbh_config_filename) + pbh_config['diagnostic_plots'] = bool(pbh_config['diagnostic_plots']) # Check pipeline stages for valid inputs _check_slurm_config(slurm_config, walltime) @@ -673,7 +685,7 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], - diagnostic_plots=True, + diagnostic_plots=pbh_config['diagnostic_plots'], new_output_root=None, seed=seed) if not skip_calc_events: From f25585ce19c78f354819dbb4709c833a94f0dc8f Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Thu, 9 Apr 2020 11:10:29 -0700 Subject: [PATCH 105/125] enforce abs.path for isochrones_dir and galaxia_galaxy_model_filename --- popsycle/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index 309db497..8c61ada9 100644 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -272,8 +272,8 @@ def generate_popsycle_config_file(radius_cut=2, obs_time=1000, 'n_obs': n_obs, 'theta_frac': theta_frac, 'blend_rad': blend_rad, - 'isochrones_dir': isochrones_dir, - 'galaxia_galaxy_model_filename': galaxia_galaxy_model_filename, + 'isochrones_dir': os.path.abspath(isochrones_dir), + 'galaxia_galaxy_model_filename': os.path.abspath(galaxia_galaxy_model_filename), 'bin_edges_number': bin_edges_number, 'BH_kick_speed_mean': BH_kick_speed_mean, 'NS_kick_speed_mean': NS_kick_speed_mean, From bf875a02a88b0dc87e5177d9c84abc3d6f62d90e Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Thu, 9 Apr 2020 11:14:41 -0700 Subject: [PATCH 106/125] move add_pbh runtime print to last statement --- popsycle/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 6d382e70..2da3fc76 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -2031,7 +2031,6 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, os.rename(output_hdf5_file, hdf5_file) t1 = time.time() - print('add_pbh runtime: {0:f} s'.format(t1 - t0)) ########## # Make log file @@ -2090,6 +2089,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, else: print('** MISSING PBHs!! **') + print('add_pbh runtime: {0:f} s'.format(t1 - t0)) return From 6a8e4623f6f271b70acb5725c875bd65e270d23c Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Thu, 9 Apr 2020 11:56:13 -0700 Subject: [PATCH 107/125] propagate diagnostic_plots to add_pbh REALLY --- popsycle/run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index b2fe16f3..a1d20c53 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -998,6 +998,7 @@ def run(): seed=args.seed) if args.pbh_config_filename is not None: pbh_config = load_config_file(args.pbh_config_filename) + pbh_config['diagnostic_plots'] = bool(pbh_config['diagnostic_plots']) _check_add_pbh(hdf5_file=filename_dict['hdf5_filename'], ebf_file=filename_dict['ebf_filename'], fdm=pbh_config['fdm'], @@ -1008,7 +1009,7 @@ def run(): v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], - diagnostic_plots=True, + diagnostic_plots=pbh_config['diagnostic_plots'], new_output_root=None, seed=args.seed) if not args.skip_calc_events: @@ -1088,6 +1089,7 @@ def run(): v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], n_lin=pbh_config['n_lin'], + diagnostic_plots=pbh_config['diagnostic_plots'], seed=args.seed) if not args.skip_calc_events: From 34892eeede9535f2b8db62df225081dcbbebe5bb Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Thu, 9 Apr 2020 14:05:06 -0700 Subject: [PATCH 108/125] correct closing figures in add_pbh_plots --- popsycle/add_pbh_plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/add_pbh_plots.py b/popsycle/add_pbh_plots.py index 054310c1..8185e7e2 100644 --- a/popsycle/add_pbh_plots.py +++ b/popsycle/add_pbh_plots.py @@ -183,5 +183,5 @@ def print_plots(output_root, galactic_lin_distance, galactic_lin_b, galactic_lin pp = PdfPages(output_root+'_pbh_diagnostic_plots.pdf') for fig in fig_arr: pp.savefig(fig) - fig.close() + plt.close(fig) pp.close() From 6447c0c0846ff4792f25777d28f19be20140a78a Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Thu, 9 Apr 2020 14:10:46 -0700 Subject: [PATCH 109/125] add diagnostic_plots to log --- popsycle/synthetic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 2da3fc76..ceba0eed 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -2053,6 +2053,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, line8 = 'v_esc , ' + str(v_esc) + ' , (km/s)' + '\n' line9 = 'rho_0 , ' + str(rho_0) + ' , (Msun / pc^3)' + '\n' line10 = 'n_lin , ' + str(n_lin) + '\n' + line10b = 'diagnostic_plots , ' + str(diagnostic_plots) + '\n' line11 = 'new_output_root , ' + str(new_output_root) + '\n' line12 = 'seed , ' + str(seed) + '\n' @@ -2071,7 +2072,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, with open(output_root + '_add_pbh.log', 'w') as out: out.writelines([line0, dash_line, line1, line2, line3, line4, line5, - line6, line7, line8, line9, line10, line11, line12, + line6, line7, line8, line9, line10, line10b, line11, line12, empty_line, line13, dash_line, line14, line15, empty_line, line16, dash_line, line17, line18, line19, line20, empty_line, line21, dash_line, line22]) From c6c7a6b117bb1f9df53506a3eb64ef9f6e78eadf Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 4 May 2020 21:47:10 -0700 Subject: [PATCH 110/125] still generate log from calc_events if no events --- popsycle/synthetic.py | 47 ++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index ceba0eed..56f669dd 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -2308,31 +2308,30 @@ def calc_events(hdf5_file, output_root2, if results[ii][1] is not None: results_bl.append(results[ii][1]) - if len(results_ev) == 0: - print('No events!') - return - else: + if len(results_ev) > 0: events_tmp = np.concatenate(results_ev, axis=0) if len(results_bl) == 0: blends_tmp = np.array([]) else: blends_tmp = np.concatenate(results_bl, axis=0) - # Convert the events numpy recarray into an - # Astropy Table for easier consumption. - events_tmp = unique_events(events_tmp) - events_final = Table(events_tmp) - N_events = len(events_final) - print('Candidate events detected: ', N_events) + # Convert the events numpy recarray into an + # Astropy Table for easier consumption. + events_tmp = unique_events(events_tmp) + events_final = Table(events_tmp) + N_events = len(events_final) + print('Candidate events detected: ', N_events) - if len(results_bl) != 0: - blends_tmp = unique_blends(blends_tmp) - blends_final = Table(blends_tmp) - - # Save out file - events_final.write(output_root2 + '_events.fits', overwrite=overwrite) - blends_final.write(output_root2 + '_blends.fits', overwrite=overwrite) + if len(results_bl) != 0: + blends_tmp = unique_blends(blends_tmp) + blends_final = Table(blends_tmp) + # Save out file + events_final.write(output_root2 + '_events.fits', overwrite=overwrite) + blends_final.write(output_root2 + '_blends.fits', overwrite=overwrite) + else: + N_events = 0 + print('No events!') t1 = time.time() ########## @@ -2342,7 +2341,7 @@ def calc_events(hdf5_file, output_root2, radius_cut = radius_cut / 1000.0 # back to arcsec popsycle_path = os.path.dirname(inspect.getfile(perform_pop_syn)) popsycle_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=popsycle_path).decode('ascii').strip() + cwd=popsycle_path).decode('ascii').strip() dash_line = '-----------------------------' + '\n' empty_line = '\n' line0 = 'FUNCTION INPUT PARAMETERS' + '\n' @@ -2354,6 +2353,7 @@ def calc_events(hdf5_file, output_root2, line6 = 'theta_frac , ' + str(theta_frac) + ' , (thetaE)' + '\n' line7 = 'blend_rad , ' + str(blend_rad) + ' , (arcsec)' + '\n' line8 = 'n_proc , ' + str(n_proc) + '\n' + line9 = 'VERSION INFORMATION' + '\n' line10 = str(now) + ' : creation date' + '\n' line11 = popsycle_hash + ' : PopSyCLE commit' + '\n' @@ -2362,9 +2362,14 @@ def calc_events(hdf5_file, output_root2, line13 = str(t1 - t0) + ' : total runtime (s)' + '\n' line14 = str(N_events) + ' : total number of events' + '\n' - line15 = 'FILES CREATED' + '\n' - line16 = output_root2 + '_events.fits : events file' + '\n' - line17 = output_root2 + '_blends.fits : blends file' + '\n' + if N_events > 0: + line15 = 'FILES CREATED' + '\n' + line16 = output_root2 + '_events.fits : events file' + '\n' + line17 = output_root2 + '_blends.fits : blends file' + '\n' + else: + line15 = 'NO FILES CREATED' + '\n' + line16 = '\n' + line17 = '\n' with open(output_root2 + '_calc_events.log', 'w') as out: out.writelines([line0, dash_line, line1, line2, line3, From 74616fad3a54cf9aa180821af4178b5216ccbff1 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 4 May 2020 21:48:20 -0700 Subject: [PATCH 111/125] still generate log from refine_events if no events --- popsycle/synthetic.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 56f669dd..eeb9ce31 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -3197,30 +3197,30 @@ def refine_events(input_root, filter_name, photometric_system, red_law, event_tab['u0'] = u0 if len(event_tab) == 0: print('No events!') - return - - ########## - # Calculate apparent magnitudes - ########## + output_file = 'NO FILE CREATED' + else: + ########## + # Calculate apparent magnitudes + ########## - # Einstein crossing time - # THIS HAS TO GO BEFORE _CALC_OBSERVABLES - t_E = event_tab['theta_E'] / event_tab['mu_rel'] # yr - t_E *= 365.25 # days - event_tab['t_E'] = t_E # days + # Einstein crossing time + # THIS HAS TO GO BEFORE _CALC_OBSERVABLES + t_E = event_tab['theta_E'] / event_tab['mu_rel'] # yr + t_E *= 365.25 # days + event_tab['t_E'] = t_E # days - # Add stuff to event_tab... shouldn't have any direct outputs - _calc_observables(filter_name, red_law, event_tab, blend_tab, photometric_system) + # Add stuff to event_tab... shouldn't have any direct outputs + _calc_observables(filter_name, red_law, event_tab, blend_tab, photometric_system) - # Relative parallax - pi_rel = event_tab['rad_L'] ** -1 - event_tab['rad_S'] ** -1 - event_tab['pi_rel'] = pi_rel # mas + # Relative parallax + pi_rel = event_tab['rad_L'] ** -1 - event_tab['rad_S'] ** -1 + event_tab['pi_rel'] = pi_rel # mas - # Microlensing parallax - pi_E = pi_rel / event_tab['theta_E'] - event_tab['pi_E'] = pi_E # dim'less + # Microlensing parallax + pi_E = pi_rel / event_tab['theta_E'] + event_tab['pi_E'] = pi_E # dim'less - event_tab.write(output_file, overwrite=overwrite) + event_tab.write(output_file, overwrite=overwrite) t_1 = time.time() From 9456d4c8a92ace79883da2578ddd5f9a8078559a Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 4 May 2020 21:49:55 -0700 Subject: [PATCH 112/125] fix typo --- popsycle/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index eeb9ce31..abb5cdc8 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1595,7 +1595,7 @@ def _check_add_pbh(hdf5_file, ebf_file, raise Exception('rho_0 (%s) must be an integer or a float.' % str(rho_0)) if type(n_lin) != int: - raise Exception('n_lin (%s) must be an integerr.' % str(n_lin)) + raise Exception('n_lin (%s) must be an integer.' % str(n_lin)) if type(diagnostic_plots) != bool: raise Exception('diagnostic_plots (%s) must be a boolean.' % str(diagnostic_plots)) From aa329f0fd7c59f79a23b0c9cdaf95a98db573a34 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Mon, 4 May 2020 21:50:14 -0700 Subject: [PATCH 113/125] fix typo --- popsycle/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index abb5cdc8..ad8fb17a 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1761,7 +1761,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, if np.logical_and(np.logical_and(np.abs(l_radian) < 0.5 * np.pi / 180, np.abs(b_radian) < 0.5 * np.pi / 180), n_lin < 100000): - print('Warning: for fields very near the center of the Milky Way it is reocmmended that the number of elements used to estimate the density be n_lin>100000') + print('Warning: for fields very near the center of the Milky Way it is recommended that the number of elements used to estimate the density be n_lin>100000') r_h_linspace = np.linspace(0, r_max, num=n_lin) # Represent the line of sight line in galactic coordinates From 2303a77281b9abcfdb754ab7e7e23feb061d0e57 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Mon, 4 May 2020 22:39:20 -0700 Subject: [PATCH 114/125] removed hardcoded n_lin --- popsycle/synthetic.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index ad8fb17a..721c6f69 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1757,7 +1757,6 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # Generate an array of heliocentric radii # These radii will just be used to numerically integrate the density - n_lin = 1000 if np.logical_and(np.logical_and(np.abs(l_radian) < 0.5 * np.pi / 180, np.abs(b_radian) < 0.5 * np.pi / 180), n_lin < 100000): @@ -1888,7 +1887,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # Interpolating v values from the above data, given the PBH r values. pbh_vrms = np.interp(pbh_r_galacto.value, vel_data['r'], vel_data['v']) - v_vals = np.arange(0, v_esc) # Goes from v to v_esc + v_vals = np.arange(0, v_esc) a = (1 / 2) * pbh_vrms * ((np.pi / 2) ** (1 / 2)) # Calculating the v_rms velocities for the PBHs by randomly sampling from the CDF. From a1422d18cb0c0bc5920778f40faf75d305feb155 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Tue, 5 May 2020 16:33:36 -0700 Subject: [PATCH 115/125] Hard coded n_lin --- popsycle/data/pbh_config.yaml | 3 +-- popsycle/run.py | 12 ++---------- popsycle/synthetic.py | 24 ++++++------------------ 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/popsycle/data/pbh_config.yaml b/popsycle/data/pbh_config.yaml index b4a8fb09..cb551090 100644 --- a/popsycle/data/pbh_config.yaml +++ b/popsycle/data/pbh_config.yaml @@ -6,5 +6,4 @@ r_max: 16.6 r_s: 18.6 gamma: 1 v_esc: 550 -rho_0 : 0.0093 -n_lin : 1000 +rho_0 : 0.0093 diff --git a/popsycle/run.py b/popsycle/run.py index a1d20c53..85c9de50 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -288,7 +288,7 @@ def generate_popsycle_config_file(radius_cut=2, obs_time=1000, def generate_pbh_config_file(fdm=1, pbh_mass=40, r_max=16.6, r_s=18.6, gamma=1, - v_esc=550, rho_0=0.0093, n_lin=1000, + v_esc=550, rho_0=0.0093, diagnostic_plots=False, config_filename='pbh_config.yaml'): """ @@ -321,10 +321,6 @@ def generate_pbh_config_file(fdm=1, pbh_mass=40, rho_0: float The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). - n_lin: int - The number of times you want the density determined along the line of sight when calculating PBH positions - Defaults to 1000. Will need to make large if you are closer to the galactic center. - diagnostic_plots: bool Generate diganostic plots when running add_pbh. Default False. @@ -346,8 +342,7 @@ def generate_pbh_config_file(fdm=1, pbh_mass=40, 'gamma': gamma, 'v_esc': v_esc, 'rho_0': rho_0, - 'diagnostic_plots': str(diagnostic_plots), - 'n_lin': n_lin} + 'diagnostic_plots': str(diagnostic_plots)} generate_config_file(config_filename, config) @@ -684,7 +679,6 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, gamma=pbh_config['gamma'], v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], - n_lin=pbh_config['n_lin'], diagnostic_plots=pbh_config['diagnostic_plots'], new_output_root=None, seed=seed) @@ -1008,7 +1002,6 @@ def run(): gamma=pbh_config['gamma'], v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], - n_lin=pbh_config['n_lin'], diagnostic_plots=pbh_config['diagnostic_plots'], new_output_root=None, seed=args.seed) @@ -1088,7 +1081,6 @@ def run(): gamma=pbh_config['gamma'], v_esc=pbh_config['v_esc'], rho_0=pbh_config['rho_0'], - n_lin=pbh_config['n_lin'], diagnostic_plots=pbh_config['diagnostic_plots'], seed=args.seed) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 721c6f69..5bdc836d 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1498,7 +1498,7 @@ def rho_dmhalo(r, rho_0=.0093, r_s=18.6, gamma=1): def _check_add_pbh(hdf5_file, ebf_file, fdm, pbh_mass, r_max, r_s, gamma, v_esc, rho_0, - n_lin, diagnostic_plots, new_output_root, seed): + diagnostic_plots, new_output_root, seed): """ Checks that the inputs of add_pbj are valid @@ -1532,10 +1532,6 @@ def _check_add_pbh(hdf5_file, ebf_file, The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2017. - n_lin: int - The number of times you want the density determined along the line of sight when calculating PBH positions - Defaults to 1000. Will need to make large if you are closer to the galactic center. - gamma: float The inner slope of the MW dark matter halo as described in LaCroix 2018. Gamma goes into the determination of the velocities and each value returns a slightly different distribution. @@ -1594,9 +1590,6 @@ def _check_add_pbh(hdf5_file, ebf_file, if type(rho_0) != float: raise Exception('rho_0 (%s) must be an integer or a float.' % str(rho_0)) - if type(n_lin) != int: - raise Exception('n_lin (%s) must be an integer.' % str(n_lin)) - if type(diagnostic_plots) != bool: raise Exception('diagnostic_plots (%s) must be a boolean.' % str(diagnostic_plots)) @@ -1611,7 +1604,7 @@ def _check_add_pbh(hdf5_file, ebf_file, def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, r_max=16.6, r_s=18.6, gamma=1, v_esc=550, - rho_0=0.0093, n_lin=1000, diagnostic_plots=False, + rho_0=0.0093, diagnostic_plots=False, new_output_root=None, seed=None): """ Given some hdf5 file from perform_pop_syn output, creates PBH positions, velocities, etc, @@ -1657,10 +1650,6 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, The initial density that will be used in the NFW profile equations (in units of Msun/pc^3). Defaults to .0093 [Msun / pc^3]. The median value given in McMillan 2017. - n_lin: int - The number of times you want the density determined along the line of sight when calculating PBH positions - Defaults to 1000. Will need to make large if you are closer to the galactic center. - Optional Parameters ------------------- diagnostic_plots: bool @@ -1691,10 +1680,8 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, _check_add_pbh(hdf5_file=hdf5_file, ebf_file=ebf_file, fdm=fdm, pbh_mass=pbh_mass, r_max=r_max, r_s=r_s, gamma=gamma, v_esc=v_esc, - rho_0=rho_0, n_lin=n_lin, - diagnostic_plots=diagnostic_plots, - new_output_root=new_output_root, - seed=seed) + rho_0=rho_0, diagnostic_plots=diagnostic_plots, + new_output_root=new_output_root, seed=seed) if new_output_root is None: output_root = hdf5_file.replace('.h5', '') @@ -1757,10 +1744,11 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # Generate an array of heliocentric radii # These radii will just be used to numerically integrate the density + n_lin = 100000 + if np.logical_and(np.logical_and(np.abs(l_radian) < 0.5 * np.pi / 180, np.abs(b_radian) < 0.5 * np.pi / 180), n_lin < 100000): - print('Warning: for fields very near the center of the Milky Way it is recommended that the number of elements used to estimate the density be n_lin>100000') r_h_linspace = np.linspace(0, r_max, num=n_lin) # Represent the line of sight line in galactic coordinates From 43e8dd2ebe8aeb6a4d9ab6bd5bd9cbda9aa318e9 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Wed, 6 May 2020 17:15:34 -0700 Subject: [PATCH 116/125] fixed indentation --- popsycle/synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 5bdc836d..7de57a84 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1749,7 +1749,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, if np.logical_and(np.logical_and(np.abs(l_radian) < 0.5 * np.pi / 180, np.abs(b_radian) < 0.5 * np.pi / 180), n_lin < 100000): - r_h_linspace = np.linspace(0, r_max, num=n_lin) + r_h_linspace = np.linspace(0, r_max, num=n_lin) # Represent the line of sight line in galactic coordinates galactic_lin = coord.Galactic(l=l_radian * units.rad, From 21e33d91212ca36623b8f8ea22a9a437a6a1c435 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Wed, 6 May 2020 19:39:17 -0700 Subject: [PATCH 117/125] removed un-needed if statement --- popsycle/synthetic.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 7de57a84..2f4a545a 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1746,10 +1746,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # These radii will just be used to numerically integrate the density n_lin = 100000 - if np.logical_and(np.logical_and(np.abs(l_radian) < 0.5 * np.pi / 180, - np.abs(b_radian) < 0.5 * np.pi / 180), - n_lin < 100000): - r_h_linspace = np.linspace(0, r_max, num=n_lin) + r_h_linspace = np.linspace(0, r_max, num=n_lin) # Represent the line of sight line in galactic coordinates galactic_lin = coord.Galactic(l=l_radian * units.rad, From 6c4b46bd75018558a8ce70d8064d516249d8a380 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Wed, 6 May 2020 20:58:29 -0700 Subject: [PATCH 118/125] removed n_lin from output log --- popsycle/synthetic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index 2f4a545a..bc2a67a0 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -2036,8 +2036,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, line7 = 'gamma , ' + str(gamma) + '\n' line8 = 'v_esc , ' + str(v_esc) + ' , (km/s)' + '\n' line9 = 'rho_0 , ' + str(rho_0) + ' , (Msun / pc^3)' + '\n' - line10 = 'n_lin , ' + str(n_lin) + '\n' - line10b = 'diagnostic_plots , ' + str(diagnostic_plots) + '\n' + line10 = 'diagnostic_plots , ' + str(diagnostic_plots) + '\n' line11 = 'new_output_root , ' + str(new_output_root) + '\n' line12 = 'seed , ' + str(seed) + '\n' @@ -2056,7 +2055,7 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, with open(output_root + '_add_pbh.log', 'w') as out: out.writelines([line0, dash_line, line1, line2, line3, line4, line5, - line6, line7, line8, line9, line10, line10b, line11, line12, + line6, line7, line8, line9, line10, line11, line12, empty_line, line13, dash_line, line14, line15, empty_line, line16, dash_line, line17, line18, line19, line20, empty_line, line21, dash_line, line22]) From a656cd469a3692f34d4f7c4547c7e07a3a4a245f Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Thu, 14 May 2020 09:14:48 -0700 Subject: [PATCH 119/125] force use of Agg backend --- popsycle/add_pbh_plots.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/popsycle/add_pbh_plots.py b/popsycle/add_pbh_plots.py index 8185e7e2..b1bfea79 100644 --- a/popsycle/add_pbh_plots.py +++ b/popsycle/add_pbh_plots.py @@ -1,3 +1,5 @@ +import matplotlib +matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np from matplotlib.backends.backend_pdf import PdfPages From 8be48982a3046b8ae06d6ee35d57039c0c6cbbef Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Thu, 14 May 2020 09:17:40 -0700 Subject: [PATCH 120/125] force use of PDF backend --- popsycle/add_pbh_plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popsycle/add_pbh_plots.py b/popsycle/add_pbh_plots.py index b1bfea79..67b68f30 100644 --- a/popsycle/add_pbh_plots.py +++ b/popsycle/add_pbh_plots.py @@ -1,5 +1,5 @@ import matplotlib -matplotlib.use("Agg") +matplotlib.use('PDF') import matplotlib.pyplot as plt import numpy as np from matplotlib.backends.backend_pdf import PdfPages From dfb0f1895ea31949d54835032f24a20d7d15d260 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 20 May 2020 12:42:57 -0700 Subject: [PATCH 121/125] test srun_options --- popsycle/data/slurm_config.yaml | 2 ++ popsycle/run.py | 16 +++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/popsycle/data/slurm_config.yaml b/popsycle/data/slurm_config.yaml index 80b7939c..ae66bada 100644 --- a/popsycle/data/slurm_config.yaml +++ b/popsycle/data/slurm_config.yaml @@ -12,6 +12,8 @@ resource: haswell # Set to true if slurm scheduler requires a CONSTRAINT specification # Set to false if slurm scheduler does not include_constraint: true +# Options for slurm's srun command +srun_options: # Additional lines to be run before executing run.py additional_lines: - module load cray-hdf5/1.10.5.2 diff --git a/popsycle/run.py b/popsycle/run.py index 85c9de50..95f2c5c0 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -162,8 +162,8 @@ def generate_slurm_config_file(path_python='python', account='ulens', Maximum number of hours for single job on the compute resource Format: hh:mm:ss - additional_lines : list of strings - Additional lines to be run before executing run.py + additional_lines : None, list of strings + None, or additional lines to be run before executing run.py Optional Parameters ------------------- @@ -641,6 +641,9 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, # Load slurm config slurm_config = load_config_file(slurm_config_filename) slurm_config['include_constraint'] = bool(slurm_config['include_constraint']) # Enforce boolean for 'include_constraint' + if slurm_config['srun_options'] is None: + slurm_config['srun_options'] = '' + if slurm_config['srun_options'] is None: # Load popsycle config popsycle_config = load_config_file(popsycle_config_filename) if popsycle_config['bin_edges_number'] == 'None': # Enforce None for 'bin_edges_number' @@ -726,6 +729,8 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, account = slurm_config['account'] # Queue queue = slurm_config['queue'] + # Options for slurm's srun command + srun_options = slurm_config['srun_options'] # Name of the resource that will be ussed for the run resource = slurm_config['resource'] # Maximum number of cores per node @@ -781,10 +786,11 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, cd {path_run} """ - for line in slurm_config['additional_lines']: - slurm_template += '%s\n' % line + if slurm_config['additional_lines'] is not None: + for line in slurm_config['additional_lines']: + slurm_template += '%s\n' % line slurm_template += """ -srun -N 1 -n 1 {path_python} {run_filepath}/run.py --output-root={output_root} --field-config-filename={field_config_filename} --popsycle-config-filename={popsycle_config_filename} --n-cores-calc-events={n_cores_calc_events} {optional_cmds} +srun -N 1 -n 1 {srun_options} {path_python} {run_filepath}/run.py --output-root={output_root} --field-config-filename={field_config_filename} --popsycle-config-filename={popsycle_config_filename} --n-cores-calc-events={n_cores_calc_events} {optional_cmds} date echo "All done!" """ From 3441362af8f0c6b6642c8af990c36b3b31363b04 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 20 May 2020 12:49:33 -0700 Subject: [PATCH 122/125] fix typo --- popsycle/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/popsycle/run.py b/popsycle/run.py index 95f2c5c0..8a499773 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -643,7 +643,6 @@ def generate_slurm_script(slurm_config_filename, popsycle_config_filename, slurm_config['include_constraint'] = bool(slurm_config['include_constraint']) # Enforce boolean for 'include_constraint' if slurm_config['srun_options'] is None: slurm_config['srun_options'] = '' - if slurm_config['srun_options'] is None: # Load popsycle config popsycle_config = load_config_file(popsycle_config_filename) if popsycle_config['bin_edges_number'] == 'None': # Enforce None for 'bin_edges_number' From 6fed659e91c417aa9ce239afbd057cff6225fd5e Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 20 May 2020 12:55:58 -0700 Subject: [PATCH 123/125] include srun_options in generate_slurm_config_file --- popsycle/run.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/popsycle/run.py b/popsycle/run.py index 8a499773..00e63826 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -129,6 +129,7 @@ def generate_field_config_file(longitude, latitude, area, def generate_slurm_config_file(path_python='python', account='ulens', queue='regular', resource='haswell', + srun_options=None, include_constraint=True, n_cores_per_node=32, n_nodes_max=2388, walltime_max='48:00:00', @@ -152,6 +153,12 @@ def generate_slurm_config_file(path_python='python', account='ulens', resource : str Computing resource name + srun_options : None, str + Options for slurm's srun command + + include_constraint : bool + Includes slurm option CONSTRAINT in slurm script header. Default True. + n_cores_per_node : int Number of cores in each node of the compute resource @@ -181,6 +188,7 @@ def generate_slurm_config_file(path_python='python', account='ulens', 'queue': queue, 'resource': resource, 'include_constraint': bool(include_constraint), + 'srun_options': srun_options, 'additional_lines': additional_lines, resource: {'n_cores_per_node': n_cores_per_node, 'n_nodes_max': n_nodes_max, From 018af97158db6974129fc4db544fd3b0083afec1 Mon Sep 17 00:00:00 2001 From: Michael Medford Date: Wed, 20 May 2020 12:56:52 -0700 Subject: [PATCH 124/125] rename docstring --- popsycle/run.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/popsycle/run.py b/popsycle/run.py index 00e63826..1a9e68ba 100755 --- a/popsycle/run.py +++ b/popsycle/run.py @@ -172,8 +172,6 @@ def generate_slurm_config_file(path_python='python', account='ulens', additional_lines : None, list of strings None, or additional lines to be run before executing run.py - Optional Parameters - ------------------- config_filename : str Name of the configuration file Default: slurm_config.yaml From aa4ca0e37a30d3746cfa0c7832c7c6ca9aeb8501 Mon Sep 17 00:00:00 2001 From: Kerianne Pruett Date: Sun, 18 Sep 2022 15:57:35 -0700 Subject: [PATCH 125/125] Velocity bug fix and code cleanup --- popsycle/synthetic.py | 204 +++++++++++++++++++++--------------------- 1 file changed, 102 insertions(+), 102 deletions(-) diff --git a/popsycle/synthetic.py b/popsycle/synthetic.py index bc2a67a0..cdef14e1 100755 --- a/popsycle/synthetic.py +++ b/popsycle/synthetic.py @@ -1702,10 +1702,10 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, t0 = time.time() - # Read in the hdf5 file that doesn't have PBHs. Product of perform_pop_syn. + # Read in the hdf5 file that doesn't have PBHs (product of perform_pop_syn) no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') key_list = list(no_pbh_hdf5_file) - # Delete lat_bin_edges, long_bin_edges, add_pbh from key_list. + # Delete lat_bin_edges, long_bin_edges, and add_pbh from key_list key_list = [key for key in key_list if 'bin_edges' not in key] key_list = [key for key in key_list if 'add_pbh' not in key] @@ -1714,8 +1714,8 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, long_bin = no_pbh_hdf5_file['long_bin_edges'][:] bin_edges_number = len(long_bin) - # Getting the maximum ID from all of the stars and compact objects. - # Later used to set the IDs of the PBHs. + # Get the maximum ID from all of the stars and compact objects + # (Later used to set the IDs of the PBHs) max_id_no_pbh = [] for key in key_list: if len(no_pbh_hdf5_file[key]) == 0: @@ -1729,113 +1729,112 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, # Read in ebf file t = ebf.read_ind(ebf_file, '/log', 0) - # Convert log to useful dictionary. + # Convert log to useful dictionary ebf_log = make_ebf_log(t) - # Obtain survey area and center latitude and longitude + # Obtain survey area, center latitude, and center longitude b = float(ebf_log['latitude']) # deg b_radian = b * np.pi / 180 # rad l = float(ebf_log['longitude']) # deg l_radian = l * np.pi / 180 # rad surveyArea = float(ebf_log['surveyArea']) # deg^2 - # Calculate the size of the field of view we are running + # Calculate the field of view for the current field field_of_view_radius = (surveyArea / np.pi) ** (1 / 2) # Generate an array of heliocentric radii - # These radii will just be used to numerically integrate the density + # (Just used to numerically integrate the line-of-sight density) n_lin = 100000 - r_h_linspace = np.linspace(0, r_max, num=n_lin) - # Represent the line of sight line in galactic coordinates + # Represent the line-of-sight line as galactic coordinates galactic_lin = coord.Galactic(l=l_radian * units.rad, b=b_radian * units.rad, distance=r_h_linspace * units.kpc) - # convert the line of sight to galactocentric coordinates - # outputs l, b, and distance in degrees. - galactocen_lin = galactic_lin.transform_to( + # Transform the line-of-sight into to galactocentric coordinates + # (Outputs l, b, and distance [units: deg, deg, kpc]) + galacto_lin = galactic_lin.transform_to( coord.Galactocentric(representation_type='spherical')) - # Determine the dark matter density at all galactocentric radii along the line of sight. - rho_lin = rho_dmhalo(galactocen_lin.spherical.distance.value, + # Determine dark matter density at all galactocentric radii along the line-of-sight + rho_lin = rho_dmhalo(galacto_lin.spherical.distance.value, rho_0=rho_0, r_s=r_s, gamma=gamma) # Estimate the total mass within the line-of-sight cylinder [units: M_sun kpc**-2] - # Projected density along line of light - # (multiply by projected area to get total mass) + # Total mass = projected line-of-sight density x projected line-of-sight area rho_marg_r = np.trapz(rho_lin, dx=(r_max) / n_lin) * 1000 ** 3 - print( - "Projected density along line-of-sight = {0:0.2e} [M_sun kpc**-2]".format( + print("Projected density along line-of-sight = {0:0.2e} [M_sun kpc**-2]".format( rho_marg_r)) - # LOS cylinder radius in kpc, assuming small angle approximation [units: kpc] + # Determine line-of-sight cylinder radius, assuming small angle approximation [units: kpc] r_proj_los_cyl = field_of_view_radius * np.pi / 180 * (r_max) - # Projected area of the LOS cylinder [units: kpc**2] + # Get projected area of the LOS cylinder [units: kpc**2] area_proj_los_cyl = np.pi * r_proj_los_cyl ** 2 - # Mass within the total cylinder + # Get the total mass within the line-of-sight cylinder mass_los_cyl = rho_marg_r * area_proj_los_cyl print("Mass within line-of-sight cylinder = {0:0.2e} [M_sun]".format( mass_los_cyl)) - # Total number of black holes to randomly draw + # Get the total number of black holes to randomly draw n_pbh = int(np.round(fdm * mass_los_cyl / pbh_mass)) # Estimate the discrete CDF based on the discrete PDF + # (Switch into galactic coordinates) rho_marg_r_cum = integrate.cumtrapz(y=rho_lin, x=galactic_lin.distance.kpc, dx=(r_max) / n_lin) cdf_los = rho_marg_r_cum / rho_marg_r_cum[-1] - # Since cumtrapz does not include zero for the first element insert it + # Since cumtrapz does not include zero for the first element, insert it cdf_los = np.insert(cdf_los, 0, 0) # Create a function to interpolate the CDF so that we can randomly sample from it f_cdf_d = interpolate.interp1d(cdf_los, galactic_lin.distance.kpc) - # Randomly sample galactic coordinates for the PBHs based on CDF - d_galac = f_cdf_d(np.random.uniform(size=n_pbh)) + # Randomly sample galactic coordinates for the PBHs based on the CDF + d_galactic = f_cdf_d(np.random.uniform(size=n_pbh)) - # Randomly assign a l & b galactic coordinate to each PBH, within the LOS cone - # sample the angle from 0 to 2pi + # Randomly assign an l & b galactic coordinate to each PBH within the + # line-of-sight cone (sample the angle from 0 to 2pi [units: radians]) theta = np.random.uniform(size=n_pbh) * 2 * np.pi - # sample radius correcting for annular area to make uniform + # Make uniform by sampling a radius and correcting for annular area r_cyl = r_proj_los_cyl * np.sqrt(np.random.uniform(size=n_pbh)) # kpc y_cyl = r_cyl * np.sin(theta) # kpc x_cyl = r_cyl * np.cos(theta) # kpc - # Mask out sampled PBH outside the observation cone - mask_obs_cone = r_cyl <= r_proj_los_cyl * d_galac / (r_max) + # Create mask for masking out PBHs falling outside of the observation cone + mask_obs_cone = r_cyl <= r_proj_los_cyl * d_galactic / (r_max) print( 'Number of PBH before and after light cone masking: {0} and {1}, respectively'.format( n_pbh, np.sum(mask_obs_cone))) - # Assuming small angle approximation - b_galac = r_cyl * np.sin(theta) / d_galac + b_radian # rad - l_galac = r_cyl * np.cos(theta) / np.cos( - b_radian) / d_galac + l_radian # rad + # Get galactic l and b, assuming the small angle approximation + b_galactic = r_cyl * np.sin(theta) / d_galactic + b_radian # rad + l_galactic = r_cyl * np.cos(theta) / np.cos(b_radian) / d_galactic + l_radian # rad if diagnostic_plots: print('Saving diagnostic plots') from popsycle.add_pbh_plots import print_plots print_plots(output_root=output_root, galactic_lin_distance=galactic_lin.distance.kpc, galactic_lin_b=galactic_lin.b.deg, galactic_lin_l=galactic_lin.l.deg, - galactocen_lin_spherical_distance=galactocen_lin.spherical.distance.kpc, - galactocen_lin_spherical_b=galactocen_lin.spherical.lat.deg, - galactocen_lin_spherical_l=galactocen_lin.spherical.lon.deg, rho_lin=rho_lin, + galactocen_lin_spherical_distance=galacto_lin.spherical.distance.kpc, + galactocen_lin_spherical_b=galacto_lin.spherical.lat.deg, + galactocen_lin_spherical_l=galacto_lin.spherical.lon.deg, rho_lin=rho_lin, r_max=r_max, n_lin=n_lin, cdf_los=cdf_los, x_cyl=x_cyl, y_cyl=y_cyl, r_cyl=r_cyl, r_proj_los_cyl=r_proj_los_cyl, n_pbh=n_pbh, - d_galac=d_galac, b_galac=b_galac, l_galac=l_galac, area_proj_los_cyl=area_proj_los_cyl, + d_galac=d_galactic, b_galac=b_galactic, l_galac=l_galactic, area_proj_los_cyl=area_proj_los_cyl, mask_obs_cone=mask_obs_cone, field_of_view_radius=field_of_view_radius, l_radian=l_radian, b_radian=b_radian, f_cdf_d=f_cdf_d, pbh_mass=pbh_mass) - d_galac = d_galac[mask_obs_cone] - b_galac = b_galac[mask_obs_cone] - l_galac = l_galac[mask_obs_cone] + # Mask out PBHs outside of the observation cone + d_galactic = d_galactic[mask_obs_cone] + b_galactic = b_galactic[mask_obs_cone] + l_galactic = l_galactic[mask_obs_cone] - latitude = b_galac * (180 / np.pi) # degrees - longitude = l_galac * (180 / np.pi) # degrees + latitude = b_galactic * (180 / np.pi) # degrees + longitude = l_galactic * (180 / np.pi) # degrees - N_PBHs_in_field = len(d_galac) + # Determine how many PBHs are in the current field + N_PBHs_in_field = len(d_galactic) print('%i PBHs in the field' % N_PBHs_in_field) if N_PBHs_in_field == 0: @@ -1844,18 +1843,16 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, shutil.copy(hdf5_file, output_hdf5_file) return - # Converting the PBH positions from the field of view back to galactocentric for determining velocities. + # Convert PBH positions back to galactocentric for determining velocities galactic_pbh = coord.Galactic(l=longitude * units.deg, b=latitude * units.deg, - distance=d_galac * units.kpc) + distance=d_galactic * units.kpc) galacto_pbh = galactic_pbh.transform_to( - coord.Galactocentric(representation_type='spherical')) - cart_pbh = coord.cartesian_to_spherical(galacto_pbh.x, - galacto_pbh.y, - galacto_pbh.z) - pbh_r_galacto = cart_pbh[0] + coord.Galactocentric(representation_type='cartesian')) + # Get the radial galactocentric PBH distances + pbh_r_galacto = galacto_pbh.spherical.distance.value - # Inner slope of the MW halo + # Inner slope of the MW halo: # From Lacroix et al 2018, Figure 11 (top left panel) data_dir = '%s/data' % os.path.dirname(inspect.getfile(add_pbh)) if gamma == 1: @@ -1870,87 +1867,90 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, else: raise Exception('gamma (%s) must be either .25, .5, 1' % str(gamma)) - # Interpolating v values from the above data, given the PBH r values. - pbh_vrms = np.interp(pbh_r_galacto.value, vel_data['r'], vel_data['v']) + # Interpolate v values from the above data, given PBH radii + pbh_vrms = np.interp(pbh_r_galacto, vel_data['r'], vel_data['v']) v_vals = np.arange(0, v_esc) a = (1 / 2) * pbh_vrms * ((np.pi / 2) ** (1 / 2)) - # Calculating the v_rms velocities for the PBHs by randomly sampling from the CDF. - rand_cdf = np.array([]) + # Calculate RMS velocities for each PBH by randomly sampling the Maxwellian CDF + rms_velocities = [] for a_val in a: - cdf = special.erf(v_vals / (a_val * 2 ** (1 / 2))) - ( - ((2 / np.pi) ** (1 / 2)) * ((v_vals * np.exp( - -v_vals ** 2 / 2 * a_val ** 2)) / a_val)) - rand_cdf = np.append(rand_cdf, np.random.uniform(0, np.amax(cdf))) - interpreted_rms_velocities = np.interp(rand_cdf, cdf, v_vals) - - # Sampling random latitude and longitude values for velocity to complete the spherical velocities. - sin_lat_vel = np.random.uniform(-1, 1, len(d_galac)) + cdf = special.erf(v_vals / (a_val * 2 ** (1 / 2))) - \ + (((2 / np.pi) ** (1 / 2)) * ((v_vals * np.exp(-v_vals ** 2 / (2 * a_val ** 2)) / a_val))) + rand = np.random.uniform(0,1) + rms_velocities.append(np.interp(rand, cdf, v_vals)) + + # Complete PBH spherical velocities by sampling a latitude and longitude + # (arcsin is used so that PBHs are placed in more realistic locations) + sin_lat_vel = np.random.uniform(-1, 1, len(d_galactic)) lat_vel = np.arcsin(sin_lat_vel) - long_vel = np.random.uniform(0, 2 * np.pi, len(d_galac)) + long_vel = np.random.uniform(0, 2 * np.pi, len(d_galactic)) - # Transforming velocities to cartesian to get vx, vy, and vz. - cart_vel = coord.spherical_to_cartesian(interpreted_rms_velocities, - lat_vel, long_vel) + # Transform velocities to cartesian to get vx, vy, and vz + cart_vel = coord.spherical_to_cartesian(rms_velocities, lat_vel, long_vel) # Load up a numpy array comp_dtype = _generate_comp_dtype(hdf5_dset_names) - pbh_data = np.empty(len(d_galac), dtype=comp_dtype) + pbh_data = np.empty(len(d_galactic), dtype=comp_dtype) - # Getting longitude into the right format for PopSyCLE + # Transform latitude and longitude into the right format for PopSyCLE longitude = np.where(longitude > 180, longitude - 360, longitude) - pbh_data['rad'] = d_galac + # Set the galactic coordinates + pbh_data['rad'] = d_galactic pbh_data['glon'] = longitude pbh_data['glat'] = latitude + # Set heliocentric velocities equal to galactocentric velocities pbh_data['vx'] = cart_vel[0] pbh_data['vy'] = cart_vel[1] pbh_data['vz'] = cart_vel[2] - # Getting the rest of the PBH data for the combined .h5 file - pbh_data['mass'] = np.full(len(d_galac), pbh_mass) - pbh_data['zams_mass'] = np.full(len(d_galac), pbh_mass) - pbh_data['age'] = np.full(len(d_galac), np.nan) - pbh_data['popid'] = np.full(len(d_galac), 10) - pbh_data['rem_id'] = np.full(len(d_galac), 104) + # Get the rest of the PBH data for the combined .h5 file + pbh_data['mass'] = np.full(len(d_galactic), pbh_mass) + pbh_data['zams_mass'] = np.full(len(d_galactic), pbh_mass) + pbh_data['age'] = np.full(len(d_galactic), np.nan) + pbh_data['popid'] = np.full(len(d_galactic), 10) + pbh_data['rem_id'] = np.full(len(d_galactic), 104) - cart_helio = coord.spherical_to_cartesian(d_galac, b_galac, l_galac) + # Set the heliocentric PBH positions + cart_helio = coord.spherical_to_cartesian(d_galactic, b_galactic, l_galactic) pbh_data['px'] = cart_helio[0] pbh_data['py'] = cart_helio[1] pbh_data['pz'] = cart_helio[2] - vr, mu_b, mu_lcosb = calc_sph_motion(pbh_data['vx'], - pbh_data['vy'], - pbh_data['vz'], - d_galac, b_galac, l_galac) + # Set the galactic velocity and proper motions + vr, mu_b, mu_lcosb = calc_sph_motion(cart_vel[0], + cart_vel[1], + cart_vel[2], + d_galactic, b_galactic, l_galactic) pbh_data['vr'] = vr pbh_data['mu_b'] = mu_b pbh_data['mu_lcosb'] = mu_lcosb - pbh_data['obj_id'] = np.arange((max_id + 1), (max_id + len(d_galac) + 1)) - - pbh_data['exbv'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_K'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_J'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_I'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_U'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_R'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_B'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_H'] = np.full(len(d_galac), np.nan) - pbh_data['ubv_V'] = np.full(len(d_galac), np.nan) - - # The first four elements may not be in the compound datatype + + pbh_data['obj_id'] = np.arange((max_id + 1), (max_id + len(d_galactic) + 1)) + + pbh_data['exbv'] = np.full(len(d_galactic), np.nan) + pbh_data['ubv_K'] = np.full(len(d_galactic), np.nan) + pbh_data['ubv_J'] = np.full(len(d_galactic), np.nan) + pbh_data['ubv_I'] = np.full(len(d_galactic), np.nan) + pbh_data['ubv_U'] = np.full(len(d_galactic), np.nan) + pbh_data['ubv_R'] = np.full(len(d_galactic), np.nan) + pbh_data['ubv_B'] = np.full(len(d_galactic), np.nan) + pbh_data['ubv_H'] = np.full(len(d_galactic), np.nan) + pbh_data['ubv_V'] = np.full(len(d_galactic), np.nan) + + # Note: The first four elements may not be in the compound datatype # due to legacy files. The ztf filters will only be present if # the hdf5 file was created with additional_photometric_systems = ['ztf'] for field in ['teff', 'grav', 'mbol', 'feh', 'ztf_g', 'ztf_r', 'ztf_i']: if field in hdf5_dset_names: - pbh_data[field] = np.full(len(d_galac), np.nan) + pbh_data[field] = np.full(len(d_galactic), np.nan) - # Calculate the maximum and minimum l and b values for each dataset in - # the no PBH file, so that we can determine which datasets to - # correctly append the PBHs. + # Calculate the maximum and minimum l and b values for each dataset in the + # file with no PBHs, to determine the datasets to correctly append the PBHs to lat_long_list = [] for key in key_list: idx_l = int(key.split('b')[0].replace('l', '')) @@ -1963,12 +1963,12 @@ def add_pbh(hdf5_file, ebf_file, fdm=1, pbh_mass=40, lat_long_list.append((min_l, max_l, min_b, max_b)) - # Opening the file with no PBHs and creating a new file for the PBHs added. + # Open the file with no PBHs and create a new file for data with PBHs added no_pbh_hdf5_file = h5py.File(hdf5_file, 'r') pbh_hdf5_file = h5py.File(output_hdf5_file, 'w') pbh_hdf5_file['add_pbh'] = True - # Appending the PBH data to the no PBH data and writing to the new .h5 file. + # Append the PBH data to the data with no PBHs, and write to an .h5 file N_objs_no_pbh = 0 N_objs_pbh = 0 N_pbhs_removed = 0