diff --git a/.gitignore b/.gitignore index ae14bb2..33f5b79 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,8 @@ docs/_build/ #Mac ignored files #################### .DS_Store + + +.history/ +.vscode/ +.spyder/ \ No newline at end of file diff --git a/obsio/providers/acis.py b/obsio/providers/acis.py index 1e95a59..6a82051 100644 --- a/obsio/providers/acis.py +++ b/obsio/providers/acis.py @@ -295,6 +295,6 @@ def _read_obs(self, stns_ids=None): obs_all = obs_all.rename(columns={'uid': 'station_id'}) obs_all = obs_all.set_index(['station_id', 'elem', 'time']) - obs_all = obs_all.sortlevel(0, sort_remaining=True) + obs_all = obs_all.sort_index(0, sort_remaining=True) return obs_all diff --git a/obsio/providers/ghcnd.py b/obsio/providers/ghcnd.py index 029b7f4..3d1dc1a 100644 --- a/obsio/providers/ghcnd.py +++ b/obsio/providers/ghcnd.py @@ -585,7 +585,7 @@ def dummy_close(): pd.set_option('mode.chained_assignment', opt_val) df_obs = df_obs.set_index(['station_id', 'elem', 'time']) - df_obs = df_obs.sortlevel(0, sort_remaining=True) + df_obs = df_obs.sort_index(0, sort_remaining=True) return df_obs @@ -679,6 +679,6 @@ def dummy_close(): pd.set_option('mode.chained_assignment', opt_val) df_obs = df_obs.set_index(['station_id', 'elem', 'time']) - df_obs = df_obs.sortlevel(0, sort_remaining=True) + df_obs = df_obs.sort_index(0, sort_remaining=True) return df_obs diff --git a/obsio/providers/hdf.py b/obsio/providers/hdf.py index 11a15c6..cd9136b 100644 --- a/obsio/providers/hdf.py +++ b/obsio/providers/hdf.py @@ -55,7 +55,7 @@ def _read_obs(self, stns_ids=None): obs.name = 'obs_value' obs.index.rename('elem', level=2, inplace=True) obs = obs.reorder_levels(['station_id', 'elem', - 'time']).sortlevel(0, sort_remaining=True) + 'time']).sort_index(0, sort_remaining=True) obs = pd.DataFrame(obs) return obs diff --git a/obsio/providers/isd.py b/obsio/providers/isd.py index ce058af..545333a 100644 --- a/obsio/providers/isd.py +++ b/obsio/providers/isd.py @@ -386,6 +386,6 @@ def dummy_close(): pd.set_option('mode.chained_assignment', opt_val) obs_all = obs_all.set_index(['station_id', 'elem', 'time']) - obs_all = obs_all.sortlevel(0, sort_remaining=True) + obs_all = obs_all.sort_index(0, sort_remaining=True) return obs_all diff --git a/obsio/providers/madis.py b/obsio/providers/madis.py index 7b024d3..03c0097 100644 --- a/obsio/providers/madis.py +++ b/obsio/providers/madis.py @@ -1744,7 +1744,7 @@ def transform_to_daily(obs_tz, a_date): all_obs = pd.concat(all_obs, ignore_index=True) all_obs = all_obs.set_index(['station_id', 'elem', 'time']) - all_obs = all_obs.sortlevel(0, sort_remaining=True) + all_obs = all_obs.sort_index(0, sort_remaining=True) return all_obs diff --git a/obsio/providers/netcdf.py b/obsio/providers/netcdf.py index dae5fe1..dc8f0f3 100644 --- a/obsio/providers/netcdf.py +++ b/obsio/providers/netcdf.py @@ -61,7 +61,7 @@ def _read_obs(self, stns_ids=None): obs = pd.concat(obs) obs = obs.reorder_levels(['station_id', 'elem', - 'time']).sortlevel(0, sort_remaining=True) + 'time']).sort_index(0, sort_remaining=True) return obs diff --git a/obsio/providers/nrcs.py b/obsio/providers/nrcs.py index a1cd568..a979625 100644 --- a/obsio/providers/nrcs.py +++ b/obsio/providers/nrcs.py @@ -501,7 +501,7 @@ def read_obs(self, start_date, end_date, stns): try: obs_stn = pd.DataFrame(a_data.values, columns=['obs_value']) - obs_stn['time'] = pd.date_range(a_data.beginDate, a_data.endDate) + obs_stn['time'] = pd.date_range(a_data.beginDate.apply(repr), a_data.endDate.apply(repr)) obs_stn['stationTriplet'] = a_data.stationTriplet obs_stn['obs_value'] = _convert_funcs[a_elem](obs_stn['obs_value']) obs_stn['elem'] = a_elem @@ -657,14 +657,15 @@ def _read_stns(self): stationTriplets=stn_triplets) stn_tups = [self._stationMetadata_to_tuple(a) for a in stn_metas] + df_stns = pd.DataFrame(stn_tups, columns=self._stnmeta_attrs) stns = df_stns.rename(columns={'actonId': 'station_id', 'name': 'station_name'}) stns['station_id'] = stns.station_id.fillna(stns.stationTriplet) stns = stns[~stns.station_id.isnull()] - stns['beginDate'] = pd.to_datetime(stns.beginDate) - stns['endDate'] = pd.to_datetime(stns.endDate) + stns['beginDate'] = pd.to_datetime(stns.beginDate.apply(repr)) + stns['endDate'] = pd.to_datetime(stns.endDate.apply(repr)) stns['elevation'] = _ft_to_m(stns.elevation) stns['provider'] = 'NRCS' stns['sub_provider'] = '' @@ -740,7 +741,7 @@ def check_empty(df_obs): obs_merge = obs_merge.drop('stationTriplet', axis=1) obs_merge = obs_merge.set_index(['station_id', 'elem', 'time']) - obs_merge = obs_merge.sortlevel(0, sort_remaining=True) + obs_merge = obs_merge.sort_index(0, sort_remaining=True) return obs_merge def _stationMetadata_to_tuple(self, a_meta): diff --git a/obsio/providers/uscrn.py b/obsio/providers/uscrn.py index 787501a..d968729 100644 --- a/obsio/providers/uscrn.py +++ b/obsio/providers/uscrn.py @@ -102,6 +102,6 @@ def _read_obs(self, stns_ids=None): obs_all = pd.concat(obs_all, ignore_index=True) obs_all = obs_all.set_index(['station_id', 'elem', 'time']) - obs_all = obs_all.sortlevel(0, sort_remaining=True) + obs_all = obs_all.sort_index(0, sort_remaining=True) return obs_all diff --git a/obsio/providers/ushcn.py b/obsio/providers/ushcn.py index b64febf..a9dcfd4 100644 --- a/obsio/providers/ushcn.py +++ b/obsio/providers/ushcn.py @@ -161,7 +161,7 @@ def _parse_stn_obs(self, stn_id, elem): obs_file.close() - obs = obs.unstack().swaplevel(0, 1).sortlevel(0, sort_remaining=True) + obs = obs.unstack().swaplevel(0, 1).sort_index(0, sort_remaining=True) obs = obs.reset_index() obs['time'] = pd.to_datetime(obs.year.astype(np.str) + obs.level_1, format='%Y%m') @@ -207,6 +207,6 @@ def _read_obs(self, stns_ids=None): pd.set_option('mode.chained_assignment', opt_val) obs = obs.set_index(['station_id', 'elem', 'time']) - obs = obs.sortlevel(0, sort_remaining=True) + obs = obs.sort_index(0, sort_remaining=True) return obs diff --git a/obsio/providers/wrcc.py b/obsio/providers/wrcc.py index 56a55ec..21ee7e5 100644 --- a/obsio/providers/wrcc.py +++ b/obsio/providers/wrcc.py @@ -517,6 +517,6 @@ def dummy_close(): df_obs = pd.concat(obs, ignore_index=True) df_obs = df_obs.set_index(['station_id', 'elem', 'time']) - df_obs = df_obs.sortlevel(0, sort_remaining=True) + df_obs = df_obs.sort_index(0, sort_remaining=True) return df_obs diff --git a/obsio/util/misc.py b/obsio/util/misc.py index e730d8e..bb7b9a1 100644 --- a/obsio/util/misc.py +++ b/obsio/util/misc.py @@ -324,14 +324,11 @@ def download_if_new_ftp(a_ftp, fpath_ftp, fpath_local): mtime_local = datetime.utcfromtimestamp(os.path.getmtime(fpath_local)) size_local = os.path.getsize(fpath_local) - except OSError as e: + except FileNotFoundError as e: - if e.args[-1] == 'No such file or directory': - mtime_local = None - size_local = None - else: - raise - + mtime_local = None + size_local = None + if (size_remote != size_local) or (mtime_remote > mtime_local): print("Downloading %s to %s..."%(fpath_ftp, fpath_local))