pandas updates and updated error catching

.gitignore

@@ -92,3 +92,8 @@ docs/_build/
 #Mac ignored files
 ####################
 .DS_Store
+
+
+.history/
+.vscode/
+.spyder/

obsio/providers/acis.py

@@ -295,6 +295,6 @@ def _read_obs(self, stns_ids=None):
         obs_all = obs_all.rename(columns={'uid': 'station_id'})
 
         obs_all = obs_all.set_index(['station_id', 'elem', 'time'])
-        obs_all = obs_all.sortlevel(0, sort_remaining=True)
+        obs_all = obs_all.sort_index(0, sort_remaining=True)
 
         return obs_all

obsio/providers/ghcnd.py

@@ -585,7 +585,7 @@ def dummy_close():
             pd.set_option('mode.chained_assignment', opt_val)
 
         df_obs = df_obs.set_index(['station_id', 'elem', 'time'])
-        df_obs = df_obs.sortlevel(0, sort_remaining=True)
+        df_obs = df_obs.sort_index(0, sort_remaining=True)
 
         return df_obs
 
@@ -679,6 +679,6 @@ def dummy_close():
             pd.set_option('mode.chained_assignment', opt_val)
 
         df_obs = df_obs.set_index(['station_id', 'elem', 'time'])
-        df_obs = df_obs.sortlevel(0, sort_remaining=True)
+        df_obs = df_obs.sort_index(0, sort_remaining=True)
 
         return df_obs

obsio/providers/hdf.py

@@ -55,7 +55,7 @@ def _read_obs(self, stns_ids=None):
         obs.name = 'obs_value'
         obs.index.rename('elem', level=2, inplace=True)
         obs = obs.reorder_levels(['station_id', 'elem',
-                                  'time']).sortlevel(0, sort_remaining=True)
+                                  'time']).sort_index(0, sort_remaining=True)
         obs = pd.DataFrame(obs)
 
         return obs

obsio/providers/isd.py

@@ -386,6 +386,6 @@ def dummy_close():
             pd.set_option('mode.chained_assignment', opt_val)
 
         obs_all = obs_all.set_index(['station_id', 'elem', 'time'])
-        obs_all = obs_all.sortlevel(0, sort_remaining=True)
+        obs_all = obs_all.sort_index(0, sort_remaining=True)
 
         return obs_all

obsio/providers/madis.py

@@ -1744,7 +1744,7 @@ def transform_to_daily(obs_tz, a_date):
 
         all_obs = pd.concat(all_obs, ignore_index=True)
         all_obs = all_obs.set_index(['station_id', 'elem', 'time'])
-        all_obs = all_obs.sortlevel(0, sort_remaining=True)
+        all_obs = all_obs.sort_index(0, sort_remaining=True)
 
         return all_obs
 

obsio/providers/netcdf.py

@@ -61,7 +61,7 @@ def _read_obs(self, stns_ids=None):
         obs = pd.concat(obs)
 
         obs = obs.reorder_levels(['station_id', 'elem',
-                                  'time']).sortlevel(0, sort_remaining=True)
+                                  'time']).sort_index(0, sort_remaining=True)
 
         return obs
 

obsio/providers/nrcs.py

@@ -501,7 +501,7 @@ def read_obs(self, start_date, end_date, stns):
 
                 try:
                     obs_stn = pd.DataFrame(a_data.values, columns=['obs_value'])
-                    obs_stn['time'] = pd.date_range(a_data.beginDate, a_data.endDate)
+                    obs_stn['time'] = pd.date_range(a_data.beginDate.apply(repr), a_data.endDate.apply(repr))
                     obs_stn['stationTriplet'] = a_data.stationTriplet
                     obs_stn['obs_value'] = _convert_funcs[a_elem](obs_stn['obs_value'])
                     obs_stn['elem'] = a_elem
@@ -657,14 +657,15 @@ def _read_stns(self):
                                        stationTriplets=stn_triplets)
 
         stn_tups = [self._stationMetadata_to_tuple(a) for a in stn_metas]
+
         df_stns = pd.DataFrame(stn_tups, columns=self._stnmeta_attrs)
 
         stns = df_stns.rename(columns={'actonId': 'station_id',
                                        'name': 'station_name'})
         stns['station_id'] = stns.station_id.fillna(stns.stationTriplet)
         stns = stns[~stns.station_id.isnull()]
-        stns['beginDate'] = pd.to_datetime(stns.beginDate)
-        stns['endDate'] = pd.to_datetime(stns.endDate)
+        stns['beginDate'] = pd.to_datetime(stns.beginDate.apply(repr))
+        stns['endDate'] = pd.to_datetime(stns.endDate.apply(repr))
         stns['elevation'] = _ft_to_m(stns.elevation)
         stns['provider'] = 'NRCS'
         stns['sub_provider'] = ''
@@ -740,7 +741,7 @@ def check_empty(df_obs):
 
         obs_merge = obs_merge.drop('stationTriplet', axis=1)
         obs_merge = obs_merge.set_index(['station_id', 'elem', 'time'])
-        obs_merge = obs_merge.sortlevel(0, sort_remaining=True)
+        obs_merge = obs_merge.sort_index(0, sort_remaining=True)
         return obs_merge
 
     def _stationMetadata_to_tuple(self, a_meta):

obsio/providers/uscrn.py

@@ -102,6 +102,6 @@ def _read_obs(self, stns_ids=None):
 
         obs_all = pd.concat(obs_all, ignore_index=True)
         obs_all = obs_all.set_index(['station_id', 'elem', 'time'])
-        obs_all = obs_all.sortlevel(0, sort_remaining=True)
+        obs_all = obs_all.sort_index(0, sort_remaining=True)
 
         return obs_all

obsio/providers/ushcn.py

@@ -161,7 +161,7 @@ def _parse_stn_obs(self, stn_id, elem):
 
         obs_file.close()
 
-        obs = obs.unstack().swaplevel(0, 1).sortlevel(0, sort_remaining=True)
+        obs = obs.unstack().swaplevel(0, 1).sort_index(0, sort_remaining=True)
         obs = obs.reset_index()
         obs['time'] = pd.to_datetime(obs.year.astype(np.str) + obs.level_1,
                                      format='%Y%m')
@@ -207,6 +207,6 @@ def _read_obs(self, stns_ids=None):
             pd.set_option('mode.chained_assignment', opt_val)
 
         obs = obs.set_index(['station_id', 'elem', 'time'])
-        obs = obs.sortlevel(0, sort_remaining=True)
+        obs = obs.sort_index(0, sort_remaining=True)
 
         return obs

obsio/providers/wrcc.py

@@ -517,6 +517,6 @@ def dummy_close():
 
         df_obs = pd.concat(obs, ignore_index=True)
         df_obs = df_obs.set_index(['station_id', 'elem', 'time'])
-        df_obs = df_obs.sortlevel(0, sort_remaining=True)
+        df_obs = df_obs.sort_index(0, sort_remaining=True)
 
         return df_obs

obsio/util/misc.py

@@ -324,14 +324,11 @@ def download_if_new_ftp(a_ftp, fpath_ftp, fpath_local):
         mtime_local = datetime.utcfromtimestamp(os.path.getmtime(fpath_local))
         size_local = os.path.getsize(fpath_local)
 
-    except OSError as e:
+    except FileNotFoundError as e:
 
-        if e.args[-1] == 'No such file or directory':
-            mtime_local = None
-            size_local = None
-        else:
-            raise
-
+        mtime_local = None
+        size_local = None
+
     if (size_remote != size_local) or (mtime_remote > mtime_local):
 
         print("Downloading %s to %s..."%(fpath_ftp, fpath_local))