Add variable description and feature when we load

data to be compatible with other data (in this case Frenger(2018) data)

Author: AntSimi

src/py_eddy_tracker/__init__.py

@@ -163,7 +163,7 @@ def parse_args(self, *args, **kwargs):
         attr_name='lon',
         compute_type='float64',
         nc_name='longitude',
-        old_nc_name=['lon'],
+        old_nc_name=['lon', 'Lon'],
         nc_type='float32',
         nc_dims=('obs',),
         nc_attr=dict(
@@ -178,7 +178,7 @@ def parse_args(self, *args, **kwargs):
         attr_name='lat',
         compute_type='float64',
         nc_name='latitude',
-        old_nc_name=['lat'],
+        old_nc_name=['lat', 'Lat'],
         nc_type='float32',
         nc_dims=('obs',),
         nc_attr=dict(
@@ -296,7 +296,7 @@ def parse_args(self, *args, **kwargs):
     radius_e=dict(
         attr_name='radius_e',
         nc_name='effective_radius',
-        old_nc_name=['radius_e'],
+        old_nc_name=['radius_e', 'Dia'],
         nc_type='float32',
         output_type='u2',
         scale_factor=50.,
@@ -324,6 +324,7 @@ def parse_args(self, *args, **kwargs):
     track=dict(
         attr_name=None,
         nc_name='track',
+        old_nc_name=['Eddy_id'],
         nc_type='uint32',
         nc_dims=('obs',),
         nc_attr=dict(
@@ -346,7 +347,7 @@ def parse_args(self, *args, **kwargs):
     n=dict(
         attr_name=None,
         nc_name='observation_number',
-        old_nc_name=['n'],
+        old_nc_name=['n', 'Eddy_tsp'],
         nc_type='uint16',
         nc_dims=('obs',),
         nc_attr=dict(
@@ -481,6 +482,72 @@ def parse_args(self, *args, **kwargs):
             units='m',
         )
     ),
+    chl=dict(
+        attr_name=None,
+        nc_name='chl',
+        old_nc_name=['Chl'],
+        nc_type='f4',
+        nc_dims=('obs',),
+        nc_attr=dict(
+            longname='Log base 10 chlorophyll',
+            units='Log(Chl/[mg/m^3])',
+        )
+    ),
+    dchl=dict(
+        attr_name=None,
+        nc_name='dchl',
+        old_nc_name=['dChl'],
+        nc_type='f4',
+        nc_dims=('obs',),
+        nc_attr=dict(
+            longname='Log base 10 chlorophyll anomaly (Chl minus Chl_bg)',
+            units='Log(Chl/[mg/m^3])',
+        )
+    ),
+    chl_bg=dict(
+        attr_name=None,
+        nc_name='chl_bg',
+        old_nc_name=['Chl_bg'],
+        nc_type='f4',
+        nc_dims=('obs',),
+        nc_attr=dict(
+            longname='Log base 10 background chlorophyll',
+            units='Log(Chl/[mg/m^3])',
+        )
+    ),
+    year=dict(
+        attr_name=None,
+        nc_name='year',
+        old_nc_name=['Year'],
+        nc_type='u2',
+        nc_dims=('obs',),
+        nc_attr=dict(
+            longname='Year',
+            units='year',
+        )
+    ),
+    month=dict(
+        attr_name=None,
+        nc_name='month',
+        old_nc_name=['Month'],
+        nc_type='u1',
+        nc_dims=('obs',),
+        nc_attr=dict(
+            longname='Month',
+            units='month',
+        )
+    ),
+    day=dict(
+        attr_name=None,
+        nc_name='day',
+        old_nc_name=['Day'],
+        nc_type='u1',
+        nc_dims=('obs',),
+        nc_attr=dict(
+            longname='Day',
+            units='day',
+        )
+    ),
     nb_contour_selected=dict(
         attr_name=None,
         nc_name='num_contours',

src/py_eddy_tracker/observations/observation.py

@@ -49,6 +49,9 @@
 from datetime import datetime
 from numba import njit, types as numba_types
 from Polygon import Polygon
+from pint import UnitRegistry
+from pint.errors import UndefinedUnitError
+from pint.compat.tokenize import TokenError
 
 
 @njit(cache=True, fastmath=True)
@@ -210,6 +213,12 @@ def __getitem__(self, attr):
             return self.observations[attr]
         raise KeyError("%s unknown" % attr)
 
+    @classmethod
+    def obs_dimension(cls, handler):
+        for candidate in ('obs', 'Nobs', 'observation', 'i'):
+            if candidate in handler.dimensions.keys():
+                return candidate
+
     @property
     def dtype(self):
         """Return dtype to build numpy array
@@ -357,7 +366,8 @@ def load_from_netcdf(cls, filename, raw_data=False):
         if not isinstance(filename, str):
             filename = filename.astype(str)
         with Dataset(filename) as h_nc:
-            nb_obs = len(h_nc.dimensions["obs"])
+            nb_obs = len(h_nc.dimensions[cls.obs_dimension(h_nc)])
+            logging.debug('%d observations will be load', nb_obs)
             kwargs = dict()
             if array_dim in h_nc.dimensions:
                 kwargs["track_array_variables"] = len(h_nc.dimensions[array_dim])
@@ -382,7 +392,32 @@ def load_from_netcdf(cls, filename, raw_data=False):
                     continue
                 # Patch
                 h_nc.variables[variable].set_auto_maskandscale(not raw_data)
-                eddies.obs[var_inv] = h_nc.variables[variable][:]
+                logging.debug('Up load %s variable%s', variable, ', with raw mode' if raw_data else '')
+                # find unit factor
+                factor = 1
+                if not raw_data:
+                    input_unit = getattr(h_nc.variables[variable], 'unit', None)
+                    output_unit = VAR_DESCR[var_inv]['nc_attr'].get('units', None)
+                    if output_unit is not None and input_unit is not None and output_unit != input_unit:
+                        units = UnitRegistry()
+                        try:
+                            input_unit = units.parse_expression(input_unit, case_sensitive=False)
+                            output_unit = units.parse_expression(output_unit, case_sensitive=False)
+                        except UndefinedUnitError:
+                            input_unit = None
+                        except TokenError:
+                            input_unit = None
+                        if input_unit is not None:
+                            factor = input_unit.to(output_unit).to_tuple()[0]
+                            # If we are able to find a conversion
+                            if factor != 1:
+                                logging.info('%s will be multiply by %f to take care of units', variable, factor)
+                if factor != 1:
+                    eddies.obs[var_inv] = h_nc.variables[variable][:] * factor
+                else:
+                    eddies.obs[var_inv] = h_nc.variables[variable][:]
+
+
             for variable in h_nc.variables:
                 var_inv = VAR_DESCR_inv[variable]
                 if var_inv == "type_cyc":
@@ -396,7 +431,7 @@ def load_from_netcdf(cls, filename, raw_data=False):
 
     @classmethod
     def from_netcdf(cls, handler):
-        nb_obs = len(handler.dimensions["obs"])
+        nb_obs = len(handler.dimensions[cls.obs_dimension(handler)])
         kwargs = dict()
         if hasattr(handler, "track_array_variables"):
             kwargs["track_array_variables"] = handler.track_array_variables

src/scripts/EddySubSetter

@@ -23,13 +23,17 @@ def id_parser():
                         help='Coordinates of bounding to extract'
                         )
     parser.add_argument('-i', '--ids', nargs='+', type=int, help='List of tracks which will be extract')
+    parser.add_argument('-n', '--no_raw_mode', action='store_true',
+                        help='Uncompress all data, could be create a memory error for huge file, but is safer for extern file of py eddy tracker')
     return parser
 
 
 if __name__ == '__main__':
     args = id_parser().parse_args()
 
-    dataset = TrackEddiesObservations.load_from_netcdf(args.filename, raw_data=True)
+    # Original dataset
+    dataset = TrackEddiesObservations.load_from_netcdf(args.filename, raw_data=False if args.no_raw_mode else True)
+
     if args.ids is not None:
         dataset = dataset.extract_ids(args.ids)
     if args.period is not None: