From c63fea2adb614410dd0db4f969eac7ce0731fdec Mon Sep 17 00:00:00 2001 From: Antoine <36040805+AntSimi@users.noreply.github.com> Date: Thu, 16 Sep 2021 13:18:39 +0200 Subject: [PATCH 1/9] remove argument for conda setup --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 941bb7aa..cf1de6f6 100644 --- a/environment.yml +++ b/environment.yml @@ -6,6 +6,6 @@ dependencies: - python=3.7 - ffmpeg - pip: - - -r file:requirements.txt + - -r requirements.txt - pyeddytrackersample - . From 3afb70969397e9f8b25e4fe95a81c10bd1e73915 Mon Sep 17 00:00:00 2001 From: Antoine <36040805+AntSimi@users.noreply.github.com> Date: Sun, 26 Sep 2021 18:27:08 +0200 Subject: [PATCH 2/9] add changelog infos --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f33f15dd..6c37a82f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,9 @@ Changed Fixed ^^^^^ +- Fix bug in convolution(filter), lowest rows was replace by zeros in convolution computation. + Important impact for tiny kernel + Added ^^^^^ From a7ef56e6ec60012e6bf6d036e1d5b3c01a35bede Mon Sep 17 00:00:00 2001 From: Antoine <36040805+AntSimi@users.noreply.github.com> Date: Fri, 17 Sep 2021 20:52:04 +0200 Subject: [PATCH 3/9] Github : Apply test on Ubuntu & Windows for python 37,38,39 --- .github/workflows/python-app.yml | 20 ++++++++++++++------ README.md | 2 ++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 7fc9f385..286d9d6c 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -7,19 +7,27 @@ on: [push, pull_request] jobs: build: - - runs-on: ubuntu-latest + strategy: + matrix: + # os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, windows-latest] + python_version: [3.7, 3.8, 3.9] + name: Run py eddy tracker build tests + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} steps: - uses: actions/checkout@v2 - - name: Set up Python 3.7 + - name: Set up Python ${{ matrix.python_version }} uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: ${{ matrix.python_version }} - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest + pip install flake8 pytest pytest-cov if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Install package run: | @@ -32,4 +40,4 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | - pytest + pytest --cov src/py_eddy_tracker diff --git a/README.md b/README.md index 736bf9b7..e26e15ac 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Method was described in : +[Pegliasco, C., Delepoulle, A., Morrow, R., Faugère, Y., and Dibarboure, G.: META3.1exp : A new Global Mesoscale Eddy Trajectories Atlas derived from altimetry, Earth Syst. Sci. Data Discuss.](https://doi.org/10.5194/essd-2021-300) + [Mason, E., A. Pascual, and J. C. McWilliams, 2014: A new sea surface height–based code for oceanic mesoscale eddy tracking.](https://doi.org/10.1175/JTECH-D-14-00019.1) ### Use case ### From 98161f3f531c23879b6394e07bbbe015c9732de0 Mon Sep 17 00:00:00 2001 From: AntSimi <36040805+AntSimi@users.noreply.github.com> Date: Sat, 25 Sep 2021 16:14:49 +0200 Subject: [PATCH 4/9] Add dummy test on convolution => which detect an index error in original code(corrected) --- requirements.txt | 2 +- src/py_eddy_tracker/dataset/grid.py | 4 +- src/py_eddy_tracker/observations/network.py | 26 +++--------- tests/test_grid.py | 45 ++++++++++++++++----- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/requirements.txt b/requirements.txt index 477cf32d..c4ff9c41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -matplotlib +matplotlib<3.5 netCDF4 numba>=0.53 numpy<1.21 diff --git a/src/py_eddy_tracker/dataset/grid.py b/src/py_eddy_tracker/dataset/grid.py index 6c8e332f..59cda040 100644 --- a/src/py_eddy_tracker/dataset/grid.py +++ b/src/py_eddy_tracker/dataset/grid.py @@ -680,6 +680,7 @@ def eddy_identification( ) z_min, z_max = z_min_p, z_max_p + logger.debug("Levels from %f to %f", z_min, z_max) levels = arange(z_min - z_min % step, z_max - z_max % step + 2 * step, step) # Get x and y values @@ -1404,7 +1405,8 @@ def convolve_filter_with_dynamic_kernel( tmp_matrix = ma.zeros((2 * d_lon + data.shape[0], k_shape[1])) tmp_matrix.mask = ones(tmp_matrix.shape, dtype=bool) # Slice to apply on input data - sl_lat_data = slice(max(0, i - d_lat), min(i + d_lat, data.shape[1])) + # +1 for upper bound, to take in acount this column + sl_lat_data = slice(max(0, i - d_lat), min(i + d_lat + 1, data.shape[1])) # slice to apply on temporary matrix to store input data sl_lat_in = slice( d_lat - (i - sl_lat_data.start), d_lat + (sl_lat_data.stop - i) diff --git a/src/py_eddy_tracker/observations/network.py b/src/py_eddy_tracker/observations/network.py index 0ae80634..90bf6b70 100644 --- a/src/py_eddy_tracker/observations/network.py +++ b/src/py_eddy_tracker/observations/network.py @@ -6,6 +6,8 @@ import time from glob import glob +import netCDF4 +import zarr from numba import njit from numpy import ( arange, @@ -23,9 +25,6 @@ zeros, ) -import netCDF4 -import zarr - from ..dataset.grid import GridCollection from ..generic import build_index, wrap_longitude from ..poly import bbox_intersection, vertice_overlap @@ -680,13 +679,7 @@ def display_timeline( """ self.only_one_network() j = 0 - line_kw = dict( - ls="-", - marker="+", - markersize=6, - zorder=1, - lw=3, - ) + line_kw = dict(ls="-", marker="+", markersize=6, zorder=1, lw=3,) line_kw.update(kwargs) mappables = dict(lines=list()) @@ -919,10 +912,7 @@ def event_map(self, ax, **kwargs): """Add the merging and splitting events to a map""" j = 0 mappables = dict() - symbol_kw = dict( - markersize=10, - color="k", - ) + symbol_kw = dict(markersize=10, color="k",) symbol_kw.update(kwargs) symbol_kw_split = symbol_kw.copy() symbol_kw_split["markersize"] += 4 @@ -951,13 +941,7 @@ def event_map(self, ax, **kwargs): return mappables def scatter( - self, - ax, - name="time", - factor=1, - ref=None, - edgecolor_cycle=None, - **kwargs, + self, ax, name="time", factor=1, ref=None, edgecolor_cycle=None, **kwargs, ): """ This function scatters the path of each network, with the merging and splitting events diff --git a/tests/test_grid.py b/tests/test_grid.py index 2c89550a..759a40e1 100644 --- a/tests/test_grid.py +++ b/tests/test_grid.py @@ -1,5 +1,5 @@ from matplotlib.path import Path -from numpy import array, isnan, ma +from numpy import arange, array, isnan, ma, nan, ones, zeros from pytest import approx from py_eddy_tracker.data import get_demo_path @@ -7,15 +7,7 @@ G = RegularGridDataset(get_demo_path("mask_1_60.nc"), "lon", "lat") X = 0.025 -contour = Path( - ( - (-X, 0), - (X, 0), - (X, X), - (-X, X), - (-X, 0), - ) -) +contour = Path(((-X, 0), (X, 0), (X, X), (-X, X), (-X, 0),)) # contour @@ -85,3 +77,36 @@ def test_interp(): assert g.interp("z", x0, y0) == 1.5 assert g.interp("z", x1, y1) == 2 assert isnan(g.interp("z", x2, y2)) + + +def test_convolution(): + """ + Add some dummy check on convolution filter + """ + # Fake grid + z = ma.array( + arange(12).reshape((-1, 1)) * arange(10).reshape((1, -1)), + mask=zeros((12, 10), dtype="bool"), + dtype="f4", + ) + g = RegularGridDataset.with_array( + coordinates=("x", "y"), + datas=dict(z=z, x=arange(0, 6, 0.5), y=arange(0, 5, 0.5),), + centered=True, + ) + + def kernel_func(lat): + return ones((3, 3)) + + # After transpose we must get same result + d = g.convolve_filter_with_dynamic_kernel("z", kernel_func) + assert (d.T[:9, :9] == d[:9, :9]).all() + # We mask one value and check convolution result + z.mask[2, 2] = True + d = g.convolve_filter_with_dynamic_kernel("z", kernel_func) + assert d[1, 1] == z[:3, :3].sum() / 8 + # Add nan and check only nearest value is contaminate + z[2, 2] = nan + d = g.convolve_filter_with_dynamic_kernel("z", kernel_func) + assert not isnan(d[0, 0]) + assert isnan(d[1:4, 1:4]).all() From e49e1232bf5979791b341386d6dddc9d0b771043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment?= <49512274+ludwigVonKoopa@users.noreply.github.com> Date: Mon, 11 Oct 2021 15:53:44 +0200 Subject: [PATCH 5/9] change get_color() to get_edgecolor --- requirements.txt | 2 +- src/py_eddy_tracker/eddy_feature.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c4ff9c41..477cf32d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -matplotlib<3.5 +matplotlib netCDF4 numba>=0.53 numpy<1.21 diff --git a/src/py_eddy_tracker/eddy_feature.py b/src/py_eddy_tracker/eddy_feature.py index 59a042fe..d2616957 100644 --- a/src/py_eddy_tracker/eddy_feature.py +++ b/src/py_eddy_tracker/eddy_feature.py @@ -646,7 +646,7 @@ def display( paths.append(i.vertices) local_kwargs = kwargs.copy() if "color" not in kwargs: - local_kwargs["color"] = collection.get_color() + local_kwargs["color"] = collection.get_edgecolor() local_kwargs.pop("label", None) elif j != 0: local_kwargs.pop("label", None) From 25bf0ee2f6287ddc68aa704e1d1f6343e03d3c1d Mon Sep 17 00:00:00 2001 From: AntSimi <36040805+AntSimi@users.noreply.github.com> Date: Tue, 19 Oct 2021 22:13:12 +0200 Subject: [PATCH 6/9] Add option to manage issue #111 --- CHANGELOG.rst | 2 ++ src/py_eddy_tracker/dataset/grid.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6c37a82f..75cc2dd0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,8 @@ Fixed Added ^^^^^ +- Allow to replace mask by isnan method to manage nan data instead of masked data + [3.5.0] - 2021-06-22 -------------------- diff --git a/src/py_eddy_tracker/dataset/grid.py b/src/py_eddy_tracker/dataset/grid.py index 59cda040..2bb5b70d 100644 --- a/src/py_eddy_tracker/dataset/grid.py +++ b/src/py_eddy_tracker/dataset/grid.py @@ -258,6 +258,7 @@ class GridDataset(object): "global_attrs", "vars", "contours", + "nan_mask", ) GRAVITY = 9.807 @@ -267,7 +268,14 @@ class GridDataset(object): N = 1 def __init__( - self, filename, x_name, y_name, centered=None, indexs=None, unset=False + self, + filename, + x_name, + y_name, + centered=None, + indexs=None, + unset=False, + nan_masking=False, ): """ :param str filename: Filename to load @@ -276,6 +284,7 @@ def __init__( :param bool,None centered: Allow to know how coordinates could be used with pixel :param dict indexs: A dictionary that sets indexes to use for non-coordinate dimensions :param bool unset: Set to True to create an empty grid object without file + :param bool nan_masking: Set to True to replace data.mask with isnan method result """ self.dimensions = None self.variables_description = None @@ -286,6 +295,7 @@ def __init__( self.y_bounds = None self.x_dim = None self.y_dim = None + self.nan_mask = nan_masking self.centered = centered self.contours = None self.filename = filename @@ -519,6 +529,10 @@ def grid(self, varname, indexs=None): if i_x > i_y: self.variables_description[varname]["infos"]["transpose"] = True self.vars[varname] = self.vars[varname].T + if self.nan_mask: + self.vars[varname] = ma.array( + self.vars[varname], mask=isnan(self.vars[varname]), + ) if not hasattr(self.vars[varname], "mask"): self.vars[varname] = ma.array( self.vars[varname], From c7255e40ec96861c62635f7cc72f72ca7ff98554 Mon Sep 17 00:00:00 2001 From: Antoine <36040805+AntSimi@users.noreply.github.com> Date: Tue, 26 Oct 2021 12:08:04 +0200 Subject: [PATCH 7/9] remove coverage --- .github/workflows/python-app.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 286d9d6c..a6fcceed 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -38,6 +38,3 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest --cov src/py_eddy_tracker From c434372c3f8a05155eb4fdbe4b9996969f3b661e Mon Sep 17 00:00:00 2001 From: Cori Pegliasco Date: Tue, 23 Nov 2021 14:18:27 +0100 Subject: [PATCH 8/9] Resample contours in output form after fitting circles - add parameter presampling_multiplier to evenly over-resample before fitting circles - fit circles to get eddy parameters (radius, area, etc) - resample the contours with the output sampling --- src/py_eddy_tracker/dataset/grid.py | 66 +++++++++++++++++++---------- src/py_eddy_tracker/poly.py | 2 +- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/src/py_eddy_tracker/dataset/grid.py b/src/py_eddy_tracker/dataset/grid.py index 2bb5b70d..5b884b68 100644 --- a/src/py_eddy_tracker/dataset/grid.py +++ b/src/py_eddy_tracker/dataset/grid.py @@ -607,6 +607,7 @@ def eddy_identification( date, step=0.005, shape_error=55, + presampling_multiplier=10, sampling=50, sampling_method="visvalingam", pixel_limit=None, @@ -624,8 +625,10 @@ def eddy_identification( :param datetime.datetime date: Date to be stored in object to date data :param float,int step: Height between two layers in m :param float,int shape_error: Maximal error allowed for outermost contour in % + :param int presampling_multiplier: + Evenly oversample the initial number of points in the contour by nb_pts x presampling_multiplier to fit circles :param int sampling: Number of points to store contours and speed profile - :param str sampling_method: Method to resample, 'uniform' or 'visvalingam' + :param str sampling_method: Method to resample the stored contours, 'uniform' or 'visvalingam' :param (int,int),None pixel_limit: Min and max number of pixels inside the inner and the outermost contour to be considered as an eddy :param float,None precision: Truncate values at the defined precision in m @@ -849,42 +852,59 @@ def eddy_identification( obs.amplitude[:] = amp.amplitude obs.speed_average[:] = max_average_speed obs.num_point_e[:] = contour.lon.shape[0] - xy_e = resample(contour.lon, contour.lat, **out_sampling) - obs.contour_lon_e[:], obs.contour_lat_e[:] = xy_e obs.num_point_s[:] = speed_contour.lon.shape[0] - xy_s = resample( - speed_contour.lon, speed_contour.lat, **out_sampling - ) - obs.contour_lon_s[:], obs.contour_lat_s[:] = xy_s - # FIXME : we use a contour without resampling - # First, get position based on innermost contour - centlon_i, centlat_i, _, _ = _fit_circle_path( - create_vertice(inner_contour.lon, inner_contour.lat) + # Evenly resample contours with nb_pts = nb_pts_original x presampling_multiplier + xy_i = uniform_resample( + inner_contour.lon, + inner_contour.lat, + num_fac=presampling_multiplier + ) + xy_e = uniform_resample( + contour.lon, + contour.lat, + num_fac=presampling_multiplier, ) - # Second, get speed-based radius based on contour of max uavg + xy_s = uniform_resample( + speed_contour.lon, + speed_contour.lat, + num_fac=presampling_multiplier, + ) + + # First, get position of max SSH based on best fit circle with resampled innermost contour + centlon_i, centlat_i, _, _ = _fit_circle_path(create_vertice(*xy_i)) + obs.lon_max[:] = centlon_i + obs.lat_max[:] = centlat_i + + # Second, get speed-based radius, shape error, eddy center, area based on resampled contour of max uavg centlon_s, centlat_s, eddy_radius_s, aerr_s = _fit_circle_path( create_vertice(*xy_s) ) - # Compute again to use resampled contour - _, _, eddy_radius_e, aerr_e = _fit_circle_path( - create_vertice(*xy_e) - ) - obs.radius_s[:] = eddy_radius_s - obs.radius_e[:] = eddy_radius_e - obs.shape_error_e[:] = aerr_e obs.shape_error_s[:] = aerr_s obs.speed_area[:] = poly_area( *coordinates_to_local(*xy_s, lon0=centlon_s, lat0=centlat_s) ) + obs.lon[:] = centlon_s + obs.lat[:] = centlat_s + + # Third, compute effective radius, shape error, area from resampled effective contour + _, _, eddy_radius_e, aerr_e = _fit_circle_path( + create_vertice(*xy_e) + ) + obs.radius_e[:] = eddy_radius_e + obs.shape_error_e[:] = aerr_e obs.effective_area[:] = poly_area( *coordinates_to_local(*xy_e, lon0=centlon_s, lat0=centlat_s) ) - obs.lon[:] = centlon_s - obs.lat[:] = centlat_s - obs.lon_max[:] = centlon_i - obs.lat_max[:] = centlat_i + + # Finally, resample contours with output parameters + xy_e_f = resample(*xy_e, **out_sampling) + xy_s_f = resample(*xy_s, **out_sampling) + + obs.contour_lon_s[:], obs.contour_lat_s[:] = xy_s_f + obs.contour_lon_e[:], obs.contour_lat_e[:] = xy_e_f + if aerr > 99.9 or aerr_s > 99.9: logger.warning( "Strange shape at this step! shape_error : %f, %f", diff --git a/src/py_eddy_tracker/poly.py b/src/py_eddy_tracker/poly.py index 0f0271ee..56fb55e7 100644 --- a/src/py_eddy_tracker/poly.py +++ b/src/py_eddy_tracker/poly.py @@ -86,7 +86,7 @@ def poly_area_vertice(v): @njit(cache=True) def poly_area(x, y): """ - Must be call with local coordinates (in m, to get an area in m²). + Must be called with local coordinates (in m, to get an area in m²). :param array x: :param array y: From 573c4f5e6991a41004fd9bcbaa72d04fe5c0cbfa Mon Sep 17 00:00:00 2001 From: AntSimi <36040805+AntSimi@users.noreply.github.com> Date: Thu, 2 Sep 2021 13:30:02 +0200 Subject: [PATCH 9/9] Add information about animation #102 --- examples/08_tracking_manipulation/pet_track_anim.py | 4 +++- .../pet_track_anim_matplotlib_animation.py | 4 +++- .../08_tracking_manipulation/pet_track_anim.ipynb | 4 ++-- .../pet_track_anim_matplotlib_animation.ipynb | 4 ++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/08_tracking_manipulation/pet_track_anim.py b/examples/08_tracking_manipulation/pet_track_anim.py index 0c18a0ba..94e09ad3 100644 --- a/examples/08_tracking_manipulation/pet_track_anim.py +++ b/examples/08_tracking_manipulation/pet_track_anim.py @@ -2,7 +2,9 @@ Track animation =============== -Run in a terminal this script, which allow to watch eddy evolution +Run in a terminal this script, which allow to watch eddy evolution. + +You could use also *EddyAnim* script to display/save animation. """ import py_eddy_tracker_sample diff --git a/examples/08_tracking_manipulation/pet_track_anim_matplotlib_animation.py b/examples/08_tracking_manipulation/pet_track_anim_matplotlib_animation.py index 6776b47e..59b21527 100644 --- a/examples/08_tracking_manipulation/pet_track_anim_matplotlib_animation.py +++ b/examples/08_tracking_manipulation/pet_track_anim_matplotlib_animation.py @@ -2,7 +2,9 @@ Track animation with standard matplotlib ======================================== -Run in a terminal this script, which allow to watch eddy evolution +Run in a terminal this script, which allow to watch eddy evolution. + +You could use also *EddyAnim* script to display/save animation. """ import re diff --git a/notebooks/python_module/08_tracking_manipulation/pet_track_anim.ipynb b/notebooks/python_module/08_tracking_manipulation/pet_track_anim.ipynb index 65768145..08364d16 100644 --- a/notebooks/python_module/08_tracking_manipulation/pet_track_anim.ipynb +++ b/notebooks/python_module/08_tracking_manipulation/pet_track_anim.ipynb @@ -15,7 +15,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n# Track animation\n\nRun in a terminal this script, which allow to watch eddy evolution\n" + "\nTrack animation\n===============\n\nRun in a terminal this script, which allow to watch eddy evolution.\n\nYou could use also *EddyAnim* script to display/save animation.\n" ] }, { @@ -82,7 +82,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" + "version": "3.7.7" } }, "nbformat": 4, diff --git a/notebooks/python_module/08_tracking_manipulation/pet_track_anim_matplotlib_animation.ipynb b/notebooks/python_module/08_tracking_manipulation/pet_track_anim_matplotlib_animation.ipynb index 6d7fcc2e..bcd4ba74 100644 --- a/notebooks/python_module/08_tracking_manipulation/pet_track_anim_matplotlib_animation.ipynb +++ b/notebooks/python_module/08_tracking_manipulation/pet_track_anim_matplotlib_animation.ipynb @@ -15,7 +15,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n# Track animation with standard matplotlib\n\nRun in a terminal this script, which allow to watch eddy evolution\n" + "\nTrack animation with standard matplotlib\n========================================\n\nRun in a terminal this script, which allow to watch eddy evolution.\n\nYou could use also *EddyAnim* script to display/save animation.\n" ] }, { @@ -93,7 +93,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" + "version": "3.7.7" } }, "nbformat": 4,