read netcdf file in memory

Author: AntSimi

src/py_eddy_tracker/data/__init__.py

@@ -1,5 +1,27 @@
 from os import path
-
+import requests
+import io
+import tarfile
+import lzma
 
 def get_path(name):
     return path.join(path.dirname(__file__), name)
+
+
+def get_remote_sample(path):
+    # url = (
+    # f"https://github.com/AntSimi/py-eddy-tracker-sample-id/raw/master/{path}.tar.xz"
+    # )
+    # content = requests.get(url).content
+    url = f"/home/toto/dev/py-eddy-tracker-sample-id/{path}.tar.xz"
+    content = open(url, "rb").read()
+
+    # Tar module could manage lzma tar, but it will apply un compress for each extractfile
+    tar = tarfile.open(mode="r", fileobj=io.BytesIO(lzma.decompress(content)))
+    # tar = tarfile.open(mode="r:xz", fileobj=io.BytesIO(content))
+    files_content = list()
+    for item in tar:
+        content = tar.extractfile(item)
+        content.filename = item.name
+        files_content.append(content)
+    return files_content

src/py_eddy_tracker/observations/observation.py

@@ -58,6 +58,7 @@
 from pint import UnitRegistry
 from pint.errors import UndefinedUnitError
 from tokenize import TokenError
+from tarfile import ExFileObject
 from matplotlib.path import Path as BasePath
 from .. import VAR_DESCR, VAR_DESCR_inv
 from ..generic import (
@@ -431,8 +432,11 @@ def zarr_dimension(filename):
 
     @classmethod
     def load_file(cls, filename, **kwargs):
-        end = b".zarr" if isinstance(filename, bytes) else ".zarr"
-        if filename.endswith(end):
+        filename_ = (
+            filename.filename if isinstance(filename, ExFileObject) else filename
+        )
+        end = b".zarr" if isinstance(filename_, bytes) else ".zarr"
+        if filename_.endswith(end):
             return cls.load_from_zarr(filename, **kwargs)
         else:
             return cls.load_from_netcdf(filename, **kwargs)
@@ -539,9 +543,14 @@ def load_from_netcdf(
         cls, filename, raw_data=False, remove_vars=None, include_vars=None
     ):
         array_dim = "NbSample"
-        if not isinstance(filename, str):
+        if isinstance(filename, bytes):
             filename = filename.astype(str)
-        with Dataset(filename) as h_nc:
+        if isinstance(filename, ExFileObject):
+            filename.seek(0)
+            args, kwargs = ("in-mem-file",), dict(memory=filename.read())
+        else:
+            args, kwargs = (filename,), dict()
+        with Dataset(*args, **kwargs) as h_nc:
             var_list = list(h_nc.variables.keys())
             if include_vars is not None:
                 var_list = [i for i in var_list if i in include_vars]