diff --git a/Pipfile b/Pipfile index e6cd44fe..50d0a8a2 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,7 @@ responses = "*" [packages] aiocache = {extras = ["redis"],version = "*"} +aiofiles = "*" aiohttp = "*" asyncache = "*" cachetools = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 08d2b350..cb95452c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c79309b1038049f74b5b8a133e931ebbe22162b613809fc39e938d84830bb039" + "sha256": "596c0a497d4f2cfa9e3a3e8b38b2cf018ab3b6d9a26f04a949ced6b025e05f62" }, "pipfile-spec": 6, "requires": { @@ -27,6 +27,14 @@ "index": "pypi", "version": "==0.11.1" }, + "aiofiles": { + "hashes": [ + "sha256:377fdf7815cc611870c59cbd07b68b180841d2a2b79812d8c218be02448c2acb", + "sha256:98e6bcfd1b50f97db4980e182ddd509b7cc35909e903a8fe50d8849e02d815af" + ], + "index": "pypi", + "version": "==0.5.0" + }, "aiohttp": { "hashes": [ "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e", @@ -97,10 +105,10 @@ }, "click": { "hashes": [ - "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", - "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], - "version": "==7.1.1" + "version": "==7.1.2" }, "dataclasses": { "hashes": [ @@ -306,11 +314,11 @@ }, "uvicorn": { "hashes": [ - "sha256:0f58170165c4495f563d8224b2f415a0829af0412baa034d6f777904613087fd", - "sha256:6fdaf8e53bf1b2ddf0fe9ed06079b5348d7d1d87b3365fe2549e6de0d49e631c" + "sha256:a19de5b2f8dec56ec95bbbf3ce334b168efca6a238b2ba7c6cb020e3a42ce71a", + "sha256:aa6db2200cff18f8c550c869fab01f78ca987464d499f654e7aeb166fe0c0616" ], "index": "pypi", - "version": "==0.11.3" + "version": "==0.11.4" }, "uvloop": { "hashes": [ @@ -387,10 +395,10 @@ }, "astroid": { "hashes": [ - "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", - "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" + "sha256:29fa5d46a2404d01c834fcb802a3943685f1fc538eb2a02a161349f5505ac196", + "sha256:2fecea42b20abb1922ed65c7b5be27edfba97211b04b2b6abc6a43549a024ea6" ], - "version": "==2.3.3" + "version": "==2.4.0" }, "async-asgi-testclient": { "hashes": [ @@ -454,10 +462,10 @@ }, "click": { "hashes": [ - "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", - "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], - "version": "==7.1.1" + "version": "==7.1.2" }, "coverage": { "hashes": [ @@ -662,11 +670,11 @@ }, "pylint": { "hashes": [ - "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", - "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" + "sha256:588e114e3f9a1630428c35b7dd1c82c1c93e1b0e78ee312ae4724c5e1a1e0245", + "sha256:bd556ba95a4cf55a1fc0004c00cf4560b1e70598a54a74c6904d933c8f3bd5a8" ], "index": "pypi", - "version": "==2.4.4" + "version": "==2.5.0" }, "pyparsing": { "hashes": [ @@ -809,6 +817,7 @@ "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" ], + "markers": "implementation_name == 'cpython' and python_version < '3.8'", "version": "==1.4.1" }, "urllib3": { @@ -827,9 +836,9 @@ }, "wrapt": { "hashes": [ - "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" ], - "version": "==1.11.2" + "version": "==1.12.1" }, "zipp": { "hashes": [ diff --git a/app/io.py b/app/io.py index 8130c146..3bd443b6 100644 --- a/app/io.py +++ b/app/io.py @@ -1,28 +1,56 @@ """app.io.py""" import json import pathlib -from typing import Dict, Union +from typing import Dict, List, Union + +import aiofiles HERE = pathlib.Path(__file__) DATA = HERE.joinpath("..", "data").resolve() def save( - name: str, content: Union[str, Dict], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs + name: str, content: Union[str, Dict, List], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs ) -> pathlib.Path: """Save content to a file. If content is a dictionary, use json.dumps().""" path = DATA / name - if isinstance(content, dict): + if isinstance(content, (dict, list)): content = json.dumps(content, indent=indent, **json_dumps_kwargs) with open(DATA / name, mode=write_mode) as f_out: f_out.write(content) return path -def load(name: str, **json_kwargs) -> Union[str, Dict]: +def load(name: str, **json_kwargs) -> Union[str, Dict, List]: """Loads content from a file. If file ends with '.json', call json.load() and return a Dictionary.""" path = DATA / name with open(path) as f_in: if path.suffix == ".json": return json.load(f_in, **json_kwargs) return f_in.read() + + +class AIO: + """Asynsc compatible file io operations.""" + + @classmethod + async def save( + cls, name: str, content: Union[str, Dict, List], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs + ): + """Save content to a file. If content is a dictionary, use json.dumps().""" + path = DATA / name + if isinstance(content, (dict, list)): + content = json.dumps(content, indent=indent, **json_dumps_kwargs) + async with aiofiles.open(DATA / name, mode=write_mode) as f_out: + await f_out.write(content) + return path + + @classmethod + async def load(cls, name: str, **json_kwargs) -> Union[str, Dict, List]: + """Loads content from a file. If file ends with '.json', call json.load() and return a Dictionary.""" + path = DATA / name + async with aiofiles.open(path) as f_in: + content = await f_in.read() + if path.suffix == ".json": + content = json.loads(content, **json_kwargs) + return content diff --git a/requirements-dev.txt b/requirements-dev.txt index d7903fdb..c226a0fe 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ -i https://pypi.org/simple appdirs==1.4.3 -astroid==2.3.3 +astroid==2.4.0 async-asgi-testclient==1.4.4 async-generator==1.10 asyncmock==0.4.2 @@ -9,7 +9,7 @@ bandit==1.6.2 black==19.10b0 certifi==2020.4.5.1 chardet==3.0.4 -click==7.1.1 +click==7.1.2 coverage==5.1 coveralls==2.0.0 docopt==0.6.2 @@ -29,7 +29,7 @@ pathspec==0.8.0 pbr==5.4.5 pluggy==0.13.1 py==1.8.1 -pylint==2.4.4 +pylint==2.5.0 pyparsing==2.4.7 pytest-asyncio==0.11.0 pytest-cov==2.8.1 @@ -42,8 +42,8 @@ six==1.14.0 smmap==3.0.2 stevedore==1.32.0 toml==0.10.0 -typed-ast==1.4.1 +typed-ast==1.4.1 ; implementation_name == 'cpython' and python_version < '3.8' urllib3==1.25.9 wcwidth==0.1.9 -wrapt==1.11.2 +wrapt==1.12.1 zipp==3.1.0 diff --git a/requirements.txt b/requirements.txt index 86ec8017..9e061190 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -i https://pypi.org/simple aiocache[redis]==0.11.1 +aiofiles==0.5.0 aiohttp==3.6.2 aioredis==1.3.1 async-timeout==3.0.1 @@ -8,7 +9,7 @@ attrs==19.3.0 cachetools==4.1.0 certifi==2020.4.5.1 chardet==3.0.4 -click==7.1.1 +click==7.1.2 dataclasses==0.6 ; python_version < '3.7' fastapi==0.54.1 gunicorn==20.0.4 @@ -25,7 +26,7 @@ requests==2.23.0 six==1.14.0 starlette==0.13.2 urllib3==1.25.9 -uvicorn==0.11.3 +uvicorn==0.11.4 uvloop==0.14.0 ; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy' websockets==8.1 yarl==1.4.2 diff --git a/tests/test_io.py b/tests/test_io.py index 83639cc9..c5d16c3a 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -5,8 +5,7 @@ import app.io - -@pytest.mark.parametrize( +IO_PARAMS = ( "name, content, kwargs", [ ("test_file.txt", string.ascii_lowercase, {}), @@ -14,6 +13,9 @@ ("test_custom_json.json", {"z": -1, "b": 1, "y": -2, "a": 0}, {"indent": 4, "sort_keys": True}), ], ) + + +@pytest.mark.parametrize(*IO_PARAMS) def test_save(tmp_path, name, content, kwargs): test_path = tmp_path / name assert not test_path.exists() @@ -23,17 +25,32 @@ def test_save(tmp_path, name, content, kwargs): assert test_path.exists() -@pytest.mark.parametrize( - "name, content, kwargs", - [ - ("test_file.txt", string.ascii_lowercase, {}), - ("test_json_file.json", {"a": 0, "b": 1, "c": 2}, {}), - ("test_custom_json.json", {"z": -1, "b": 1, "y": -2, "a": 0}, {"indent": 4, "sort_keys": True}), - ], -) +@pytest.mark.parametrize(*IO_PARAMS) def test_round_trip(tmp_path, name, content, kwargs): test_path = tmp_path / name assert not test_path.exists() app.io.save(test_path, content, **kwargs) assert app.io.load(test_path) == content + + +@pytest.mark.asyncio +@pytest.mark.parametrize(*IO_PARAMS) +async def test_async_save(tmp_path, name, content, kwargs): + test_path = tmp_path / name + assert not test_path.exists() + + result = await app.io.AIO.save(test_path, content, **kwargs) + assert result == test_path + assert test_path.exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(*IO_PARAMS) +async def test_async_round_trip(tmp_path, name, content, kwargs): + test_path = tmp_path / name + assert not test_path.exists() + + await app.io.AIO.save(test_path, content, **kwargs) + load_results = await app.io.AIO.load(test_path) + assert load_results == content