Use shared Redis cache for jhu data (#306)

* add & log a data_id string for each cache
* dev mode should run with debug log level
* log missing country code at sub debug level
* use pydantic for config managment
* add redis config settings
* add aiocache with redis support
* use memory cache
* use shared cache for jhu data
* cleanup
* fix url type
* add aiofiles
* add async save/read
* update tests
* update dependencies
* change pylint config to pyproject.toml
* cache jhu data (locally) for 30 minutes

Author: Kilo59

.env.example

@@ -1,2 +1,3 @@
 # Port to serve app on.
-PORT = 5000
+PORT = 5000
+LOCAL_REDIS_URL = redis://localhost:6379

Pipfile

@@ -20,15 +20,17 @@ pytest-cov = "*"
 responses = "*"
 
 [packages]
+aiocache = {extras = ["redis"],version = "*"}
+aiofiles = "*"
 aiohttp = "*"
 asyncache = "*"
 cachetools = "*"
 dataclasses = {version = "*",markers = "python_version<'3.7'"}
 fastapi = "*"
 gunicorn = "*"
 idna_ssl = {version = "*",markers = "python_version<'3.7'"}
+pydantic = {extras = ["dotenv"],version = "*"}
 python-dateutil = "*"
-python-dotenv = "*"
 requests = "*"
 uvicorn = "*"
 

Pipfile.lock

@@ -0,0 +1,52 @@
+"""app.caches.py"""
+import functools
+import logging
+from typing import Union
+
+import aiocache
+
+from .config import get_settings
+
+LOGGER = logging.getLogger(name="app.caches")
+
+SETTINGS = get_settings()
+
+if SETTINGS.rediscloud_url:
+    REDIS_URL = SETTINGS.rediscloud_url
+    LOGGER.info("Using Rediscloud")
+else:
+    REDIS_URL = SETTINGS.local_redis_url
+    LOGGER.info("Using Local Redis")
+
+
+@functools.lru_cache()
+def get_cache(namespace) -> Union[aiocache.RedisCache, aiocache.SimpleMemoryCache]:
+    """Retunr """
+    if REDIS_URL:
+        LOGGER.info("using RedisCache")
+        return aiocache.RedisCache(
+            endpoint=REDIS_URL.host,
+            port=REDIS_URL.port,
+            password=REDIS_URL.password,
+            namespace=namespace,
+            create_connection_timeout=5,
+        )
+    LOGGER.info("using SimpleMemoryCache")
+    return aiocache.SimpleMemoryCache(namespace=namespace)
+
+
+async def check_cache(data_id: str, namespace: str = None):
+    """Check the data of a cache given an id."""
+    cache = get_cache(namespace)
+    result = await cache.get(data_id, None)
+    LOGGER.info(f"{data_id} cache pulled")
+    await cache.close()
+    return result
+
+
+async def load_cache(data_id: str, data, namespace: str = None, cache_life: int = 3600):
+    """Load data into the cache."""
+    cache = get_cache(namespace)
+    await cache.set(data_id, data, ttl=cache_life)
+    LOGGER.info(f"{data_id} cache loaded")
+    await cache.close()

app/caches.py

@@ -0,0 +1,29 @@
+"""app.config.py"""
+import functools
+import logging
+
+from pydantic import AnyUrl, BaseSettings
+
+CFG_LOGGER = logging.getLogger("app.config")
+
+
+class _Settings(BaseSettings):
+    port: int = 5000
+    rediscloud_url: AnyUrl = None
+    local_redis_url: AnyUrl = None
+
+
+@functools.lru_cache()
+def get_settings(**kwargs) -> BaseSettings:
+    """
+    Read settings from the environment or `.env` file.
+    https://pydantic-docs.helpmanual.io/usage/settings/#dotenv-env-support
+
+    Usage:
+        import app.config
+
+        settings = app.config.get_settings(_env_file="")
+        port_number = settings.port
+    """
+    CFG_LOGGER.info("Loading Config settings from Environment ...")
+    return _Settings(**kwargs)