diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 00000000..f772ce8c --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,10 @@ +version = 1 + +test_patterns = ["tests/**"] + +[[analyzers]] +name = "python" +enabled = true + + [analyzers.meta] + runtime_version = "3.x.x" diff --git a/.gitignore b/.gitignore index 9c41818c..ab6f17ff 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ htmlcov/ nosetests.xml coverage.xml *,cover +locustfile.py # Translations *.mo diff --git a/Pipfile b/Pipfile index 584e4e10..57ef4ae7 100644 --- a/Pipfile +++ b/Pipfile @@ -33,6 +33,7 @@ pydantic = {extras = ["dotenv"],version = "*"} python-dateutil = "*" requests = "*" scout-apm = "*" +sentry-sdk = "*" uvicorn = "*" [requires] diff --git a/Pipfile.lock b/Pipfile.lock index b1551678..7cdfd8a1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1a448c6a753787b0b71c702c6b3baa4063b468afef4847b413459801d6e56592" + "sha256": "dfa074e03982c046ee011817d151762138abfc1f13ae4e67700233599af18c3e" }, "pipfile-spec": 6, "requires": { @@ -411,6 +411,14 @@ "index": "pypi", "version": "==2.14.1" }, + "sentry-sdk": { + "hashes": [ + "sha256:23808d571d2461a4ce3784ec12bbee5bdb8c026c143fe79d36cef8a6d653e71f", + "sha256:bb90a4e19c7233a580715fc986cc44be2c48fc10b31e71580a2037e1c94b6950" + ], + "index": "pypi", + "version": "==0.14.3" + }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", @@ -525,10 +533,10 @@ }, "astroid": { "hashes": [ - "sha256:29fa5d46a2404d01c834fcb802a3943685f1fc538eb2a02a161349f5505ac196", - "sha256:2fecea42b20abb1922ed65c7b5be27edfba97211b04b2b6abc6a43549a024ea6" + "sha256:4c17cea3e592c21b6e222f673868961bad77e1f985cb1694ed077475a89229c1", + "sha256:d8506842a3faf734b81599c8b98dcc423de863adcc1999248480b18bd31a0f38" ], - "version": "==2.4.0" + "version": "==2.4.1" }, "async-asgi-testclient": { "hashes": [ @@ -649,17 +657,17 @@ }, "gitdb": { "hashes": [ - "sha256:6f0ecd46f99bb4874e5678d628c3a198e2b4ef38daea2756a2bfd8df7dd5c1a5", - "sha256:ba1132c0912e8c917aa8aa990bee26315064c7b7f171ceaaac0afeb1dc656c6a" + "sha256:91f36bfb1ab7949b3b40e23736db18231bf7593edada2ba5c3a174a7b23657ac", + "sha256:c9e1f2d0db7ddb9a704c2a0217be31214e91a4fe1dea1efad19ae42ba0c285c9" ], - "version": "==4.0.4" + "version": "==4.0.5" }, "gitpython": { "hashes": [ - "sha256:6d4f10e2aaad1864bb0f17ec06a2c2831534140e5883c350d58b4e85189dab74", - "sha256:71b8dad7409efbdae4930f2b0b646aaeccce292484ffa0bc74f1195582578b3d" + "sha256:864a47472548f3ba716ca202e034c1900f197c0fb3a08f641c20c3cafd15ed94", + "sha256:da3b2cf819974789da34f95ac218ef99f515a928685db141327c09b73dd69c09" ], - "version": "==3.1.1" + "version": "==3.1.2" }, "idna": { "hashes": [ @@ -800,11 +808,11 @@ }, "pylint": { "hashes": [ - "sha256:588e114e3f9a1630428c35b7dd1c82c1c93e1b0e78ee312ae4724c5e1a1e0245", - "sha256:bd556ba95a4cf55a1fc0004c00cf4560b1e70598a54a74c6904d933c8f3bd5a8" + "sha256:b95e31850f3af163c2283ed40432f053acbc8fc6eba6a069cb518d9dbf71848c", + "sha256:dd506acce0427e9e08fb87274bcaa953d38b50a58207170dbf5b36cf3e16957b" ], "index": "pypi", - "version": "==2.5.0" + "version": "==2.5.2" }, "pyparsing": { "hashes": [ @@ -815,19 +823,18 @@ }, "pytest": { "hashes": [ - "sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172", - "sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970" + "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3", + "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698" ], "index": "pypi", - "version": "==5.4.1" + "version": "==5.4.2" }, "pytest-asyncio": { "hashes": [ - "sha256:6096d101a1ae350d971df05e25f4a8b4d3cd13ffb1b32e42d902ac49670d2bfa", - "sha256:c54866f3cf5dd2063992ba2c34784edae11d3ed19e006d220a3cf0bfc4191fcb" + "sha256:475bd2f3dc0bc11d2463656b3cbaafdbec5a47b47508ea0b329ee693040eebd2" ], "index": "pypi", - "version": "==0.11.0" + "version": "==0.12.0" }, "pytest-cov": { "hashes": [ @@ -855,29 +862,29 @@ }, "regex": { "hashes": [ - "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b", - "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8", - "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3", - "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e", - "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683", - "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1", - "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142", - "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3", - "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468", - "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e", - "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3", - "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a", - "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f", - "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6", - "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156", - "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b", - "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db", - "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd", - "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a", - "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948", - "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89" - ], - "version": "==2020.4.4" + "sha256:021a0ae4d2baeeb60a3014805a2096cb329bd6d9f30669b7ad0da51a9cb73349", + "sha256:04d6e948ef34d3eac133bedc0098364a9e635a7914f050edb61272d2ddae3608", + "sha256:099568b372bda492be09c4f291b398475587d49937c659824f891182df728cdf", + "sha256:0ff50843535593ee93acab662663cb2f52af8e31c3f525f630f1dc6156247938", + "sha256:1b17bf37c2aefc4cac8436971fe6ee52542ae4225cfc7762017f7e97a63ca998", + "sha256:1e2255ae938a36e9bd7db3b93618796d90c07e5f64dd6a6750c55f51f8b76918", + "sha256:2bc6a17a7fa8afd33c02d51b6f417fc271538990297167f68a98cae1c9e5c945", + "sha256:3ab5e41c4ed7cd4fa426c50add2892eb0f04ae4e73162155cd668257d02259dd", + "sha256:3b059e2476b327b9794c792c855aa05531a3f3044737e455d283c7539bd7534d", + "sha256:4df91094ced6f53e71f695c909d9bad1cca8761d96fd9f23db12245b5521136e", + "sha256:5493a02c1882d2acaaf17be81a3b65408ff541c922bfd002535c5f148aa29f74", + "sha256:5b741ecc3ad3e463d2ba32dce512b412c319993c1bb3d999be49e6092a769fb2", + "sha256:652ab4836cd5531d64a34403c00ada4077bb91112e8bcdae933e2eae232cf4a8", + "sha256:669a8d46764a09f198f2e91fc0d5acdac8e6b620376757a04682846ae28879c4", + "sha256:73a10404867b835f1b8a64253e4621908f0d71150eb4e97ab2e7e441b53e9451", + "sha256:7ce4a213a96d6c25eeae2f7d60d4dad89ac2b8134ec3e69db9bc522e2c0f9388", + "sha256:8127ca2bf9539d6a64d03686fd9e789e8c194fc19af49b69b081f8c7e6ecb1bc", + "sha256:b5b5b2e95f761a88d4c93691716ce01dc55f288a153face1654f868a8034f494", + "sha256:b7c9f65524ff06bf70c945cd8d8d1fd90853e27ccf86026af2afb4d9a63d06b1", + "sha256:f7f2f4226db6acd1da228adf433c5c3792858474e49d80668ea82ac87cf74a03", + "sha256:fa09da4af4e5b15c0e8b4986a083f3fd159302ea115a6cc0649cd163435538b8" + ], + "version": "==2020.5.7" }, "requests": { "hashes": [ @@ -904,10 +911,10 @@ }, "smmap": { "hashes": [ - "sha256:52ea78b3e708d2c2b0cfe93b6fc3fbeec53db913345c26be6ed84c11ed8bebc1", - "sha256:b46d3fc69ba5f367df96d91f8271e8ad667a198d5a28e215a6c3d9acd133a911" + "sha256:54c44c197c819d5ef1991799a7e30b662d1e520f2ac75c9efbeb54a742214cf4", + "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24" ], - "version": "==3.0.2" + "version": "==3.0.4" }, "stevedore": { "hashes": [ diff --git a/Procfile b/Procfile index 094ebc0a..aa7dd193 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: gunicorn app.main:APP -w 4 -k uvicorn.workers.UvicornWorker +web: gunicorn app.main:APP -w 2 --max-requests 1000 --max-requests-jitter 400 -k uvicorn.workers.UvicornWorker diff --git a/app/config.py b/app/config.py index ab60d42b..377ebc13 100644 --- a/app/config.py +++ b/app/config.py @@ -13,6 +13,8 @@ class _Settings(BaseSettings): local_redis_url: AnyUrl = None # Scout APM scout_name: str = None + # Sentry + sentry_dsn: str = None @functools.lru_cache() diff --git a/app/data/__init__.py b/app/data/__init__.py index 265bf3d3..60a75dac 100644 --- a/app/data/__init__.py +++ b/app/data/__init__.py @@ -4,7 +4,11 @@ from ..services.location.nyt import NYTLocationService # Mapping of services to data-sources. -DATA_SOURCES = {"jhu": JhuLocationService(), "csbs": CSBSLocationService(), "nyt": NYTLocationService()} +DATA_SOURCES = { + "jhu": JhuLocationService(), + "csbs": CSBSLocationService(), + "nyt": NYTLocationService(), +} def data_source(source): diff --git a/app/io.py b/app/io.py index 3bd443b6..2a563b15 100644 --- a/app/io.py +++ b/app/io.py @@ -10,7 +10,11 @@ def save( - name: str, content: Union[str, Dict, List], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs + name: str, + content: Union[str, Dict, List], + write_mode: str = "w", + indent: int = 2, + **json_dumps_kwargs, ) -> pathlib.Path: """Save content to a file. If content is a dictionary, use json.dumps().""" path = DATA / name @@ -35,7 +39,12 @@ class AIO: @classmethod async def save( - cls, name: str, content: Union[str, Dict, List], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs + cls, + name: str, + content: Union[str, Dict, List], + write_mode: str = "w", + indent: int = 2, + **json_dumps_kwargs, ): """Save content to a file. If content is a dictionary, use json.dumps().""" path = DATA / name diff --git a/app/location/__init__.py b/app/location/__init__.py index d12f28c3..1da5e9e5 100644 --- a/app/location/__init__.py +++ b/app/location/__init__.py @@ -11,7 +11,7 @@ class Location: # pylint: disable=too-many-instance-attributes """ def __init__( - self, id, country, province, coordinates, last_updated, confirmed, deaths, recovered + self, id, country, province, coordinates, last_updated, confirmed, deaths, recovered, ): # pylint: disable=too-many-arguments # General info. self.id = id @@ -66,7 +66,11 @@ def serialize(self): # Last updated. "last_updated": self.last_updated, # Latest data (statistics). - "latest": {"confirmed": self.confirmed, "deaths": self.deaths, "recovered": self.recovered}, + "latest": { + "confirmed": self.confirmed, + "deaths": self.deaths, + "recovered": self.recovered, + }, } diff --git a/app/main.py b/app/main.py index f8ed1b41..b43b4aae 100644 --- a/app/main.py +++ b/app/main.py @@ -4,12 +4,14 @@ import logging import pydantic +import sentry_sdk import uvicorn from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import JSONResponse from scout_apm.async_.starlette import ScoutMiddleware +from sentry_sdk.integrations.asgi import SentryAsgiMiddleware from .config import get_settings from .data import data_source @@ -23,6 +25,9 @@ SETTINGS = get_settings() +if SETTINGS.sentry_dsn: # pragma: no cover + sentry_sdk.init(dsn=SETTINGS.sentry_dsn) + APP = FastAPI( title="Coronavirus Tracker", description=( @@ -47,9 +52,18 @@ else: LOGGER.debug("No SCOUT_NAME config") +# Sentry Error Tracking +if SETTINGS.sentry_dsn: # pragma: no cover + LOGGER.info("Adding Sentry middleware") + APP.add_middleware(SentryAsgiMiddleware) + # Enable CORS. APP.add_middleware( - CORSMiddleware, allow_credentials=True, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], + CORSMiddleware, + allow_credentials=True, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], ) APP.add_middleware(GZipMiddleware, minimum_size=1000) diff --git a/app/models.py b/app/models.py index 8875a92c..8acf2243 100644 --- a/app/models.py +++ b/app/models.py @@ -1,7 +1,7 @@ """app.models.py""" from typing import Dict, List -from pydantic import BaseModel +from pydantic import BaseModel, validator class Latest(BaseModel): @@ -27,8 +27,25 @@ class Timeline(BaseModel): Timeline model. """ - latest: int - timeline: Dict[str, int] = {} + timeline: Dict[str, int] = dict() + + @validator("timeline") + @classmethod + def sort_timeline(cls, value): + """Sort the timeline history before inserting into the model""" + return dict(sorted(value.items())) + + @property + def latest(self): + """Get latest available history value.""" + return list(self.timeline.values())[-1] if self.timeline else 0 + + def serialize(self): + """ + Serialize the model into dict + TODO: override dict() instead of using serialize + """ + return {**self.dict(), "latest": self.latest} class Timelines(BaseModel): diff --git a/app/routers/v1.py b/app/routers/v1.py index 662514a0..517bc625 100644 --- a/app/routers/v1.py +++ b/app/routers/v1.py @@ -19,7 +19,11 @@ async def all_categories(): "deaths": deaths, "recovered": recovered, # Latest. - "latest": {"confirmed": confirmed["latest"], "deaths": deaths["latest"], "recovered": recovered["latest"],}, + "latest": { + "confirmed": confirmed["latest"], + "deaths": deaths["latest"], + "recovered": recovered["latest"], + }, } diff --git a/app/routers/v2.py b/app/routers/v2.py index de5a5312..fe9d2475 100644 --- a/app/routers/v2.py +++ b/app/routers/v2.py @@ -65,11 +65,17 @@ async def get_locations( # Do filtering. try: - locations = [location for location in locations if str(getattr(location, key)).lower() == str(value)] + locations = [ + location + for location in locations + if str(getattr(location, key)).lower() == str(value) + ] except AttributeError: pass if not locations: - raise HTTPException(404, detail=f"Source `{source}` does not have the desired location data.") + raise HTTPException( + 404, detail=f"Source `{source}` does not have the desired location data.", + ) # Return final serialized data. return { @@ -84,7 +90,9 @@ async def get_locations( # pylint: disable=invalid-name @V2.get("/locations/{id}", response_model=LocationResponse) -async def get_location_by_id(request: Request, id: int, source: Sources = "jhu", timelines: bool = True): +async def get_location_by_id( + request: Request, id: int, source: Sources = "jhu", timelines: bool = True +): """ Getting specific location by id. """ diff --git a/app/services/location/csbs.py b/app/services/location/csbs.py index 68bdb01c..444ebad6 100644 --- a/app/services/location/csbs.py +++ b/app/services/location/csbs.py @@ -6,6 +6,7 @@ from asyncache import cached from cachetools import TTLCache +from ...caches import check_cache, load_cache from ...coordinates import Coordinates from ...location.csbs import CSBSLocation from ...utils import httputils @@ -34,7 +35,7 @@ async def get(self, loc_id): # pylint: disable=arguments-differ BASE_URL = "https://facts.csbs.org/covid-19/covid19_county.csv" -@cached(cache=TTLCache(maxsize=1, ttl=3600)) +@cached(cache=TTLCache(maxsize=1, ttl=1800)) async def get_locations(): """ Retrieves county locations; locations are cached for 1 hour @@ -44,48 +45,58 @@ async def get_locations(): """ data_id = "csbs.locations" LOGGER.info(f"{data_id} Requesting data...") - async with httputils.CLIENT_SESSION.get(BASE_URL) as response: - text = await response.text() - - LOGGER.debug(f"{data_id} Data received") - - data = list(csv.DictReader(text.splitlines())) - LOGGER.debug(f"{data_id} CSV parsed") - - locations = [] - - for i, item in enumerate(data): - # General info. - state = item["State Name"] - county = item["County Name"] - - # Ensure country is specified. - if county in {"Unassigned", "Unknown"}: - continue - - # Coordinates. - coordinates = Coordinates(item["Latitude"], item["Longitude"]) # pylint: disable=unused-variable - - # Date string without "EDT" at end. - last_update = " ".join(item["Last Update"].split(" ")[0:2]) - - # Append to locations. - locations.append( - CSBSLocation( - # General info. - i, - state, - county, - # Coordinates. - Coordinates(item["Latitude"], item["Longitude"]), - # Last update (parse as ISO). - datetime.strptime(last_update, "%Y-%m-%d %H:%M").isoformat() + "Z", - # Statistics. - int(item["Confirmed"] or 0), - int(item["Death"] or 0), + # check shared cache + cache_results = await check_cache(data_id) + if cache_results: + LOGGER.info(f"{data_id} using shared cache results") + locations = cache_results + else: + LOGGER.info(f"{data_id} shared cache empty") + async with httputils.CLIENT_SESSION.get(BASE_URL) as response: + text = await response.text() + + LOGGER.debug(f"{data_id} Data received") + + data = list(csv.DictReader(text.splitlines())) + LOGGER.debug(f"{data_id} CSV parsed") + + locations = [] + + for i, item in enumerate(data): + # General info. + state = item["State Name"] + county = item["County Name"] + + # Ensure country is specified. + if county in {"Unassigned", "Unknown"}: + continue + + # Date string without "EDT" at end. + last_update = " ".join(item["Last Update"].split(" ")[0:2]) + + # Append to locations. + locations.append( + CSBSLocation( + # General info. + i, + state, + county, + # Coordinates. + Coordinates(item["Latitude"], item["Longitude"]), + # Last update (parse as ISO). + datetime.strptime(last_update, "%Y-%m-%d %H:%M").isoformat() + "Z", + # Statistics. + int(item["Confirmed"] or 0), + int(item["Death"] or 0), + ) ) - ) - LOGGER.info(f"{data_id} Data normalized") + LOGGER.info(f"{data_id} Data normalized") + # save the results to distributed cache + # TODO: fix json serialization + try: + await load_cache(data_id, locations) + except TypeError as type_err: + LOGGER.error(type_err) # Return the locations. return locations diff --git a/app/services/location/jhu.py b/app/services/location/jhu.py index 1a11e8ac..fcb1756d 100644 --- a/app/services/location/jhu.py +++ b/app/services/location/jhu.py @@ -11,7 +11,7 @@ from ...caches import check_cache, load_cache from ...coordinates import Coordinates from ...location import TimelinedLocation -from ...timeline import Timeline +from ...models import Timeline from ...utils import countries from ...utils import date as date_util from ...utils import httputils @@ -41,12 +41,10 @@ async def get(self, loc_id): # pylint: disable=arguments-differ # Base URL for fetching category. -BASE_URL = ( - "https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/" -) +BASE_URL = "https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/" -@cached(cache=TTLCache(maxsize=128, ttl=1800)) +@cached(cache=TTLCache(maxsize=4, ttl=1800)) async def get_category(category): """ Retrieves the data for the provided category. The data is cached for 30 minutes locally, 1 hour via shared Redis. @@ -129,7 +127,7 @@ async def get_category(category): return results -@cached(cache=TTLCache(maxsize=1024, ttl=1800)) +@cached(cache=TTLCache(maxsize=1, ttl=1800)) async def get_locations(): """ Retrieves the locations from the categories. The locations are cached for 1 hour. @@ -177,18 +175,18 @@ async def get_locations(): # Timelines (parse dates as ISO). { "confirmed": Timeline( - { + timeline={ datetime.strptime(date, "%m/%d/%y").isoformat() + "Z": amount for date, amount in timelines["confirmed"].items() } ), "deaths": Timeline( - { + timeline={ datetime.strptime(date, "%m/%d/%y").isoformat() + "Z": amount for date, amount in timelines["deaths"].items() } ), - "recovered": Timeline({}), + "recovered": Timeline(), }, ) ) diff --git a/app/services/location/nyt.py b/app/services/location/nyt.py index 8b70c5cc..1f25ec34 100644 --- a/app/services/location/nyt.py +++ b/app/services/location/nyt.py @@ -6,9 +6,10 @@ from asyncache import cached from cachetools import TTLCache +from ...caches import check_cache, load_cache from ...coordinates import Coordinates from ...location.nyt import NYTLocation -from ...timeline import Timeline +from ...models import Timeline from ...utils import httputils from . import LocationService @@ -66,7 +67,7 @@ def get_grouped_locations_dict(data): return grouped_locations -@cached(cache=TTLCache(maxsize=128, ttl=3600)) +@cached(cache=TTLCache(maxsize=1, ttl=1800)) async def get_locations(): """ Returns a list containing parsed NYT data by US county. The data is cached for 1 hour. @@ -77,55 +78,68 @@ async def get_locations(): data_id = "nyt.locations" # Request the data. LOGGER.info(f"{data_id} Requesting data...") - async with httputils.CLIENT_SESSION.get(BASE_URL) as response: - text = await response.text() - - LOGGER.debug(f"{data_id} Data received") - - # Parse the CSV. - data = list(csv.DictReader(text.splitlines())) - LOGGER.debug(f"{data_id} CSV parsed") - - # Group together locations (NYT data ordered by dates not location). - grouped_locations = get_grouped_locations_dict(data) - - # The normalized locations. - locations = [] - - for idx, (county_state, histories) in enumerate(grouped_locations.items()): - # Make location history for confirmed and deaths from dates. - # List is tuples of (date, amount) in order of increasing dates. - confirmed_list = histories["confirmed"] - confirmed_history = {date: int(amount or 0) for date, amount in confirmed_list} - - deaths_list = histories["deaths"] - deaths_history = {date: int(amount or 0) for date, amount in deaths_list} - - # Normalize the item and append to locations. - locations.append( - NYTLocation( - id=idx, - state=county_state[1], - county=county_state[0], - coordinates=Coordinates(None, None), # NYT does not provide coordinates - last_updated=datetime.utcnow().isoformat() + "Z", # since last request - timelines={ - "confirmed": Timeline( - { - datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount - for date, amount in confirmed_history.items() - } - ), - "deaths": Timeline( - { - datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount - for date, amount in deaths_history.items() - } - ), - "recovered": Timeline({}), - }, + # check shared cache + cache_results = await check_cache(data_id) + if cache_results: + LOGGER.info(f"{data_id} using shared cache results") + locations = cache_results + else: + LOGGER.info(f"{data_id} shared cache empty") + async with httputils.CLIENT_SESSION.get(BASE_URL) as response: + text = await response.text() + + LOGGER.debug(f"{data_id} Data received") + + # Parse the CSV. + data = list(csv.DictReader(text.splitlines())) + LOGGER.debug(f"{data_id} CSV parsed") + + # Group together locations (NYT data ordered by dates not location). + grouped_locations = get_grouped_locations_dict(data) + + # The normalized locations. + locations = [] + + for idx, (county_state, histories) in enumerate(grouped_locations.items()): + # Make location history for confirmed and deaths from dates. + # List is tuples of (date, amount) in order of increasing dates. + confirmed_list = histories["confirmed"] + confirmed_history = {date: int(amount or 0) for date, amount in confirmed_list} + + deaths_list = histories["deaths"] + deaths_history = {date: int(amount or 0) for date, amount in deaths_list} + + # Normalize the item and append to locations. + locations.append( + NYTLocation( + id=idx, + state=county_state[1], + county=county_state[0], + coordinates=Coordinates(None, None), # NYT does not provide coordinates + last_updated=datetime.utcnow().isoformat() + "Z", # since last request + timelines={ + "confirmed": Timeline( + timeline={ + datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount + for date, amount in confirmed_history.items() + } + ), + "deaths": Timeline( + timeline={ + datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount + for date, amount in deaths_history.items() + } + ), + "recovered": Timeline(), + }, + ) ) - ) - LOGGER.info(f"{data_id} Data normalized") + LOGGER.info(f"{data_id} Data normalized") + # save the results to distributed cache + # TODO: fix json serialization + try: + await load_cache(data_id, locations) + except TypeError as type_err: + LOGGER.error(type_err) return locations diff --git a/app/timeline.py b/app/timeline.py deleted file mode 100644 index 0b40d496..00000000 --- a/app/timeline.py +++ /dev/null @@ -1,42 +0,0 @@ -"""app.timeline.py""" -from collections import OrderedDict - - -class Timeline: - """ - Timeline with history of data. - """ - - def __init__(self, history=None): - self.__timeline = history if history else {} - - @property - def timeline(self): - """ - Gets the history sorted by date (key). - """ - return OrderedDict(sorted(self.__timeline.items())) - - @property - def latest(self): - """ - Gets the latest available history value. - """ - # Get values in a list. - values = list(self.timeline.values()) - - # Last item is the latest. - if values: - return values[-1] or 0 - - # Fallback value of 0. - return 0 - - def serialize(self): - """ - Serializes the timeline into a dict. - - :returns: The serialized timeline. - :rtype: dict - """ - return {"latest": self.latest, "timeline": self.timeline} diff --git a/app/utils/populations.py b/app/utils/populations.py index 24f0fa4e..c02f15a9 100644 --- a/app/utils/populations.py +++ b/app/utils/populations.py @@ -28,7 +28,9 @@ def fetch_populations(save=False): # Fetch the countries. try: - countries = requests.get(GEONAMES_URL, params={"username": "dperic"}, timeout=1.25).json()["geonames"] + countries = requests.get(GEONAMES_URL, params={"username": "dperic"}, timeout=1.25).json()[ + "geonames" + ] # Go through all the countries and perform the mapping. for country in countries: mappings.update({country["countryCode"]: int(country["population"]) or None}) diff --git a/pyproject.toml b/pyproject.toml index b6bc6af6..df1ad168 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.black] -line-length = 120 +line-length = 100 target-version = ['py36', 'py37', 'py38'] include = '\.pyi?$' exclude = ''' @@ -23,7 +23,7 @@ multi_line_output = 3 include_trailing_comma = "True" force_grid_wrap = 0 use_parentheses = "True" -line_length = 120 +line_length = 100 [tool.pylint.master] extension-pkg-whitelist = "pydantic" @@ -42,7 +42,7 @@ logging-modules = "logging" allow-wildcard-with-all = "no" [tool.pylint.format] indent-after-paren = "4" -max-line-length = "120" # matches black setting +max-line-length = "100" # matches black setting max-module-lines = "800" no-space-check = ''' trailing-comma, diff --git a/requirements-dev.txt b/requirements-dev.txt index 1d919ece..d95c199e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ -i https://pypi.org/simple appdirs==1.4.3 -astroid==2.4.0 +astroid==2.4.1 async-asgi-testclient==1.4.4 async-generator==1.10 asyncmock==0.4.2 @@ -13,8 +13,8 @@ click==7.1.2 coverage==5.1 coveralls==2.0.0 docopt==0.6.2 -gitdb==4.0.4 -gitpython==3.1.1 +gitdb==4.0.5 +gitpython==3.1.2 idna==2.9 importlib-metadata==1.6.0 ; python_version < '3.8' invoke==1.4.1 @@ -29,17 +29,17 @@ pathspec==0.8.0 pbr==5.4.5 pluggy==0.13.1 py==1.8.1 -pylint==2.5.0 +pylint==2.5.2 pyparsing==2.4.7 -pytest-asyncio==0.11.0 +pytest-asyncio==0.12.0 pytest-cov==2.8.1 -pytest==5.4.1 +pytest==5.4.2 pyyaml==5.3.1 -regex==2020.4.4 +regex==2020.5.7 requests==2.23.0 responses==0.10.14 six==1.14.0 -smmap==3.0.2 +smmap==3.0.4 stevedore==1.32.0 toml==0.10.0 typed-ast==1.4.1 diff --git a/requirements.txt b/requirements.txt index dd2ece5a..02ab222e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,6 +30,7 @@ python-dateutil==2.8.1 python-dotenv==0.13.0 requests==2.23.0 scout-apm==2.14.1 +sentry-sdk==0.14.3 six==1.14.0 starlette==0.13.2 urllib3[secure]==1.25.9 ; python_version >= '3.5' diff --git a/tasks.py b/tasks.py index ae1f09cd..0f6d6995 100644 --- a/tasks.py +++ b/tasks.py @@ -72,12 +72,21 @@ def test(ctx): @invoke.task def generate_reqs(ctx): """Generate requirements.txt""" - reqs = ["pipenv lock -r > requirements.txt", "pipenv lock -r --dev > requirements-dev.txt"] + reqs = [ + "pipenv lock -r > requirements.txt", + "pipenv lock -r --dev > requirements-dev.txt", + ] [ctx.run(req) for req in reqs] @invoke.task -def docker(ctx, build=False, run=False, tag="covid-tracker-api:latest", name=f"covid-api-{random.randint(0,999)}"): +def docker( + ctx, + build=False, + run=False, + tag="covid-tracker-api:latest", + name=f"covid-api-{random.randint(0,999)}", +): """Build and run docker container.""" if not any([build, run]): raise invoke.Exit(message="Specify either --build or --run", code=1) diff --git a/tests/test_io.py b/tests/test_io.py index c5d16c3a..ba926011 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -10,7 +10,11 @@ [ ("test_file.txt", string.ascii_lowercase, {}), ("test_json_file.json", {"a": 0, "b": 1, "c": 2}, {}), - ("test_custom_json.json", {"z": -1, "b": 1, "y": -2, "a": 0}, {"indent": 4, "sort_keys": True}), + ( + "test_custom_json.json", + {"z": -1, "b": 1, "y": -2, "a": 0}, + {"indent": 4, "sort_keys": True}, + ), ], ) diff --git a/tests/test_location.py b/tests/test_location.py index 567eddcd..08d9d7c8 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -3,7 +3,7 @@ import pytest -from app import coordinates, location, timeline +from app import coordinates, location, models def mocked_timeline(*args, **kwargs): @@ -22,7 +22,7 @@ def __init__(self, latest): (2, "Cruise Ship", "XX", "", 15, 100, 1000, 1111, 22222), ], ) -@mock.patch("app.timeline.Timeline", side_effect=mocked_timeline) +@mock.patch("app.models.Timeline", side_effect=mocked_timeline) def test_location_class( mocked_timeline, test_id, @@ -39,16 +39,21 @@ def test_location_class( coords = coordinates.Coordinates(latitude=latitude, longitude=longitude) # Timelines - confirmed = timeline.Timeline(confirmed_latest) - deaths = timeline.Timeline(deaths_latest) - recovered = timeline.Timeline(recovered_latest) + confirmed = models.Timeline(confirmed_latest) + deaths = models.Timeline(deaths_latest) + recovered = models.Timeline(recovered_latest) # Date now. now = datetime.utcnow().isoformat() + "Z" # Location. location_obj = location.TimelinedLocation( - test_id, country, province, coords, now, {"confirmed": confirmed, "deaths": deaths, "recovered": recovered,} + test_id, + country, + province, + coords, + now, + {"confirmed": confirmed, "deaths": deaths, "recovered": recovered,}, ) assert location_obj.country_code == country_code diff --git a/tests/test_routes.py b/tests/test_routes.py index 52d26843..eea153bc 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -126,7 +126,9 @@ async def test_v2_locations_id(self): return_data = response.json() - filepath = "tests/expected_output/v2_{state}_id_{test_id}.json".format(state=state, test_id=test_id) + filepath = "tests/expected_output/v2_{state}_id_{test_id}.json".format( + state=state, test_id=test_id + ) with open(filepath, "r") as file: expected_json_output = file.read() @@ -151,7 +153,9 @@ async def test_v2_locations_id(self): ({"source": "jhu", "country_code": "US"}, 404), ], ) -async def test_locations_status_code(async_api_client, query_params, expected_status, mock_client_session): +async def test_locations_status_code( + async_api_client, query_params, expected_status, mock_client_session +): response = await async_api_client.get("/v2/locations", query_string=query_params) print(f"GET {response.url}\n{response}") diff --git a/tests/test_timeline.py b/tests/test_timeline.py index 056286aa..18a914ca 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -3,7 +3,7 @@ import pytest -from app import timeline +from app import models def test_timeline_class(): @@ -15,13 +15,18 @@ def test_timeline_class(): "1/23/20": 3, } - history_data = timeline.Timeline(history=timeseries) + history_data = models.Timeline(timeline=timeseries) # validate last value assert history_data.latest == 7 # validate order - assert list(dict(history_data.timeline).keys()) == ["1/22/20", "1/23/20", "1/24/20", "1/25/20"] + assert list(dict(history_data.timeline).keys()) == [ + "1/22/20", + "1/23/20", + "1/24/20", + "1/25/20", + ] # validate serialize check_serialize = {