From 8fc0eb72f5058899c5b68740115d79d55c0978a8 Mon Sep 17 00:00:00 2001 From: ExpDev07 Date: Wed, 24 Aug 2022 16:24:44 +0200 Subject: [PATCH 1/3] Handle indexes differing for locations between the JHU timeseries --- app/routers/v2.py | 1 + app/services/location/csbs.py | 27 ++++++------ app/services/location/jhu.py | 80 ++++++++++++++++------------------- app/services/location/nyt.py | 5 +-- 4 files changed, 52 insertions(+), 61 deletions(-) diff --git a/app/routers/v2.py b/app/routers/v2.py index 31eb408c..da6a81c2 100644 --- a/app/routers/v2.py +++ b/app/routers/v2.py @@ -99,6 +99,7 @@ async def get_location_by_id( Getting specific location by id. """ location = await request.state.source.get(id) + return {"location": location.serialize(timelines)} diff --git a/app/services/location/csbs.py b/app/services/location/csbs.py index 444ebad6..d4758e1b 100644 --- a/app/services/location/csbs.py +++ b/app/services/location/csbs.py @@ -14,7 +14,6 @@ LOGGER = logging.getLogger("services.location.csbs") - class CSBSLocationService(LocationService): """ Service for retrieving locations from csbs @@ -23,18 +22,18 @@ class CSBSLocationService(LocationService): async def get_all(self): # Get the locations. locations = await get_locations() + return locations async def get(self, loc_id): # pylint: disable=arguments-differ # Get location at the index equal to the provided id. locations = await self.get_all() - return locations[loc_id] + return locations[loc_id] # Base URL for fetching data BASE_URL = "https://facts.csbs.org/covid-19/covid19_county.csv" - @cached(cache=TTLCache(maxsize=1, ttl=1800)) async def get_locations(): """ @@ -43,8 +42,10 @@ async def get_locations(): :returns: The locations. :rtype: dict """ + data_id = "csbs.locations" LOGGER.info(f"{data_id} Requesting data...") + # check shared cache cache_results = await check_cache(data_id) if cache_results: @@ -77,19 +78,16 @@ async def get_locations(): # Append to locations. locations.append( CSBSLocation( - # General info. - i, - state, - county, - # Coordinates. - Coordinates(item["Latitude"], item["Longitude"]), - # Last update (parse as ISO). - datetime.strptime(last_update, "%Y-%m-%d %H:%M").isoformat() + "Z", - # Statistics. - int(item["Confirmed"] or 0), - int(item["Death"] or 0), + id=i, + state=state, + county=county, + coordinates=Coordinates(item["Latitude"], item["Longitude"]), + last_updated=datetime.strptime(last_update, "%Y-%m-%d %H:%M").isoformat() + "Z", + confirmed=int(item["Confirmed"] or 0), + deaths=int(item["Death"] or 0), ) ) + LOGGER.info(f"{data_id} Data normalized") # save the results to distributed cache # TODO: fix json serialization @@ -98,5 +96,4 @@ async def get_locations(): except TypeError as type_err: LOGGER.error(type_err) - # Return the locations. return locations diff --git a/app/services/location/jhu.py b/app/services/location/jhu.py index ebed3960..29b22db6 100644 --- a/app/services/location/jhu.py +++ b/app/services/location/jhu.py @@ -20,7 +20,6 @@ LOGGER = logging.getLogger("services.location.jhu") PID = os.getpid() - class JhuLocationService(LocationService): """ Service for retrieving locations from Johns Hopkins CSSE (https://github.com/CSSEGISandData/COVID-19). @@ -34,6 +33,7 @@ async def get_all(self): async def get(self, loc_id): # pylint: disable=arguments-differ # Get location at the index equal to provided id. locations = await self.get_all() + return locations[loc_id] @@ -52,6 +52,7 @@ async def get_category(category): :returns: The data for category. :rtype: dict """ + # Adhere to category naming standard. category = category.lower() data_id = f"jhu.{category}" @@ -87,27 +88,24 @@ async def get_category(category): # Make location history from dates. history = {date: int(float(amount or 0)) for date, amount in dates.items()} - # Country for this location. - country = item["Country/Region"] - # Latest data insert value. latest = list(history.values())[-1] + # Country for this location. + country = item["Country/Region"] + # Normalize the item and append to locations. - locations.append( - { - # General info. - "country": country, - "country_code": countries.country_code(country), - "province": item["Province/State"], - # Coordinates. - "coordinates": {"lat": item["Lat"], "long": item["Long"],}, - # History. - "history": history, - # Latest statistic. - "latest": int(latest or 0), - } - ) + locations.append({ + "country": country, + "country_code": countries.country_code(country), + "province": item["Province/State"], + "coordinates": { + "lat": item["Lat"], + "long": item["Long"], + }, + "history": history, + "latest": int(latest or 0), + }) LOGGER.debug(f"{data_id} Data normalized") # Latest total. @@ -135,8 +133,10 @@ async def get_locations(): :returns: The locations. :rtype: List[Location] """ + data_id = "jhu.locations" LOGGER.info(f"pid:{PID}: {data_id} Requesting data...") + # Get all of the data categories locations. confirmed = await get_category("confirmed") deaths = await get_category("deaths") @@ -163,8 +163,8 @@ async def get_locations(): timelines = { "confirmed": location["history"], - "deaths": parse_history(key, locations_deaths, index), - "recovered": parse_history(key, locations_recovered, index), + "deaths": parse_history(key, locations_deaths), + "recovered": parse_history(key, locations_recovered), } # Grab coordinates. @@ -173,16 +173,12 @@ async def get_locations(): # Create location (supporting timelines) and append. locations.append( TimelinedLocation( - # General info. - index, - location["country"], - location["province"], - # Coordinates. - Coordinates(latitude=coordinates["lat"], longitude=coordinates["long"]), - # Last update. - datetime.utcnow().isoformat() + "Z", - # Timelines (parse dates as ISO). - { + id=index, + country=location["country"], + province=location["province"], + coordinates=Coordinates(latitude=coordinates["lat"], longitude=coordinates["long"]), + last_updated=datetime.utcnow().isoformat() + "Z", + timelines={ "confirmed": Timeline( timeline={ datetime.strptime(date, "%m/%d/%y").isoformat() + "Z": amount @@ -204,25 +200,23 @@ async def get_locations(): }, ) ) + LOGGER.info(f"{data_id} Data normalized") - # Finally, return the locations. return locations -def parse_history(key: tuple, locations: list, index: int): +def parse_history(key: tuple, locations: list): """ Helper for validating and extracting history content from - locations data based on index. Validates with the current country/province + locations data based on key. Validates with the current country/province key to make sure no index/column issue. - - TEMP: solution because implement a more efficient and better approach in the refactor. """ - location_history = {} - try: - if key == (locations[index]["country"], locations[index]["province"]): - location_history = locations[index]["history"] - except (IndexError, KeyError): - LOGGER.debug(f"iteration data merge error: {index} {key}") - - return location_history + + for i, location in enumerate(locations): + if (location["country"], location["province"]) == key: + return location["history"] + + LOGGER.debug(f"iteration data merge error: {key}") + + return {} diff --git a/app/services/location/nyt.py b/app/services/location/nyt.py index 1f25ec34..455315f6 100644 --- a/app/services/location/nyt.py +++ b/app/services/location/nyt.py @@ -15,7 +15,6 @@ LOGGER = logging.getLogger("services.location.nyt") - class NYTLocationService(LocationService): """ Service for retrieving locations from New York Times (https://github.com/nytimes/covid-19-data). @@ -24,21 +23,21 @@ class NYTLocationService(LocationService): async def get_all(self): # Get the locations. locations = await get_locations() + return locations async def get(self, loc_id): # pylint: disable=arguments-differ # Get location at the index equal to provided id. locations = await self.get_all() + return locations[loc_id] # --------------------------------------------------------------- - # Base URL for fetching category. BASE_URL = "https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-counties.csv" - def get_grouped_locations_dict(data): """ Helper function to group history for locations into one dict. From acdc002a52a450d0c607752d6f8468ef47caf7a6 Mon Sep 17 00:00:00 2001 From: ExpDev07 Date: Wed, 24 Aug 2022 16:35:53 +0200 Subject: [PATCH 2/3] update runtime to a supported one by Heroku --- runtime.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime.txt b/runtime.txt index 724c203e..5b3694c1 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.8.2 +python-3.8.13 From 67d9d22f1c56f77d0d91545d74ee1d796548943c Mon Sep 17 00:00:00 2001 From: Marius Date: Mon, 21 Nov 2022 19:53:33 +0100 Subject: [PATCH 3/3] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7355d0af..7c3024ed 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Coronavirus Tracker API -Provides up-to-date data about Coronavirus outbreak. Includes numbers about confirmed cases, deaths and recovered. +> Provides up-to-date data about Coronavirus outbreak. Includes numbers about confirmed cases, deaths and recovered. Support multiple data-sources. ![Travis build](https://api.travis-ci.com/ExpDev07/coronavirus-tracker-api.svg?branch=master)