Skip to content

Commit 170eb12

Browse files
Kilo59Kilo59
authored
Cache mem optimize (#23)
* formatting (120 length -> 100) * csbs to/from Redis * nyt to/from Redis * ignore locustfile * unused coordinates
1 parent e54724e commit 170eb12

File tree

17 files changed

+194
-116
lines changed

17 files changed

+194
-116
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ htmlcov/
5151
nosetests.xml
5252
coverage.xml
5353
*,cover
54+
locustfile.py
5455

5556
# Translations
5657
*.mo

‎app/data/__init__.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@
44
from ..services.location.nyt import NYTLocationService
55

66
# Mapping of services to data-sources.
7-
DATA_SOURCES = {"jhu": JhuLocationService(), "csbs": CSBSLocationService(), "nyt": NYTLocationService()}
7+
DATA_SOURCES = {
8+
"jhu": JhuLocationService(),
9+
"csbs": CSBSLocationService(),
10+
"nyt": NYTLocationService(),
11+
}
812

913

1014
def data_source(source):

‎app/io.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,11 @@
1010

1111

1212
def save(
13-
name: str, content: Union[str, Dict, List], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs
13+
name: str,
14+
content: Union[str, Dict, List],
15+
write_mode: str = "w",
16+
indent: int = 2,
17+
**json_dumps_kwargs,
1418
) -> pathlib.Path:
1519
"""Save content to a file. If content is a dictionary, use json.dumps()."""
1620
path = DATA / name
@@ -35,7 +39,12 @@ class AIO:
3539

3640
@classmethod
3741
async def save(
38-
cls, name: str, content: Union[str, Dict, List], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs
42+
cls,
43+
name: str,
44+
content: Union[str, Dict, List],
45+
write_mode: str = "w",
46+
indent: int = 2,
47+
**json_dumps_kwargs,
3948
):
4049
"""Save content to a file. If content is a dictionary, use json.dumps()."""
4150
path = DATA / name

‎app/location/__init__.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ class Location: # pylint: disable=too-many-instance-attributes
1111
"""
1212

1313
def __init__(
14-
self, id, country, province, coordinates, last_updated, confirmed, deaths, recovered
14+
self, id, country, province, coordinates, last_updated, confirmed, deaths, recovered,
1515
): # pylint: disable=too-many-arguments
1616
# General info.
1717
self.id = id
@@ -66,7 +66,11 @@ def serialize(self):
6666
# Last updated.
6767
"last_updated": self.last_updated,
6868
# Latest data (statistics).
69-
"latest": {"confirmed": self.confirmed, "deaths": self.deaths, "recovered": self.recovered},
69+
"latest": {
70+
"confirmed": self.confirmed,
71+
"deaths": self.deaths,
72+
"recovered": self.recovered,
73+
},
7074
}
7175

7276

‎app/main.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,11 @@
4949

5050
# Enable CORS.
5151
APP.add_middleware(
52-
CORSMiddleware, allow_credentials=True, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"],
52+
CORSMiddleware,
53+
allow_credentials=True,
54+
allow_origins=["*"],
55+
allow_methods=["*"],
56+
allow_headers=["*"],
5357
)
5458
APP.add_middleware(GZipMiddleware, minimum_size=1000)
5559

‎app/routers/v1.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,11 @@ async def all_categories():
1919
"deaths": deaths,
2020
"recovered": recovered,
2121
# Latest.
22-
"latest": {"confirmed": confirmed["latest"], "deaths": deaths["latest"], "recovered": recovered["latest"],},
22+
"latest": {
23+
"confirmed": confirmed["latest"],
24+
"deaths": deaths["latest"],
25+
"recovered": recovered["latest"],
26+
},
2327
}
2428

2529

‎app/routers/v2.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,17 @@ async def get_locations(
6565

6666
# Do filtering.
6767
try:
68-
locations = [location for location in locations if str(getattr(location, key)).lower() == str(value)]
68+
locations = [
69+
location
70+
for location in locations
71+
if str(getattr(location, key)).lower() == str(value)
72+
]
6973
except AttributeError:
7074
pass
7175
if not locations:
72-
raise HTTPException(404, detail=f"Source `{source}` does not have the desired location data.")
76+
raise HTTPException(
77+
404, detail=f"Source `{source}` does not have the desired location data.",
78+
)
7379

7480
# Return final serialized data.
7581
return {
@@ -84,7 +90,9 @@ async def get_locations(
8490

8591
# pylint: disable=invalid-name
8692
@V2.get("/locations/{id}", response_model=LocationResponse)
87-
async def get_location_by_id(request: Request, id: int, source: Sources = "jhu", timelines: bool = True):
93+
async def get_location_by_id(
94+
request: Request, id: int, source: Sources = "jhu", timelines: bool = True
95+
):
8896
"""
8997
Getting specific location by id.
9098
"""

‎app/services/location/csbs.py‎

Lines changed: 49 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from asyncache import cached
77
from cachetools import TTLCache
88

9+
from ...caches import check_cache, load_cache
910
from ...coordinates import Coordinates
1011
from ...location.csbs import CSBSLocation
1112
from ...utils import httputils
@@ -34,7 +35,7 @@ async def get(self, loc_id): # pylint: disable=arguments-differ
3435
BASE_URL = "https://facts.csbs.org/covid-19/covid19_county.csv"
3536

3637

37-
@cached(cache=TTLCache(maxsize=1, ttl=3600))
38+
@cached(cache=TTLCache(maxsize=1, ttl=1800))
3839
async def get_locations():
3940
"""
4041
Retrieves county locations; locations are cached for 1 hour
@@ -44,48 +45,54 @@ async def get_locations():
4445
"""
4546
data_id = "csbs.locations"
4647
LOGGER.info(f"{data_id} Requesting data...")
47-
async with httputils.CLIENT_SESSION.get(BASE_URL) as response:
48-
text = await response.text()
49-
50-
LOGGER.debug(f"{data_id} Data received")
51-
52-
data = list(csv.DictReader(text.splitlines()))
53-
LOGGER.debug(f"{data_id} CSV parsed")
54-
55-
locations = []
56-
57-
for i, item in enumerate(data):
58-
# General info.
59-
state = item["State Name"]
60-
county = item["County Name"]
61-
62-
# Ensure country is specified.
63-
if county in {"Unassigned", "Unknown"}:
64-
continue
65-
66-
# Coordinates.
67-
coordinates = Coordinates(item["Latitude"], item["Longitude"]) # pylint: disable=unused-variable
68-
69-
# Date string without "EDT" at end.
70-
last_update = " ".join(item["Last Update"].split(" ")[0:2])
71-
72-
# Append to locations.
73-
locations.append(
74-
CSBSLocation(
75-
# General info.
76-
i,
77-
state,
78-
county,
79-
# Coordinates.
80-
Coordinates(item["Latitude"], item["Longitude"]),
81-
# Last update (parse as ISO).
82-
datetime.strptime(last_update, "%Y-%m-%d %H:%M").isoformat() + "Z",
83-
# Statistics.
84-
int(item["Confirmed"] or 0),
85-
int(item["Death"] or 0),
48+
# check shared cache
49+
cache_results = await check_cache(data_id)
50+
if cache_results:
51+
LOGGER.info(f"{data_id} using shared cache results")
52+
locations = cache_results
53+
else:
54+
LOGGER.info(f"{data_id} shared cache empty")
55+
async with httputils.CLIENT_SESSION.get(BASE_URL) as response:
56+
text = await response.text()
57+
58+
LOGGER.debug(f"{data_id} Data received")
59+
60+
data = list(csv.DictReader(text.splitlines()))
61+
LOGGER.debug(f"{data_id} CSV parsed")
62+
63+
locations = []
64+
65+
for i, item in enumerate(data):
66+
# General info.
67+
state = item["State Name"]
68+
county = item["County Name"]
69+
70+
# Ensure country is specified.
71+
if county in {"Unassigned", "Unknown"}:
72+
continue
73+
74+
# Date string without "EDT" at end.
75+
last_update = " ".join(item["Last Update"].split(" ")[0:2])
76+
77+
# Append to locations.
78+
locations.append(
79+
CSBSLocation(
80+
# General info.
81+
i,
82+
state,
83+
county,
84+
# Coordinates.
85+
Coordinates(item["Latitude"], item["Longitude"]),
86+
# Last update (parse as ISO).
87+
datetime.strptime(last_update, "%Y-%m-%d %H:%M").isoformat() + "Z",
88+
# Statistics.
89+
int(item["Confirmed"] or 0),
90+
int(item["Death"] or 0),
91+
)
8692
)
87-
)
88-
LOGGER.info(f"{data_id} Data normalized")
93+
LOGGER.info(f"{data_id} Data normalized")
94+
# save the results to distributed cache
95+
await load_cache(data_id, locations)
8996

9097
# Return the locations.
9198
return locations

‎app/services/location/jhu.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,7 @@ async def get(self, loc_id): # pylint: disable=arguments-differ
4141

4242

4343
# Base URL for fetching category.
44-
BASE_URL = (
45-
"https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/"
46-
)
44+
BASE_URL = "https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/"
4745

4846

4947
@cached(cache=TTLCache(maxsize=4, ttl=1800))

‎app/services/location/nyt.py‎

Lines changed: 60 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from asyncache import cached
77
from cachetools import TTLCache
88

9+
from ...caches import check_cache, load_cache
910
from ...coordinates import Coordinates
1011
from ...location.nyt import NYTLocation
1112
from ...timeline import Timeline
@@ -66,7 +67,7 @@ def get_grouped_locations_dict(data):
6667
return grouped_locations
6768

6869

69-
@cached(cache=TTLCache(maxsize=1, ttl=3600))
70+
@cached(cache=TTLCache(maxsize=1, ttl=1800))
7071
async def get_locations():
7172
"""
7273
Returns a list containing parsed NYT data by US county. The data is cached for 1 hour.
@@ -77,55 +78,64 @@ async def get_locations():
7778
data_id = "nyt.locations"
7879
# Request the data.
7980
LOGGER.info(f"{data_id} Requesting data...")
80-
async with httputils.CLIENT_SESSION.get(BASE_URL) as response:
81-
text = await response.text()
82-
83-
LOGGER.debug(f"{data_id} Data received")
84-
85-
# Parse the CSV.
86-
data = list(csv.DictReader(text.splitlines()))
87-
LOGGER.debug(f"{data_id} CSV parsed")
88-
89-
# Group together locations (NYT data ordered by dates not location).
90-
grouped_locations = get_grouped_locations_dict(data)
91-
92-
# The normalized locations.
93-
locations = []
94-
95-
for idx, (county_state, histories) in enumerate(grouped_locations.items()):
96-
# Make location history for confirmed and deaths from dates.
97-
# List is tuples of (date, amount) in order of increasing dates.
98-
confirmed_list = histories["confirmed"]
99-
confirmed_history = {date: int(amount or 0) for date, amount in confirmed_list}
100-
101-
deaths_list = histories["deaths"]
102-
deaths_history = {date: int(amount or 0) for date, amount in deaths_list}
103-
104-
# Normalize the item and append to locations.
105-
locations.append(
106-
NYTLocation(
107-
id=idx,
108-
state=county_state[1],
109-
county=county_state[0],
110-
coordinates=Coordinates(None, None), # NYT does not provide coordinates
111-
last_updated=datetime.utcnow().isoformat() + "Z", # since last request
112-
timelines={
113-
"confirmed": Timeline(
114-
{
115-
datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount
116-
for date, amount in confirmed_history.items()
117-
}
118-
),
119-
"deaths": Timeline(
120-
{
121-
datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount
122-
for date, amount in deaths_history.items()
123-
}
124-
),
125-
"recovered": Timeline({}),
126-
},
81+
# check shared cache
82+
cache_results = await check_cache(data_id)
83+
if cache_results:
84+
LOGGER.info(f"{data_id} using shared cache results")
85+
locations = cache_results
86+
else:
87+
LOGGER.info(f"{data_id} shared cache empty")
88+
async with httputils.CLIENT_SESSION.get(BASE_URL) as response:
89+
text = await response.text()
90+
91+
LOGGER.debug(f"{data_id} Data received")
92+
93+
# Parse the CSV.
94+
data = list(csv.DictReader(text.splitlines()))
95+
LOGGER.debug(f"{data_id} CSV parsed")
96+
97+
# Group together locations (NYT data ordered by dates not location).
98+
grouped_locations = get_grouped_locations_dict(data)
99+
100+
# The normalized locations.
101+
locations = []
102+
103+
for idx, (county_state, histories) in enumerate(grouped_locations.items()):
104+
# Make location history for confirmed and deaths from dates.
105+
# List is tuples of (date, amount) in order of increasing dates.
106+
confirmed_list = histories["confirmed"]
107+
confirmed_history = {date: int(amount or 0) for date, amount in confirmed_list}
108+
109+
deaths_list = histories["deaths"]
110+
deaths_history = {date: int(amount or 0) for date, amount in deaths_list}
111+
112+
# Normalize the item and append to locations.
113+
locations.append(
114+
NYTLocation(
115+
id=idx,
116+
state=county_state[1],
117+
county=county_state[0],
118+
coordinates=Coordinates(None, None), # NYT does not provide coordinates
119+
last_updated=datetime.utcnow().isoformat() + "Z", # since last request
120+
timelines={
121+
"confirmed": Timeline(
122+
{
123+
datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount
124+
for date, amount in confirmed_history.items()
125+
}
126+
),
127+
"deaths": Timeline(
128+
{
129+
datetime.strptime(date, "%Y-%m-%d").isoformat() + "Z": amount
130+
for date, amount in deaths_history.items()
131+
}
132+
),
133+
"recovered": Timeline({}),
134+
},
135+
)
127136
)
128-
)
129-
LOGGER.info(f"{data_id} Data normalized")
137+
LOGGER.info(f"{data_id} Data normalized")
138+
# save the results to distributed cache
139+
await load_cache(data_id, locations)
130140

131141
return locations

0 commit comments

Comments
 (0)