Skip to content

Commit 56f51c5

Browse files
Kilo59Kilo59
authored
Population backup (#291)
* add io module for writing to and reading from the file-system * use io module to save and load population data backups * add responses test dependency (update deps) * test population fallback conditions * don't write to file-system by default * update population fallback data
1 parent 0d1ff74 commit 56f51c5

File tree

9 files changed

+564
-142
lines changed

9 files changed

+564
-142
lines changed

‎Pipfile‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,22 +10,23 @@ asyncmock = "*"
1010
bandit = "*"
1111
black = "==19.10b0"
1212
coveralls = "*"
13-
importlib-metadata = {version="*", markers="python_version<'3.8'"}
13+
importlib-metadata = {version = "*",markers = "python_version<'3.8'"}
1414
invoke = "*"
1515
isort = "*"
1616
pylint = "*"
1717
pytest = "*"
1818
pytest-asyncio = "*"
1919
pytest-cov = "*"
20+
responses = "*"
2021

2122
[packages]
2223
aiohttp = "*"
2324
asyncache = "*"
2425
cachetools = "*"
25-
dataclasses = {version="*", markers="python_version<'3.7'"}
26+
dataclasses = {version = "*",markers = "python_version<'3.7'"}
2627
fastapi = "*"
2728
gunicorn = "*"
28-
idna_ssl = {version="*", markers="python_version<'3.7'"}
29+
idna_ssl = {version = "*",markers = "python_version<'3.7'"}
2930
python-dateutil = "*"
3031
python-dotenv = "*"
3132
requests = "*"

‎Pipfile.lock‎

Lines changed: 126 additions & 115 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎app/data/geonames_population_mappings.json‎

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
{
2+
"AD": 77006,
3+
"AE": 9630959,
4+
"AF": 37172386,
5+
"AG": 96286,
6+
"AI": 13254,
7+
"AL": 2866376,
8+
"AM": 2951776,
9+
"AO": 30809762,
10+
"AQ": null,
11+
"AR": 44494502,
12+
"AS": 55465,
13+
"AT": 8847037,
14+
"AU": 24992369,
15+
"AW": 105845,
16+
"AX": 26711,
17+
"AZ": 9942334,
18+
"BA": 3323929,
19+
"BB": 286641,
20+
"BD": 161356039,
21+
"BE": 11422068,
22+
"BF": 19751535,
23+
"BG": 7000039,
24+
"BH": 1569439,
25+
"BI": 11175378,
26+
"BJ": 11485048,
27+
"BL": 8450,
28+
"BM": 63968,
29+
"BN": 428962,
30+
"BO": 11353142,
31+
"BQ": 18012,
32+
"BR": 209469333,
33+
"BS": 385640,
34+
"BT": 754394,
35+
"BV": null,
36+
"BW": 2254126,
37+
"BY": 9485386,
38+
"BZ": 383071,
39+
"CA": 37058856,
40+
"CC": 628,
41+
"CD": 84068091,
42+
"CF": 4666377,
43+
"CG": 5244363,
44+
"CH": 8516543,
45+
"CI": 25069229,
46+
"CK": 21388,
47+
"CL": 18729160,
48+
"CM": 25216237,
49+
"CN": 1392730000,
50+
"CO": 49648685,
51+
"CR": 4999441,
52+
"CU": 11338138,
53+
"CV": 543767,
54+
"CW": 159849,
55+
"CX": 1500,
56+
"CY": 1189265,
57+
"CZ": 10625695,
58+
"DE": 82927922,
59+
"DJ": 958920,
60+
"DK": 5797446,
61+
"DM": 71625,
62+
"DO": 10627165,
63+
"DZ": 42228429,
64+
"EC": 17084357,
65+
"EE": 1320884,
66+
"EG": 98423595,
67+
"EH": 273008,
68+
"ER": null,
69+
"ES": 46723749,
70+
"ET": 109224559,
71+
"FI": 5518050,
72+
"FJ": 883483,
73+
"FK": 2638,
74+
"FM": 112640,
75+
"FO": 48497,
76+
"FR": 66987244,
77+
"GA": 2119275,
78+
"GB": 66488991,
79+
"GD": 111454,
80+
"GE": 3731000,
81+
"GF": 195506,
82+
"GG": 65228,
83+
"GH": 29767108,
84+
"GI": 33718,
85+
"GL": 56025,
86+
"GM": 2280102,
87+
"GN": 12414318,
88+
"GP": 443000,
89+
"GQ": 1308974,
90+
"GR": 10727668,
91+
"GS": 30,
92+
"GT": 17247807,
93+
"GU": 165768,
94+
"GW": 1874309,
95+
"GY": 779004,
96+
"HK": 7451000,
97+
"HM": null,
98+
"HN": 9587522,
99+
"HR": 4089400,
100+
"HT": 11123176,
101+
"HU": 9768785,
102+
"ID": 267663435,
103+
"IE": 4853506,
104+
"IL": 8883800,
105+
"IM": 84077,
106+
"IN": 1352617328,
107+
"IO": 4000,
108+
"IQ": 38433600,
109+
"IR": 81800269,
110+
"IS": 353574,
111+
"IT": 60431283,
112+
"JE": 90812,
113+
"JM": 2934855,
114+
"JO": 9956011,
115+
"JP": 126529100,
116+
"KE": 51393010,
117+
"KG": 6315800,
118+
"KH": 16249798,
119+
"KI": 115847,
120+
"KM": 832322,
121+
"KN": 52441,
122+
"KP": 25549819,
123+
"KR": 51635256,
124+
"KW": 4137309,
125+
"KY": 64174,
126+
"KZ": 18276499,
127+
"LA": 7061507,
128+
"LB": 6848925,
129+
"LC": 181889,
130+
"LI": 37910,
131+
"LK": 21670000,
132+
"LR": 4818977,
133+
"LS": 2108132,
134+
"LT": 2789533,
135+
"LU": 607728,
136+
"LV": 1926542,
137+
"LY": 6678567,
138+
"MA": 36029138,
139+
"MC": 38682,
140+
"MD": 3545883,
141+
"ME": 622345,
142+
"MF": 37264,
143+
"MG": 26262368,
144+
"MH": 58413,
145+
"MK": 2082958,
146+
"ML": 19077690,
147+
"MM": 53708395,
148+
"MN": 3170208,
149+
"MO": 631636,
150+
"MP": 56882,
151+
"MQ": 432900,
152+
"MR": 4403319,
153+
"MS": 9341,
154+
"MT": 483530,
155+
"MU": 1265303,
156+
"MV": 515696,
157+
"MW": 17563749,
158+
"MX": 126190788,
159+
"MY": 31528585,
160+
"MZ": 29495962,
161+
"NA": 2448255,
162+
"NC": 284060,
163+
"NE": 22442948,
164+
"NF": 1828,
165+
"NG": 195874740,
166+
"NI": 6465513,
167+
"NL": 17231017,
168+
"NO": 5314336,
169+
"NP": 28087871,
170+
"NR": 12704,
171+
"NU": 2166,
172+
"NZ": 4885500,
173+
"OM": 4829483,
174+
"PA": 4176873,
175+
"PE": 31989256,
176+
"PF": 277679,
177+
"PG": 8606316,
178+
"PH": 106651922,
179+
"PK": 212215030,
180+
"PL": 37978548,
181+
"PM": 7012,
182+
"PN": 46,
183+
"PR": 3195153,
184+
"PS": 4569087,
185+
"PT": 10281762,
186+
"PW": 17907,
187+
"PY": 6956071,
188+
"QA": 2781677,
189+
"RE": 776948,
190+
"RO": 19473936,
191+
"RS": 6982084,
192+
"RU": 144478050,
193+
"RW": 12301939,
194+
"SA": 33699947,
195+
"SB": 652858,
196+
"SC": 96762,
197+
"SD": 41801533,
198+
"SE": 10183175,
199+
"SG": 5638676,
200+
"SH": 7460,
201+
"SI": 2067372,
202+
"SJ": 2550,
203+
"SK": 5447011,
204+
"SL": 7650154,
205+
"SM": 33785,
206+
"SN": 15854360,
207+
"SO": 15008154,
208+
"SR": 575991,
209+
"SS": 8260490,
210+
"ST": 197700,
211+
"SV": 6420744,
212+
"SX": 40654,
213+
"SY": 16906283,
214+
"SZ": 1136191,
215+
"TC": 37665,
216+
"TD": 15477751,
217+
"TF": 140,
218+
"TG": 7889094,
219+
"TH": 69428524,
220+
"TJ": 9100837,
221+
"TK": 1466,
222+
"TL": 1267972,
223+
"TM": 5850908,
224+
"TN": 11565204,
225+
"TO": 103197,
226+
"TR": 82319724,
227+
"TT": 1389858,
228+
"TV": 11508,
229+
"TW": 22894384,
230+
"TZ": 56318348,
231+
"UA": 44622516,
232+
"UG": 42723139,
233+
"UM": null,
234+
"US": 327167434,
235+
"UY": 3449299,
236+
"UZ": 32955400,
237+
"VA": 921,
238+
"VC": 110211,
239+
"VE": 28870195,
240+
"VG": 29802,
241+
"VI": 106977,
242+
"VN": 95540395,
243+
"VU": 292680,
244+
"WF": 16025,
245+
"WS": 196130,
246+
"XK": 1845300,
247+
"YE": 28498687,
248+
"YT": 159042,
249+
"ZA": 57779622,
250+
"ZM": 17351822,
251+
"ZW": 14439018
252+
}

‎app/io.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
"""app.io.py"""
2+
import json
3+
import pathlib
4+
from typing import Dict, Union
5+
6+
HERE = pathlib.Path(__file__)
7+
DATA = HERE.joinpath("..", "data").resolve()
8+
9+
10+
def save(
11+
name: str, content: Union[str, Dict], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs
12+
) -> pathlib.Path:
13+
"""Save content to a file. If content is a dictionary, use json.dumps()."""
14+
path = DATA / name
15+
if isinstance(content, dict):
16+
content = json.dumps(content, indent=indent, **json_dumps_kwargs)
17+
with open(DATA / name, mode=write_mode) as f_out:
18+
f_out.write(content)
19+
return path
20+
21+
22+
def load(name: str, **json_kwargs) -> Union[str, Dict]:
23+
"""Loads content from a file. If file ends with '.json', call json.load() and return a Dictionary."""
24+
path = DATA / name
25+
with open(path) as f_in:
26+
if path.suffix == ".json":
27+
return json.load(f_in, **json_kwargs)
28+
return f_in.read()

‎app/utils/populations.py‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,23 @@
11
"""app.utils.populations.py"""
2+
import json
23
import logging
34

45
import requests
56

7+
import app.io
8+
69
LOGGER = logging.getLogger(__name__)
10+
GEONAMES_URL = "http://api.geonames.org/countryInfoJSON"
11+
GEONAMES_BACKUP_PATH = "geonames_population_mappings.json"
712

813
# Fetching of the populations.
9-
def fetch_populations():
14+
def fetch_populations(save=False):
1015
"""
1116
Returns a dictionary containing the population of each country fetched from the GeoNames.
1217
https://www.geonames.org/
1318
19+
TODO: only skip writing to the filesystem when deployed with gunicorn, or handle concurent access, or use DB.
20+
1421
:returns: The mapping of populations.
1522
:rtype: dict
1623
"""
@@ -20,12 +27,18 @@ def fetch_populations():
2027
mappings = {}
2128

2229
# Fetch the countries.
23-
countries = requests.get("http://api.geonames.org/countryInfoJSON?username=dperic").json()["geonames"]
24-
25-
# Go through all the countries and perform the mapping.
26-
for country in countries:
27-
mappings.update({country["countryCode"]: int(country["population"]) or None})
28-
30+
try:
31+
countries = requests.get(GEONAMES_URL, params={"username": "dperic"}, timeout=1.5).json()["geonames"]
32+
# Go through all the countries and perform the mapping.
33+
for country in countries:
34+
mappings.update({country["countryCode"]: int(country["population"]) or None})
35+
36+
if mappings and save:
37+
LOGGER.info(f"Saving population data to {app.io.save(GEONAMES_BACKUP_PATH, mappings)}")
38+
except (json.JSONDecodeError, KeyError, requests.exceptions.Timeout) as err:
39+
LOGGER.warning(f"Error pulling population data. {err.__class__.__name__}: {err}")
40+
mappings = app.io.load(GEONAMES_BACKUP_PATH)
41+
LOGGER.info(f"Using backup data from {GEONAMES_BACKUP_PATH}")
2942
# Finally, return the mappings.
3043
return mappings
3144

0 commit comments

Comments
 (0)