Population backup

Pipfile

@@ -10,22 +10,23 @@ asyncmock = "*"
 bandit = "*"
 black = "==19.10b0"
 coveralls = "*"
-importlib-metadata = {version="*", markers="python_version<'3.8'"}
+importlib-metadata = {version = "*",markers = "python_version<'3.8'"}
 invoke = "*"
 isort = "*"
 pylint = "*"
 pytest = "*"
 pytest-asyncio = "*"
 pytest-cov = "*"
+responses = "*"
 
 [packages]
 aiohttp = "*"
 asyncache = "*"
 cachetools = "*"
-dataclasses = {version="*", markers="python_version<'3.7'"}
+dataclasses = {version = "*",markers = "python_version<'3.7'"}
 fastapi = "*"
 gunicorn = "*"
-idna_ssl = {version="*", markers="python_version<'3.7'"}
+idna_ssl = {version = "*",markers = "python_version<'3.7'"}
 python-dateutil = "*"
 python-dotenv = "*"
 requests = "*"

app/data/geonames_population_mappings.json

@@ -0,0 +1,252 @@
+{
+  "AD": 77006,
+  "AE": 9630959,
+  "AF": 37172386,
+  "AG": 96286,
+  "AI": 13254,
+  "AL": 2866376,
+  "AM": 2951776,
+  "AO": 30809762,
+  "AQ": null,
+  "AR": 44494502,
+  "AS": 55465,
+  "AT": 8847037,
+  "AU": 24992369,
+  "AW": 105845,
+  "AX": 26711,
+  "AZ": 9942334,
+  "BA": 3323929,
+  "BB": 286641,
+  "BD": 161356039,
+  "BE": 11422068,
+  "BF": 19751535,
+  "BG": 7000039,
+  "BH": 1569439,
+  "BI": 11175378,
+  "BJ": 11485048,
+  "BL": 8450,
+  "BM": 63968,
+  "BN": 428962,
+  "BO": 11353142,
+  "BQ": 18012,
+  "BR": 209469333,
+  "BS": 385640,
+  "BT": 754394,
+  "BV": null,
+  "BW": 2254126,
+  "BY": 9485386,
+  "BZ": 383071,
+  "CA": 37058856,
+  "CC": 628,
+  "CD": 84068091,
+  "CF": 4666377,
+  "CG": 5244363,
+  "CH": 8516543,
+  "CI": 25069229,
+  "CK": 21388,
+  "CL": 18729160,
+  "CM": 25216237,
+  "CN": 1392730000,
+  "CO": 49648685,
+  "CR": 4999441,
+  "CU": 11338138,
+  "CV": 543767,
+  "CW": 159849,
+  "CX": 1500,
+  "CY": 1189265,
+  "CZ": 10625695,
+  "DE": 82927922,
+  "DJ": 958920,
+  "DK": 5797446,
+  "DM": 71625,
+  "DO": 10627165,
+  "DZ": 42228429,
+  "EC": 17084357,
+  "EE": 1320884,
+  "EG": 98423595,
+  "EH": 273008,
+  "ER": null,
+  "ES": 46723749,
+  "ET": 109224559,
+  "FI": 5518050,
+  "FJ": 883483,
+  "FK": 2638,
+  "FM": 112640,
+  "FO": 48497,
+  "FR": 66987244,
+  "GA": 2119275,
+  "GB": 66488991,
+  "GD": 111454,
+  "GE": 3731000,
+  "GF": 195506,
+  "GG": 65228,
+  "GH": 29767108,
+  "GI": 33718,
+  "GL": 56025,
+  "GM": 2280102,
+  "GN": 12414318,
+  "GP": 443000,
+  "GQ": 1308974,
+  "GR": 10727668,
+  "GS": 30,
+  "GT": 17247807,
+  "GU": 165768,
+  "GW": 1874309,
+  "GY": 779004,
+  "HK": 7451000,
+  "HM": null,
+  "HN": 9587522,
+  "HR": 4089400,
+  "HT": 11123176,
+  "HU": 9768785,
+  "ID": 267663435,
+  "IE": 4853506,
+  "IL": 8883800,
+  "IM": 84077,
+  "IN": 1352617328,
+  "IO": 4000,
+  "IQ": 38433600,
+  "IR": 81800269,
+  "IS": 353574,
+  "IT": 60431283,
+  "JE": 90812,
+  "JM": 2934855,
+  "JO": 9956011,
+  "JP": 126529100,
+  "KE": 51393010,
+  "KG": 6315800,
+  "KH": 16249798,
+  "KI": 115847,
+  "KM": 832322,
+  "KN": 52441,
+  "KP": 25549819,
+  "KR": 51635256,
+  "KW": 4137309,
+  "KY": 64174,
+  "KZ": 18276499,
+  "LA": 7061507,
+  "LB": 6848925,
+  "LC": 181889,
+  "LI": 37910,
+  "LK": 21670000,
+  "LR": 4818977,
+  "LS": 2108132,
+  "LT": 2789533,
+  "LU": 607728,
+  "LV": 1926542,
+  "LY": 6678567,
+  "MA": 36029138,
+  "MC": 38682,
+  "MD": 3545883,
+  "ME": 622345,
+  "MF": 37264,
+  "MG": 26262368,
+  "MH": 58413,
+  "MK": 2082958,
+  "ML": 19077690,
+  "MM": 53708395,
+  "MN": 3170208,
+  "MO": 631636,
+  "MP": 56882,
+  "MQ": 432900,
+  "MR": 4403319,
+  "MS": 9341,
+  "MT": 483530,
+  "MU": 1265303,
+  "MV": 515696,
+  "MW": 17563749,
+  "MX": 126190788,
+  "MY": 31528585,
+  "MZ": 29495962,
+  "NA": 2448255,
+  "NC": 284060,
+  "NE": 22442948,
+  "NF": 1828,
+  "NG": 195874740,
+  "NI": 6465513,
+  "NL": 17231017,
+  "NO": 5314336,
+  "NP": 28087871,
+  "NR": 12704,
+  "NU": 2166,
+  "NZ": 4885500,
+  "OM": 4829483,
+  "PA": 4176873,
+  "PE": 31989256,
+  "PF": 277679,
+  "PG": 8606316,
+  "PH": 106651922,
+  "PK": 212215030,
+  "PL": 37978548,
+  "PM": 7012,
+  "PN": 46,
+  "PR": 3195153,
+  "PS": 4569087,
+  "PT": 10281762,
+  "PW": 17907,
+  "PY": 6956071,
+  "QA": 2781677,
+  "RE": 776948,
+  "RO": 19473936,
+  "RS": 6982084,
+  "RU": 144478050,
+  "RW": 12301939,
+  "SA": 33699947,
+  "SB": 652858,
+  "SC": 96762,
+  "SD": 41801533,
+  "SE": 10183175,
+  "SG": 5638676,
+  "SH": 7460,
+  "SI": 2067372,
+  "SJ": 2550,
+  "SK": 5447011,
+  "SL": 7650154,
+  "SM": 33785,
+  "SN": 15854360,
+  "SO": 15008154,
+  "SR": 575991,
+  "SS": 8260490,
+  "ST": 197700,
+  "SV": 6420744,
+  "SX": 40654,
+  "SY": 16906283,
+  "SZ": 1136191,
+  "TC": 37665,
+  "TD": 15477751,
+  "TF": 140,
+  "TG": 7889094,
+  "TH": 69428524,
+  "TJ": 9100837,
+  "TK": 1466,
+  "TL": 1267972,
+  "TM": 5850908,
+  "TN": 11565204,
+  "TO": 103197,
+  "TR": 82319724,
+  "TT": 1389858,
+  "TV": 11508,
+  "TW": 22894384,
+  "TZ": 56318348,
+  "UA": 44622516,
+  "UG": 42723139,
+  "UM": null,
+  "US": 327167434,
+  "UY": 3449299,
+  "UZ": 32955400,
+  "VA": 921,
+  "VC": 110211,
+  "VE": 28870195,
+  "VG": 29802,
+  "VI": 106977,
+  "VN": 95540395,
+  "VU": 292680,
+  "WF": 16025,
+  "WS": 196130,
+  "XK": 1845300,
+  "YE": 28498687,
+  "YT": 159042,
+  "ZA": 57779622,
+  "ZM": 17351822,
+  "ZW": 14439018
+}

app/io.py

@@ -0,0 +1,28 @@
+"""app.io.py"""
+import json
+import pathlib
+from typing import Dict, Union
+
+HERE = pathlib.Path(__file__)
+DATA = HERE.joinpath("..", "data").resolve()
+
+
+def save(
+    name: str, content: Union[str, Dict], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs
+) -> pathlib.Path:
+    """Save content to a file. If content is a dictionary, use json.dumps()."""
+    path = DATA / name
+    if isinstance(content, dict):
+        content = json.dumps(content, indent=indent, **json_dumps_kwargs)
+    with open(DATA / name, mode=write_mode) as f_out:
+        f_out.write(content)
+    return path
+
+
+def load(name: str, **json_kwargs) -> Union[str, Dict]:
+    """Loads content from a file. If file ends with '.json', call json.load() and return a Dictionary."""
+    path = DATA / name
+    with open(path) as f_in:
+        if path.suffix == ".json":
+            return json.load(f_in, **json_kwargs)
+        return f_in.read()

app/utils/populations.py

@@ -1,16 +1,23 @@
 """app.utils.populations.py"""
+import json
 import logging
 
 import requests
 
+import app.io
+
 LOGGER = logging.getLogger(__name__)
+GEONAMES_URL = "http://api.geonames.org/countryInfoJSON"
+GEONAMES_BACKUP_PATH = "geonames_population_mappings.json"
 
 # Fetching of the populations.
-def fetch_populations():
+def fetch_populations(save=False):
     """
     Returns a dictionary containing the population of each country fetched from the GeoNames.
     https://www.geonames.org/
 
+    TODO: only skip writing to the filesystem when deployed with gunicorn, or handle concurent access, or use DB.
+
     :returns: The mapping of populations.
     :rtype: dict
     """
@@ -20,12 +27,18 @@ def fetch_populations():
     mappings = {}
 
     # Fetch the countries.
-    countries = requests.get("http://api.geonames.org/countryInfoJSON?username=dperic").json()["geonames"]
-
-    # Go through all the countries and perform the mapping.
-    for country in countries:
-        mappings.update({country["countryCode"]: int(country["population"]) or None})
-
+    try:
+        countries = requests.get(GEONAMES_URL, params={"username": "dperic"}, timeout=1.5).json()["geonames"]
+        # Go through all the countries and perform the mapping.
+        for country in countries:
+            mappings.update({country["countryCode"]: int(country["population"]) or None})
+
+        if mappings and save:
+            LOGGER.info(f"Saving population data to {app.io.save(GEONAMES_BACKUP_PATH, mappings)}")
+    except (json.JSONDecodeError, KeyError, requests.exceptions.Timeout) as err:
+        LOGGER.warning(f"Error pulling population data. {err.__class__.__name__}: {err}")
+        mappings = app.io.load(GEONAMES_BACKUP_PATH)
+        LOGGER.info(f"Using backup data from {GEONAMES_BACKUP_PATH}")
     # Finally, return the mappings.
     return mappings