From d6a332d34ed4fa4cfe13ea094ead1d2b9e6b1125 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 09:03:07 -0400 Subject: [PATCH 1/6] Update issue templates (#288) --- .github/ISSUE_TEMPLATE/bug_report.md | 61 ++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..4a80756e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,61 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. Please include timestamps and HTTP status codes. +If possible include the [httpie](https://httpie.org/) or `curl` request and response. +Please include the verbose flag. `-v` + +**To Reproduce** +`httpie/curl` request to reproduce the behavior: +1. Getting Italy data at `v2/locations/IT` gives a 422. +2. Expected to same data as `/v2/locations?country_code=IT` +2. See httpie request & response below +```sh + http GET https://coronavirus-tracker-api.herokuapp.com/v2/locations/IT -v +GET /v2/locations/IT HTTP/1.1 +Accept: */* +Accept-Encoding: gzip, deflate +Connection: keep-alive +Host: coronavirus-tracker-api.herokuapp.com +User-Agent: HTTPie/2.0.0 + + + +HTTP/1.1 422 Unprocessable Entity +Connection: keep-alive +Content-Length: 99 +Content-Type: application/json +Date: Sat, 18 Apr 2020 12:50:29 GMT +Server: uvicorn +Via: 1.1 vegur + +{ + "detail": [ + { + "loc": [ + "path", + "id" + ], + "msg": "value is not a valid integer", + "type": "type_error.integer" + } + ] +} +``` + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. +Does the other instance at https://covid-tracker-us.herokuapp.com/ produce the same result? From 9c817214ba8618b1a80dc4e587eb0135d99d6e64 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 09:06:50 -0400 Subject: [PATCH 2/6] Update bug_report.md --- .github/ISSUE_TEMPLATE/bug_report.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 4a80756e..afbb3e1d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -17,6 +17,12 @@ Please include the verbose flag. `-v` 1. Getting Italy data at `v2/locations/IT` gives a 422. 2. Expected to same data as `/v2/locations?country_code=IT` 2. See httpie request & response below + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots or Requests** +If applicable, add screenshots or `httpie/curl`requests to help explain your problem. ```sh http GET https://coronavirus-tracker-api.herokuapp.com/v2/locations/IT -v GET /v2/locations/IT HTTP/1.1 @@ -50,11 +56,6 @@ Via: 1.1 vegur } ``` -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. **Additional context** Add any other context about the problem here. From 81abb260bd710cb5670b9ea664e9996ea79e53e7 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 09:07:17 -0400 Subject: [PATCH 3/6] Update bug_report.md --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index afbb3e1d..0625e1fb 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -24,7 +24,7 @@ A clear and concise description of what you expected to happen. **Screenshots or Requests** If applicable, add screenshots or `httpie/curl`requests to help explain your problem. ```sh - http GET https://coronavirus-tracker-api.herokuapp.com/v2/locations/IT -v +$ http GET https://coronavirus-tracker-api.herokuapp.com/v2/locations/IT -v GET /v2/locations/IT HTTP/1.1 Accept: */* Accept-Encoding: gzip, deflate From a9f3b732c6770a7b14ed7c6bdcacc75c6d1d4927 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 14:10:46 -0400 Subject: [PATCH 4/6] add GZIP support for responses > 1000bytes (#294) * add GZIP support for responses > 1000bytes * increment version --- app/__init__.py | 2 +- app/main.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index c43ae7ac..57721529 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -4,4 +4,4 @@ API for tracking the global coronavirus (COVID-19, SARS-CoV-2) outbreak. """ # See PEP396. -__version__ = "2.0.1" +__version__ = "2.0.3" diff --git a/app/main.py b/app/main.py index 437b2395..1224c24e 100644 --- a/app/main.py +++ b/app/main.py @@ -8,6 +8,7 @@ import uvicorn from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import JSONResponse from .data import data_source @@ -26,7 +27,7 @@ "API for tracking the global coronavirus (COVID-19, SARS-CoV-2) outbreak." " Project page: https://github.com/ExpDev07/coronavirus-tracker-api." ), - version="2.0.2", + version="2.0.3", docs_url="/", redoc_url="/docs", on_startup=[setup_client_session], @@ -41,6 +42,7 @@ APP.add_middleware( CORSMiddleware, allow_credentials=True, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], ) +APP.add_middleware(GZipMiddleware, minimum_size=1000) @APP.middleware("http") From 6995938b42a3d6744a3116cb9f9bd8e9984ec4a8 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 12:46:02 -0400 Subject: [PATCH 5/6] add io module for writing to and reading from the file-system --- app/io.py | 28 ++++++++++++++++++++++++++++ tests/test_io.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 app/io.py create mode 100644 tests/test_io.py diff --git a/app/io.py b/app/io.py new file mode 100644 index 00000000..8130c146 --- /dev/null +++ b/app/io.py @@ -0,0 +1,28 @@ +"""app.io.py""" +import json +import pathlib +from typing import Dict, Union + +HERE = pathlib.Path(__file__) +DATA = HERE.joinpath("..", "data").resolve() + + +def save( + name: str, content: Union[str, Dict], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs +) -> pathlib.Path: + """Save content to a file. If content is a dictionary, use json.dumps().""" + path = DATA / name + if isinstance(content, dict): + content = json.dumps(content, indent=indent, **json_dumps_kwargs) + with open(DATA / name, mode=write_mode) as f_out: + f_out.write(content) + return path + + +def load(name: str, **json_kwargs) -> Union[str, Dict]: + """Loads content from a file. If file ends with '.json', call json.load() and return a Dictionary.""" + path = DATA / name + with open(path) as f_in: + if path.suffix == ".json": + return json.load(f_in, **json_kwargs) + return f_in.read() diff --git a/tests/test_io.py b/tests/test_io.py new file mode 100644 index 00000000..83639cc9 --- /dev/null +++ b/tests/test_io.py @@ -0,0 +1,39 @@ +"""test.test_io.py""" +import string + +import pytest + +import app.io + + +@pytest.mark.parametrize( + "name, content, kwargs", + [ + ("test_file.txt", string.ascii_lowercase, {}), + ("test_json_file.json", {"a": 0, "b": 1, "c": 2}, {}), + ("test_custom_json.json", {"z": -1, "b": 1, "y": -2, "a": 0}, {"indent": 4, "sort_keys": True}), + ], +) +def test_save(tmp_path, name, content, kwargs): + test_path = tmp_path / name + assert not test_path.exists() + + result = app.io.save(test_path, content, **kwargs) + assert result == test_path + assert test_path.exists() + + +@pytest.mark.parametrize( + "name, content, kwargs", + [ + ("test_file.txt", string.ascii_lowercase, {}), + ("test_json_file.json", {"a": 0, "b": 1, "c": 2}, {}), + ("test_custom_json.json", {"z": -1, "b": 1, "y": -2, "a": 0}, {"indent": 4, "sort_keys": True}), + ], +) +def test_round_trip(tmp_path, name, content, kwargs): + test_path = tmp_path / name + assert not test_path.exists() + + app.io.save(test_path, content, **kwargs) + assert app.io.load(test_path) == content From f16df566c9c13d0c91bfda65ec6591203f4d5e17 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 12:47:08 -0400 Subject: [PATCH 6/6] use io module to save and load poplutation data backups --- app/data/geonames_population_mappings.json | 252 +++++++++++++++++++++ app/utils/populations.py | 24 +- 2 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 app/data/geonames_population_mappings.json diff --git a/app/data/geonames_population_mappings.json b/app/data/geonames_population_mappings.json new file mode 100644 index 00000000..4f85e1a0 --- /dev/null +++ b/app/data/geonames_population_mappings.json @@ -0,0 +1,252 @@ +{ + "AD": 84000, + "AE": 4975593, + "AF": 29121286, + "AG": 86754, + "AI": 13254, + "AL": 2986952, + "AM": 2968000, + "AO": 13068161, + "AQ": null, + "AR": 41343201, + "AS": 57881, + "AT": 8205000, + "AU": 21515754, + "AW": 71566, + "AX": 26711, + "AZ": 8303512, + "BA": 4590000, + "BB": 285653, + "BD": 156118464, + "BE": 10403000, + "BF": 16241811, + "BG": 7000039, + "BH": 738004, + "BI": 9863117, + "BJ": 9056010, + "BL": 8450, + "BM": 65365, + "BN": 395027, + "BO": 9947418, + "BQ": 18012, + "BR": 201103330, + "BS": 301790, + "BT": 699847, + "BV": null, + "BW": 2029307, + "BY": 9685000, + "BZ": 314522, + "CA": 33679000, + "CC": 628, + "CD": 70916439, + "CF": 4844927, + "CG": 3039126, + "CH": 8484100, + "CI": 21058798, + "CK": 21388, + "CL": 16746491, + "CM": 19294149, + "CN": 1330044000, + "CO": 47790000, + "CR": 4516220, + "CU": 11423000, + "CV": 508659, + "CW": 141766, + "CX": 1500, + "CY": 1102677, + "CZ": 10476000, + "DE": 81802257, + "DJ": 740528, + "DK": 5484000, + "DM": 72813, + "DO": 9823821, + "DZ": 34586184, + "EC": 14790608, + "EE": 1291170, + "EG": 80471869, + "EH": 273008, + "ER": 5792984, + "ES": 46505963, + "ET": 88013491, + "FI": 5244000, + "FJ": 875983, + "FK": 2638, + "FM": 107708, + "FO": 48228, + "FR": 64768389, + "GA": 1545255, + "GB": 62348447, + "GD": 107818, + "GE": 4630000, + "GF": 195506, + "GG": 65228, + "GH": 24339838, + "GI": 27884, + "GL": 56375, + "GM": 1593256, + "GN": 10324025, + "GP": 443000, + "GQ": 1014999, + "GR": 11000000, + "GS": 30, + "GT": 13550440, + "GU": 159358, + "GW": 1565126, + "GY": 748486, + "HK": 6898686, + "HM": null, + "HN": 7989415, + "HR": 4284889, + "HT": 9648924, + "HU": 9982000, + "ID": 242968342, + "IE": 4622917, + "IL": 7353985, + "IM": 75049, + "IN": 1173108018, + "IO": 4000, + "IQ": 29671605, + "IR": 76923300, + "IS": 308910, + "IT": 60340328, + "JE": 90812, + "JM": 2847232, + "JO": 6407085, + "JP": 127288000, + "KE": 40046566, + "KG": 5776500, + "KH": 14453680, + "KI": 92533, + "KM": 773407, + "KN": 51134, + "KP": 22912177, + "KR": 48422644, + "KW": 2789132, + "KY": 44270, + "KZ": 15340000, + "LA": 6368162, + "LB": 4125247, + "LC": 160922, + "LI": 35000, + "LK": 21513990, + "LR": 3685076, + "LS": 1919552, + "LT": 2944459, + "LU": 497538, + "LV": 2217969, + "LY": 6461454, + "MA": 33848242, + "MC": 32965, + "MD": 4324000, + "ME": 666730, + "MF": 35925, + "MG": 21281844, + "MH": 65859, + "MK": 2062294, + "ML": 13796354, + "MM": 53414374, + "MN": 3086918, + "MO": 449198, + "MP": 53883, + "MQ": 432900, + "MR": 3205060, + "MS": 9341, + "MT": 403000, + "MU": 1294104, + "MV": 395650, + "MW": 17563749, + "MX": 112468855, + "MY": 28274729, + "MZ": 22061451, + "NA": 2128471, + "NC": 216494, + "NE": 15878271, + "NF": 1828, + "NG": 154000000, + "NI": 5995928, + "NL": 16645000, + "NO": 5009150, + "NP": 28951852, + "NR": 10065, + "NU": 2166, + "NZ": 4252277, + "OM": 2967717, + "PA": 3410676, + "PE": 29907003, + "PF": 270485, + "PG": 6064515, + "PH": 99900177, + "PK": 184404791, + "PL": 38500000, + "PM": 7012, + "PN": 46, + "PR": 3916632, + "PS": 3800000, + "PT": 10676000, + "PW": 19907, + "PY": 6375830, + "QA": 840926, + "RE": 776948, + "RO": 21959278, + "RS": 7344847, + "RU": 140702000, + "RW": 11055976, + "SA": 25731776, + "SB": 559198, + "SC": 88340, + "SD": 35000000, + "SE": 9828655, + "SG": 4701069, + "SH": 7460, + "SI": 2007000, + "SJ": 2550, + "SK": 5455000, + "SL": 5245695, + "SM": 31477, + "SN": 12323252, + "SO": 10112453, + "SR": 492829, + "SS": 8260490, + "ST": 197700, + "SV": 6052064, + "SX": 37429, + "SY": 22198110, + "SZ": 1354051, + "TC": 20556, + "TD": 10543464, + "TF": 140, + "TG": 6587239, + "TH": 67089500, + "TJ": 7487489, + "TK": 1466, + "TL": 1154625, + "TM": 4940916, + "TN": 10589025, + "TO": 122580, + "TR": 77804122, + "TT": 1328019, + "TV": 10472, + "TW": 22894384, + "TZ": 41892895, + "UA": 45415596, + "UG": 33398682, + "UM": null, + "US": 310232863, + "UY": 3477000, + "UZ": 27865738, + "VA": 921, + "VC": 104217, + "VE": 27223228, + "VG": 21730, + "VI": 108708, + "VN": 89571130, + "VU": 221552, + "WF": 16025, + "WS": 192001, + "XK": 1800000, + "YE": 23495361, + "YT": 159042, + "ZA": 49000000, + "ZM": 13460305, + "ZW": 13061000 +} \ No newline at end of file diff --git a/app/utils/populations.py b/app/utils/populations.py index 1d8bd843..cacf0ff3 100644 --- a/app/utils/populations.py +++ b/app/utils/populations.py @@ -1,9 +1,13 @@ """app.utils.populations.py""" +import json import logging import requests +import app.io + LOGGER = logging.getLogger(__name__) +GEONAMES_BACKUP_PATH = "geonames_population_mappings.json" # Fetching of the populations. def fetch_populations(): @@ -20,12 +24,20 @@ def fetch_populations(): mappings = {} # Fetch the countries. - countries = requests.get("http://api.geonames.org/countryInfoJSON?username=dperic").json()["geonames"] - - # Go through all the countries and perform the mapping. - for country in countries: - mappings.update({country["countryCode"]: int(country["population"]) or None}) - + try: + countries = requests.get( + "http://api.geonames.org/countryInfoJSON", params={"username": "dperic"}, timeout=2 + ).json()["geonames"] + # Go through all the countries and perform the mapping. + for country in countries: + mappings.update({country["countryCode"]: int(country["population"]) or None}) + + if mappings: + app.io.save(GEONAMES_BACKUP_PATH, mappings) + except (json.JSONDecodeError, KeyError, requests.exceptions.Timeout) as err: + LOGGER.warning(f"Error pulling population data. {err.__class__.__name__}: {err}") + mappings = app.io.load(GEONAMES_BACKUP_PATH) + LOGGER.info(f"Using backup data from {GEONAMES_BACKUP_PATH}") # Finally, return the mappings. return mappings