From d6a332d34ed4fa4cfe13ea094ead1d2b9e6b1125 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 09:03:07 -0400 Subject: [PATCH 01/12] Update issue templates (#288) --- .github/ISSUE_TEMPLATE/bug_report.md | 61 ++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..4a80756e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,61 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. Please include timestamps and HTTP status codes. +If possible include the [httpie](https://httpie.org/) or `curl` request and response. +Please include the verbose flag. `-v` + +**To Reproduce** +`httpie/curl` request to reproduce the behavior: +1. Getting Italy data at `v2/locations/IT` gives a 422. +2. Expected to same data as `/v2/locations?country_code=IT` +2. See httpie request & response below +```sh + http GET https://coronavirus-tracker-api.herokuapp.com/v2/locations/IT -v +GET /v2/locations/IT HTTP/1.1 +Accept: */* +Accept-Encoding: gzip, deflate +Connection: keep-alive +Host: coronavirus-tracker-api.herokuapp.com +User-Agent: HTTPie/2.0.0 + + + +HTTP/1.1 422 Unprocessable Entity +Connection: keep-alive +Content-Length: 99 +Content-Type: application/json +Date: Sat, 18 Apr 2020 12:50:29 GMT +Server: uvicorn +Via: 1.1 vegur + +{ + "detail": [ + { + "loc": [ + "path", + "id" + ], + "msg": "value is not a valid integer", + "type": "type_error.integer" + } + ] +} +``` + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. +Does the other instance at https://covid-tracker-us.herokuapp.com/ produce the same result? From 9c817214ba8618b1a80dc4e587eb0135d99d6e64 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 09:06:50 -0400 Subject: [PATCH 02/12] Update bug_report.md --- .github/ISSUE_TEMPLATE/bug_report.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 4a80756e..afbb3e1d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -17,6 +17,12 @@ Please include the verbose flag. `-v` 1. Getting Italy data at `v2/locations/IT` gives a 422. 2. Expected to same data as `/v2/locations?country_code=IT` 2. See httpie request & response below + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots or Requests** +If applicable, add screenshots or `httpie/curl`requests to help explain your problem. ```sh http GET https://coronavirus-tracker-api.herokuapp.com/v2/locations/IT -v GET /v2/locations/IT HTTP/1.1 @@ -50,11 +56,6 @@ Via: 1.1 vegur } ``` -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. **Additional context** Add any other context about the problem here. From 81abb260bd710cb5670b9ea664e9996ea79e53e7 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 09:07:17 -0400 Subject: [PATCH 03/12] Update bug_report.md --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index afbb3e1d..0625e1fb 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -24,7 +24,7 @@ A clear and concise description of what you expected to happen. **Screenshots or Requests** If applicable, add screenshots or `httpie/curl`requests to help explain your problem. ```sh - http GET https://coronavirus-tracker-api.herokuapp.com/v2/locations/IT -v +$ http GET https://coronavirus-tracker-api.herokuapp.com/v2/locations/IT -v GET /v2/locations/IT HTTP/1.1 Accept: */* Accept-Encoding: gzip, deflate From a9f3b732c6770a7b14ed7c6bdcacc75c6d1d4927 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 14:10:46 -0400 Subject: [PATCH 04/12] add GZIP support for responses > 1000bytes (#294) * add GZIP support for responses > 1000bytes * increment version --- app/__init__.py | 2 +- app/main.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index c43ae7ac..57721529 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -4,4 +4,4 @@ API for tracking the global coronavirus (COVID-19, SARS-CoV-2) outbreak. """ # See PEP396. -__version__ = "2.0.1" +__version__ = "2.0.3" diff --git a/app/main.py b/app/main.py index 437b2395..1224c24e 100644 --- a/app/main.py +++ b/app/main.py @@ -8,6 +8,7 @@ import uvicorn from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import JSONResponse from .data import data_source @@ -26,7 +27,7 @@ "API for tracking the global coronavirus (COVID-19, SARS-CoV-2) outbreak." " Project page: https://github.com/ExpDev07/coronavirus-tracker-api." ), - version="2.0.2", + version="2.0.3", docs_url="/", redoc_url="/docs", on_startup=[setup_client_session], @@ -41,6 +42,7 @@ APP.add_middleware( CORSMiddleware, allow_credentials=True, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], ) +APP.add_middleware(GZipMiddleware, minimum_size=1000) @APP.middleware("http") From 0d1ff7458e9b032f8aa22fca03f2e79a5e202538 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 16:42:33 -0400 Subject: [PATCH 05/12] add minimal logging for location services (#290) --- app/services/location/csbs.py | 7 +++++++ app/services/location/jhu.py | 7 +++++++ app/services/location/nyt.py | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/app/services/location/csbs.py b/app/services/location/csbs.py index dbd8d82d..d0de3837 100644 --- a/app/services/location/csbs.py +++ b/app/services/location/csbs.py @@ -1,5 +1,6 @@ """app.services.location.csbs.py""" import csv +import logging from datetime import datetime from asyncache import cached @@ -39,10 +40,15 @@ async def get_locations(): :returns: The locations. :rtype: dict """ + logger = logging.getLogger("services.location.csbs") + logger.info("Requesting data...") async with httputils.CLIENT_SESSION.get(BASE_URL) as response: text = await response.text() + logger.info("Data received") + data = list(csv.DictReader(text.splitlines())) + logger.info("CSV parsed") locations = [] @@ -77,6 +83,7 @@ async def get_locations(): int(item["Death"] or 0), ) ) + logger.info("Data normalized") # Return the locations. return locations diff --git a/app/services/location/jhu.py b/app/services/location/jhu.py index 316de367..39e07ac2 100644 --- a/app/services/location/jhu.py +++ b/app/services/location/jhu.py @@ -1,5 +1,6 @@ """app.services.location.jhu.py""" import csv +import logging from datetime import datetime from asyncache import cached @@ -47,6 +48,7 @@ async def get_category(category): :returns: The data for category. :rtype: dict """ + logger = logging.getLogger("services.location.jhu") # Adhere to category naming standard. category = category.lower() @@ -55,11 +57,15 @@ async def get_category(category): url = BASE_URL + "time_series_covid19_%s_global.csv" % category # Request the data + logger.info("Requesting data...") async with httputils.CLIENT_SESSION.get(url) as response: text = await response.text() + logger.info("Data received") + # Parse the CSV. data = list(csv.DictReader(text.splitlines())) + logger.info("CSV parsed") # The normalized locations. locations = [] @@ -92,6 +98,7 @@ async def get_category(category): "latest": int(latest or 0), } ) + logger.info("Data normalized") # Latest total. latest = sum(map(lambda location: location["latest"], locations)) diff --git a/app/services/location/nyt.py b/app/services/location/nyt.py index 7f73c1de..7f12eb62 100644 --- a/app/services/location/nyt.py +++ b/app/services/location/nyt.py @@ -1,5 +1,6 @@ """app.services.location.nyt.py""" import csv +import logging from datetime import datetime from asyncache import cached @@ -71,13 +72,18 @@ async def get_locations(): :returns: The complete data for US Counties. :rtype: dict """ + logger = logging.getLogger("services.location.nyt") # Request the data. + logger.info("Requesting data...") async with httputils.CLIENT_SESSION.get(BASE_URL) as response: text = await response.text() + logger.info("Data received") + # Parse the CSV. data = list(csv.DictReader(text.splitlines())) + logger.info("CSV parsed") # Group together locations (NYT data ordered by dates not location). grouped_locations = get_grouped_locations_dict(data) @@ -119,5 +125,6 @@ async def get_locations(): }, ) ) + logger.info("Data normalized") return locations From 56f51c5880b541c469ebb0176ec826f0da900e1e Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 17:19:50 -0400 Subject: [PATCH 06/12] Population backup (#291) * add io module for writing to and reading from the file-system * use io module to save and load population data backups * add responses test dependency (update deps) * test population fallback conditions * don't write to file-system by default * update population fallback data --- Pipfile | 7 +- Pipfile.lock | 241 ++++++++++---------- app/data/geonames_population_mappings.json | 252 +++++++++++++++++++++ app/io.py | 28 +++ app/utils/populations.py | 27 ++- requirements-dev.txt | 23 +- requirements.txt | 12 +- tests/test_io.py | 39 ++++ tests/test_populations.py | 77 +++++++ 9 files changed, 564 insertions(+), 142 deletions(-) create mode 100644 app/data/geonames_population_mappings.json create mode 100644 app/io.py create mode 100644 tests/test_io.py create mode 100644 tests/test_populations.py diff --git a/Pipfile b/Pipfile index b337c22a..9a0839af 100644 --- a/Pipfile +++ b/Pipfile @@ -10,22 +10,23 @@ asyncmock = "*" bandit = "*" black = "==19.10b0" coveralls = "*" -importlib-metadata = {version="*", markers="python_version<'3.8'"} +importlib-metadata = {version = "*",markers = "python_version<'3.8'"} invoke = "*" isort = "*" pylint = "*" pytest = "*" pytest-asyncio = "*" pytest-cov = "*" +responses = "*" [packages] aiohttp = "*" asyncache = "*" cachetools = "*" -dataclasses = {version="*", markers="python_version<'3.7'"} +dataclasses = {version = "*",markers = "python_version<'3.7'"} fastapi = "*" gunicorn = "*" -idna_ssl = {version="*", markers="python_version<'3.7'"} +idna_ssl = {version = "*",markers = "python_version<'3.7'"} python-dateutil = "*" python-dotenv = "*" requests = "*" diff --git a/Pipfile.lock b/Pipfile.lock index a699f880..9ac79d0f 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1911b081cecdda482b2a9c7c03ebba985c447846506b607df01563600c23126b" + "sha256": "9c469c96db1ae3a7e4c239d3a9c7028ecf49a0ab5e3ea50aed304ea2ab1a113e" }, "pipfile-spec": 6, "requires": { @@ -57,18 +57,18 @@ }, "cachetools": { "hashes": [ - "sha256:9a52dd97a85f257f4e4127f15818e71a0c7899f121b34591fcc1173ea79a0198", - "sha256:b304586d357c43221856be51d73387f93e2a961598a9b6b6670664746f3b6c6c" + "sha256:1d057645db16ca7fe1f3bd953558897603d6f0b9c51ed9d11eb4d071ec4e2aab", + "sha256:de5d88f87781602201cde465d3afe837546663b168e8b39df67411b0bf10cefc" ], "index": "pypi", - "version": "==4.0.0" + "version": "==4.1.0" }, "certifi": { "hashes": [ - "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", - "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" + "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", + "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" ], - "version": "==2019.11.28" + "version": "==2020.4.5.1" }, "chardet": { "hashes": [ @@ -95,11 +95,11 @@ }, "fastapi": { "hashes": [ - "sha256:a5cb9100d5f2b5dd82addbc2cdf8009258bce45b03ba21d3f5eecc88c7b5a716", - "sha256:cf26d47ede6bc6e179df951312f55fea7d4005dd53370245e216436ca4e22f22" + "sha256:1ee9a49f28d510b62b3b51a9452b274853bfc9c5d4b947ed054366e2d49f9efa", + "sha256:72f40f47e5235cb5cbbad1d4f97932ede6059290c07e12e9784028dcd1063d28" ], "index": "pypi", - "version": "==0.53.2" + "version": "==0.54.1" }, "gunicorn": { "hashes": [ @@ -173,22 +173,25 @@ }, "pydantic": { "hashes": [ - "sha256:012c422859bac2e03ab3151ea6624fecf0e249486be7eb8c6ee69c91740c6752", - "sha256:07911aab70f3bc52bb845ce1748569c5e70478ac977e106a150dd9d0465ebf04", - "sha256:47b8db7024ba3d46c3d4768535e1cf87b6c8cf92ccd81e76f4e1cb8ee47688b3", - "sha256:50e4e948892a6815649ad5a9a9379ad1e5f090f17842ac206535dfaed75c6f2f", - "sha256:51f11c8bbf794a68086540da099aae4a9107447c7a9d63151edbb7d50110cf21", - "sha256:6100d7862371115c40be55cc4b8d766a74b1d0dbaf99dbfe72bb4bac0faf89ed", - "sha256:61d22d36808087d3184ed6ac0d91dd71c533b66addb02e4a9930e1e30833202f", - "sha256:72184c1421103cca128300120f8f1185fb42a9ea73a1c9845b1c53db8c026a7d", - "sha256:831a0265a9e3933b3d0f04d1a81bba543bafbe4119c183ff2771871db70524ab", - "sha256:8848b4eb458469739126e4c1a202d723dd092e087f8dbe3104371335f87ba5df", - "sha256:bbbed364376f4a0aebb9ea452ff7968b306499a9e74f4db69b28ff2cd4043a11", - "sha256:e27559cedbd7f59d2375bfd6eea29a330ea1a5b0589c34d6b4e0d7bec6027bbf", - "sha256:f17ec336e64d4583311249fb179528e9a2c27c8a2eaf590ec6ec2c6dece7cb3f", - "sha256:f863456d3d4bf817f2e5248553dee3974c5dc796f48e6ddb599383570f4215ac" - ], - "version": "==1.4" + "sha256:0b7aadfa1de28057656064e04d9f018d1b186fe2a8e953a2fb41545873b7cf95", + "sha256:0f61e67291b99a927816558a218a4e794db72a33621c836e63d12613a2202cd4", + "sha256:20946280c750753b3e3177c748825ef189d7ab86c514f6a0b118621110d5f0d3", + "sha256:22139ee446992c222977ac0a9269c4da2e9ecc1834f84804ebde008a4649b929", + "sha256:3c0f39e884d7a3572d5cc8322b0fe9bf66114283e22e05a5c4b8961c19588945", + "sha256:446ce773a552a2cb90065d4aa645e16fa7494369b5f0d199e4d41a992a98204d", + "sha256:475e6606873e40717cc3b0eebc7d1101cbfc774e01dadeeea24c121eb5826b86", + "sha256:66124752662de0479a9d0c17bdebdc8a889bccad8846626fb66d8669e8eafb63", + "sha256:896637b7d8e4cdc0bcee1704fcadacdd167c35ac29f02a4395fce7a033925f26", + "sha256:9af44d06db33896a2176603c9cb876df3a60297a292a24d3018956a910cc1402", + "sha256:9e46fac8a4674db0777fd0133aa56817e1481beee50971bab39dded7639f9b2b", + "sha256:ae206e103e976c40ec294cd6c8fcbfbdaced3ab9b736bc53d03fa11b5aaa1628", + "sha256:b11d0bd7ecf41098894e8777ee623c29554dbaa37e862c51bcc5a2b950d1bf77", + "sha256:d73070028f7b046a5b2e611a9799c238d7bd245f8fe30f4ad7ff29ddb63aac40", + "sha256:ddedcdf9d5c24939578449a8e099ceeec3b3d76243fc143aff63ebf6d5aade10", + "sha256:e08e21f4d5395ac17cde19de26be63fb16fb870f0cfde1481ddc22d5e2353548", + "sha256:e6239199b363bc53262bcb57f1441206d4b2d46b392eccba2213d8358d6e284a" + ], + "version": "==1.5" }, "python-dateutil": { "hashes": [ @@ -200,11 +203,11 @@ }, "python-dotenv": { "hashes": [ - "sha256:81822227f771e0cab235a2939f0f265954ac4763cafd806d845801c863bf372f", - "sha256:92b3123fb2d58a284f76cc92bfe4ee6c502c32ded73e8b051c4f6afc8b6751ed" + "sha256:25c0ff1a3e12f4bde8d592cc254ab075cfe734fc5dd989036716fd17ee7e5ec7", + "sha256:3b9909bc96b0edc6b01586e1eed05e71174ef4e04c71da5786370cebea53ad74" ], "index": "pypi", - "version": "==0.12.0" + "version": "==0.13.0" }, "requests": { "hashes": [ @@ -230,10 +233,10 @@ }, "urllib3": { "hashes": [ - "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", - "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" + "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", + "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" ], - "version": "==1.25.8" + "version": "==1.25.9" }, "uvicorn": { "hashes": [ @@ -371,10 +374,10 @@ }, "certifi": { "hashes": [ - "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", - "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" + "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", + "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" ], - "version": "==2019.11.28" + "version": "==2020.4.5.1" }, "chardet": { "hashes": [ @@ -392,47 +395,47 @@ }, "coverage": { "hashes": [ - "sha256:03f630aba2b9b0d69871c2e8d23a69b7fe94a1e2f5f10df5049c0df99db639a0", - "sha256:046a1a742e66d065d16fb564a26c2a15867f17695e7f3d358d7b1ad8a61bca30", - "sha256:0a907199566269e1cfa304325cc3b45c72ae341fbb3253ddde19fa820ded7a8b", - "sha256:165a48268bfb5a77e2d9dbb80de7ea917332a79c7adb747bd005b3a07ff8caf0", - "sha256:1b60a95fc995649464e0cd48cecc8288bac5f4198f21d04b8229dc4097d76823", - "sha256:1f66cf263ec77af5b8fe14ef14c5e46e2eb4a795ac495ad7c03adc72ae43fafe", - "sha256:2e08c32cbede4a29e2a701822291ae2bc9b5220a971bba9d1e7615312efd3037", - "sha256:3844c3dab800ca8536f75ae89f3cf566848a3eb2af4d9f7b1103b4f4f7a5dad6", - "sha256:408ce64078398b2ee2ec08199ea3fcf382828d2f8a19c5a5ba2946fe5ddc6c31", - "sha256:443be7602c790960b9514567917af538cac7807a7c0c0727c4d2bbd4014920fd", - "sha256:4482f69e0701139d0f2c44f3c395d1d1d37abd81bfafbf9b6efbe2542679d892", - "sha256:4a8a259bf990044351baf69d3b23e575699dd60b18460c71e81dc565f5819ac1", - "sha256:513e6526e0082c59a984448f4104c9bf346c2da9961779ede1fc458e8e8a1f78", - "sha256:5f587dfd83cb669933186661a351ad6fc7166273bc3e3a1531ec5c783d997aac", - "sha256:62061e87071497951155cbccee487980524d7abea647a1b2a6eb6b9647df9006", - "sha256:641e329e7f2c01531c45c687efcec8aeca2a78a4ff26d49184dce3d53fc35014", - "sha256:65a7e00c00472cd0f59ae09d2fb8a8aaae7f4a0cf54b2b74f3138d9f9ceb9cb2", - "sha256:6ad6ca45e9e92c05295f638e78cd42bfaaf8ee07878c9ed73e93190b26c125f7", - "sha256:73aa6e86034dad9f00f4bbf5a666a889d17d79db73bc5af04abd6c20a014d9c8", - "sha256:7c9762f80a25d8d0e4ab3cb1af5d9dffbddb3ee5d21c43e3474c84bf5ff941f7", - "sha256:85596aa5d9aac1bf39fe39d9fa1051b0f00823982a1de5766e35d495b4a36ca9", - "sha256:86a0ea78fd851b313b2e712266f663e13b6bc78c2fb260b079e8b67d970474b1", - "sha256:8a620767b8209f3446197c0e29ba895d75a1e272a36af0786ec70fe7834e4307", - "sha256:922fb9ef2c67c3ab20e22948dcfd783397e4c043a5c5fa5ff5e9df5529074b0a", - "sha256:9fad78c13e71546a76c2f8789623eec8e499f8d2d799f4b4547162ce0a4df435", - "sha256:a37c6233b28e5bc340054cf6170e7090a4e85069513320275a4dc929144dccf0", - "sha256:c3fc325ce4cbf902d05a80daa47b645d07e796a80682c1c5800d6ac5045193e5", - "sha256:cda33311cb9fb9323958a69499a667bd728a39a7aa4718d7622597a44c4f1441", - "sha256:db1d4e38c9b15be1521722e946ee24f6db95b189d1447fa9ff18dd16ba89f732", - "sha256:eda55e6e9ea258f5e4add23bcf33dc53b2c319e70806e180aecbff8d90ea24de", - "sha256:f372cdbb240e09ee855735b9d85e7f50730dcfb6296b74b95a3e5dea0615c4c1" - ], - "version": "==5.0.4" + "sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a", + "sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355", + "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65", + "sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7", + "sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9", + "sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1", + "sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0", + "sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55", + "sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c", + "sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6", + "sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef", + "sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019", + "sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e", + "sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0", + "sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf", + "sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24", + "sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2", + "sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c", + "sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4", + "sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0", + "sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd", + "sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04", + "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e", + "sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730", + "sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2", + "sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768", + "sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796", + "sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7", + "sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a", + "sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489", + "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052" + ], + "version": "==5.1" }, "coveralls": { "hashes": [ - "sha256:4b6bfc2a2a77b890f556bc631e35ba1ac21193c356393b66c84465c06218e135", - "sha256:67188c7ec630c5f708c31552f2bcdac4580e172219897c4136504f14b823132f" + "sha256:41bd57b60321dfd5b56e990ab3f7ed876090691c21a9e3b005e1f6e42e6ba4b9", + "sha256:d213f5edd49053d03f0db316ccabfe17725f2758147afc9a37eaca9d8e8602b5" ], "index": "pypi", - "version": "==1.11.1" + "version": "==2.0.0" }, "docopt": { "hashes": [ @@ -442,17 +445,17 @@ }, "gitdb": { "hashes": [ - "sha256:284a6a4554f954d6e737cddcff946404393e030b76a282c6640df8efd6b3da5e", - "sha256:598e0096bb3175a0aab3a0b5aedaa18a9a25c6707e0eca0695ba1a0baf1b2150" + "sha256:6f0ecd46f99bb4874e5678d628c3a198e2b4ef38daea2756a2bfd8df7dd5c1a5", + "sha256:ba1132c0912e8c917aa8aa990bee26315064c7b7f171ceaaac0afeb1dc656c6a" ], - "version": "==4.0.2" + "version": "==4.0.4" }, "gitpython": { "hashes": [ - "sha256:43da89427bdf18bf07f1164c6d415750693b4d50e28fc9b68de706245147b9dd", - "sha256:e426c3b587bd58c482f0b7fe6145ff4ac7ae6c82673fc656f489719abca6f4cb" + "sha256:6d4f10e2aaad1864bb0f17ec06a2c2831534140e5883c350d58b4e85189dab74", + "sha256:71b8dad7409efbdae4930f2b0b646aaeccce292484ffa0bc74f1195582578b3d" ], - "version": "==3.1.0" + "version": "==3.1.1" }, "idna": { "hashes": [ @@ -565,17 +568,17 @@ }, "pathspec": { "hashes": [ - "sha256:163b0632d4e31cef212976cf57b43d9fd6b0bac6e67c26015d611a647d5e7424", - "sha256:562aa70af2e0d434367d9790ad37aed893de47f1693e4201fd1d3dca15d19b96" + "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0", + "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061" ], - "version": "==0.7.0" + "version": "==0.8.0" }, "pbr": { "hashes": [ - "sha256:139d2625547dbfa5fb0b81daebb39601c478c21956dc57e2e07b74450a8c506b", - "sha256:61aa52a0f18b71c5cc58232d2cf8f8d09cd67fcad60b742a60124cb8d6951488" + "sha256:07f558fece33b05caf857474a366dfcc00562bca13dd8b47b2b3e22d9f9bf55c", + "sha256:579170e23f8e0c2f24b0de612f71f648eccb79fb1322c814ae6b3c07b5ba23e8" ], - "version": "==5.4.4" + "version": "==5.4.5" }, "pluggy": { "hashes": [ @@ -601,10 +604,10 @@ }, "pyparsing": { "hashes": [ - "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", - "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "version": "==2.4.6" + "version": "==2.4.7" }, "pytest": { "hashes": [ @@ -648,29 +651,29 @@ }, "regex": { "hashes": [ - "sha256:01b2d70cbaed11f72e57c1cfbaca71b02e3b98f739ce33f5f26f71859ad90431", - "sha256:046e83a8b160aff37e7034139a336b660b01dbfe58706f9d73f5cdc6b3460242", - "sha256:113309e819634f499d0006f6200700c8209a2a8bf6bd1bdc863a4d9d6776a5d1", - "sha256:200539b5124bc4721247a823a47d116a7a23e62cc6695744e3eb5454a8888e6d", - "sha256:25f4ce26b68425b80a233ce7b6218743c71cf7297dbe02feab1d711a2bf90045", - "sha256:269f0c5ff23639316b29f31df199f401e4cb87529eafff0c76828071635d417b", - "sha256:5de40649d4f88a15c9489ed37f88f053c15400257eeb18425ac7ed0a4e119400", - "sha256:7f78f963e62a61e294adb6ff5db901b629ef78cb2a1cfce3cf4eeba80c1c67aa", - "sha256:82469a0c1330a4beb3d42568f82dffa32226ced006e0b063719468dcd40ffdf0", - "sha256:8c2b7fa4d72781577ac45ab658da44c7518e6d96e2a50d04ecb0fd8f28b21d69", - "sha256:974535648f31c2b712a6b2595969f8ab370834080e00ab24e5dbb9d19b8bfb74", - "sha256:99272d6b6a68c7ae4391908fc15f6b8c9a6c345a46b632d7fdb7ef6c883a2bbb", - "sha256:9b64a4cc825ec4df262050c17e18f60252cdd94742b4ba1286bcfe481f1c0f26", - "sha256:9e9624440d754733eddbcd4614378c18713d2d9d0dc647cf9c72f64e39671be5", - "sha256:9ff16d994309b26a1cdf666a6309c1ef51ad4f72f99d3392bcd7b7139577a1f2", - "sha256:b33ebcd0222c1d77e61dbcd04a9fd139359bded86803063d3d2d197b796c63ce", - "sha256:bba52d72e16a554d1894a0cc74041da50eea99a8483e591a9edf1025a66843ab", - "sha256:bed7986547ce54d230fd8721aba6fd19459cdc6d315497b98686d0416efaff4e", - "sha256:c7f58a0e0e13fb44623b65b01052dae8e820ed9b8b654bb6296bc9c41f571b70", - "sha256:d58a4fa7910102500722defbde6e2816b0372a4fcc85c7e239323767c74f5cbc", - "sha256:f1ac2dc65105a53c1c2d72b1d3e98c2464a133b4067a51a3d2477b28449709a0" - ], - "version": "==2020.2.20" + "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b", + "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8", + "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3", + "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e", + "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683", + "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1", + "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142", + "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3", + "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468", + "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e", + "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3", + "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a", + "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f", + "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6", + "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156", + "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b", + "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db", + "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd", + "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a", + "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948", + "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89" + ], + "version": "==2020.4.4" }, "requests": { "hashes": [ @@ -680,6 +683,14 @@ "index": "pypi", "version": "==2.23.0" }, + "responses": { + "hashes": [ + "sha256:0474ce3c897fbcc1aef286117c93499882d5c440f06a805947e4b1cb5ab3d474", + "sha256:f83613479a021e233e82d52ffb3e2e0e2836d24b0cc88a0fa31978789f78d0e5" + ], + "index": "pypi", + "version": "==0.10.12" + }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", @@ -689,10 +700,10 @@ }, "smmap": { "hashes": [ - "sha256:171484fe62793e3626c8b05dd752eb2ca01854b0c55a1efc0dc4210fccb65446", - "sha256:5fead614cf2de17ee0707a8c6a5f2aa5a2fc6c698c70993ba42f515485ffda78" + "sha256:52ea78b3e708d2c2b0cfe93b6fc3fbeec53db913345c26be6ed84c11ed8bebc1", + "sha256:b46d3fc69ba5f367df96d91f8271e8ad667a198d5a28e215a6c3d9acd133a911" ], - "version": "==3.0.1" + "version": "==3.0.2" }, "stevedore": { "hashes": [ @@ -736,10 +747,10 @@ }, "urllib3": { "hashes": [ - "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", - "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" + "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", + "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" ], - "version": "==1.25.8" + "version": "==1.25.9" }, "wcwidth": { "hashes": [ diff --git a/app/data/geonames_population_mappings.json b/app/data/geonames_population_mappings.json new file mode 100644 index 00000000..7b293caa --- /dev/null +++ b/app/data/geonames_population_mappings.json @@ -0,0 +1,252 @@ +{ + "AD": 77006, + "AE": 9630959, + "AF": 37172386, + "AG": 96286, + "AI": 13254, + "AL": 2866376, + "AM": 2951776, + "AO": 30809762, + "AQ": null, + "AR": 44494502, + "AS": 55465, + "AT": 8847037, + "AU": 24992369, + "AW": 105845, + "AX": 26711, + "AZ": 9942334, + "BA": 3323929, + "BB": 286641, + "BD": 161356039, + "BE": 11422068, + "BF": 19751535, + "BG": 7000039, + "BH": 1569439, + "BI": 11175378, + "BJ": 11485048, + "BL": 8450, + "BM": 63968, + "BN": 428962, + "BO": 11353142, + "BQ": 18012, + "BR": 209469333, + "BS": 385640, + "BT": 754394, + "BV": null, + "BW": 2254126, + "BY": 9485386, + "BZ": 383071, + "CA": 37058856, + "CC": 628, + "CD": 84068091, + "CF": 4666377, + "CG": 5244363, + "CH": 8516543, + "CI": 25069229, + "CK": 21388, + "CL": 18729160, + "CM": 25216237, + "CN": 1392730000, + "CO": 49648685, + "CR": 4999441, + "CU": 11338138, + "CV": 543767, + "CW": 159849, + "CX": 1500, + "CY": 1189265, + "CZ": 10625695, + "DE": 82927922, + "DJ": 958920, + "DK": 5797446, + "DM": 71625, + "DO": 10627165, + "DZ": 42228429, + "EC": 17084357, + "EE": 1320884, + "EG": 98423595, + "EH": 273008, + "ER": null, + "ES": 46723749, + "ET": 109224559, + "FI": 5518050, + "FJ": 883483, + "FK": 2638, + "FM": 112640, + "FO": 48497, + "FR": 66987244, + "GA": 2119275, + "GB": 66488991, + "GD": 111454, + "GE": 3731000, + "GF": 195506, + "GG": 65228, + "GH": 29767108, + "GI": 33718, + "GL": 56025, + "GM": 2280102, + "GN": 12414318, + "GP": 443000, + "GQ": 1308974, + "GR": 10727668, + "GS": 30, + "GT": 17247807, + "GU": 165768, + "GW": 1874309, + "GY": 779004, + "HK": 7451000, + "HM": null, + "HN": 9587522, + "HR": 4089400, + "HT": 11123176, + "HU": 9768785, + "ID": 267663435, + "IE": 4853506, + "IL": 8883800, + "IM": 84077, + "IN": 1352617328, + "IO": 4000, + "IQ": 38433600, + "IR": 81800269, + "IS": 353574, + "IT": 60431283, + "JE": 90812, + "JM": 2934855, + "JO": 9956011, + "JP": 126529100, + "KE": 51393010, + "KG": 6315800, + "KH": 16249798, + "KI": 115847, + "KM": 832322, + "KN": 52441, + "KP": 25549819, + "KR": 51635256, + "KW": 4137309, + "KY": 64174, + "KZ": 18276499, + "LA": 7061507, + "LB": 6848925, + "LC": 181889, + "LI": 37910, + "LK": 21670000, + "LR": 4818977, + "LS": 2108132, + "LT": 2789533, + "LU": 607728, + "LV": 1926542, + "LY": 6678567, + "MA": 36029138, + "MC": 38682, + "MD": 3545883, + "ME": 622345, + "MF": 37264, + "MG": 26262368, + "MH": 58413, + "MK": 2082958, + "ML": 19077690, + "MM": 53708395, + "MN": 3170208, + "MO": 631636, + "MP": 56882, + "MQ": 432900, + "MR": 4403319, + "MS": 9341, + "MT": 483530, + "MU": 1265303, + "MV": 515696, + "MW": 17563749, + "MX": 126190788, + "MY": 31528585, + "MZ": 29495962, + "NA": 2448255, + "NC": 284060, + "NE": 22442948, + "NF": 1828, + "NG": 195874740, + "NI": 6465513, + "NL": 17231017, + "NO": 5314336, + "NP": 28087871, + "NR": 12704, + "NU": 2166, + "NZ": 4885500, + "OM": 4829483, + "PA": 4176873, + "PE": 31989256, + "PF": 277679, + "PG": 8606316, + "PH": 106651922, + "PK": 212215030, + "PL": 37978548, + "PM": 7012, + "PN": 46, + "PR": 3195153, + "PS": 4569087, + "PT": 10281762, + "PW": 17907, + "PY": 6956071, + "QA": 2781677, + "RE": 776948, + "RO": 19473936, + "RS": 6982084, + "RU": 144478050, + "RW": 12301939, + "SA": 33699947, + "SB": 652858, + "SC": 96762, + "SD": 41801533, + "SE": 10183175, + "SG": 5638676, + "SH": 7460, + "SI": 2067372, + "SJ": 2550, + "SK": 5447011, + "SL": 7650154, + "SM": 33785, + "SN": 15854360, + "SO": 15008154, + "SR": 575991, + "SS": 8260490, + "ST": 197700, + "SV": 6420744, + "SX": 40654, + "SY": 16906283, + "SZ": 1136191, + "TC": 37665, + "TD": 15477751, + "TF": 140, + "TG": 7889094, + "TH": 69428524, + "TJ": 9100837, + "TK": 1466, + "TL": 1267972, + "TM": 5850908, + "TN": 11565204, + "TO": 103197, + "TR": 82319724, + "TT": 1389858, + "TV": 11508, + "TW": 22894384, + "TZ": 56318348, + "UA": 44622516, + "UG": 42723139, + "UM": null, + "US": 327167434, + "UY": 3449299, + "UZ": 32955400, + "VA": 921, + "VC": 110211, + "VE": 28870195, + "VG": 29802, + "VI": 106977, + "VN": 95540395, + "VU": 292680, + "WF": 16025, + "WS": 196130, + "XK": 1845300, + "YE": 28498687, + "YT": 159042, + "ZA": 57779622, + "ZM": 17351822, + "ZW": 14439018 +} \ No newline at end of file diff --git a/app/io.py b/app/io.py new file mode 100644 index 00000000..8130c146 --- /dev/null +++ b/app/io.py @@ -0,0 +1,28 @@ +"""app.io.py""" +import json +import pathlib +from typing import Dict, Union + +HERE = pathlib.Path(__file__) +DATA = HERE.joinpath("..", "data").resolve() + + +def save( + name: str, content: Union[str, Dict], write_mode: str = "w", indent: int = 2, **json_dumps_kwargs +) -> pathlib.Path: + """Save content to a file. If content is a dictionary, use json.dumps().""" + path = DATA / name + if isinstance(content, dict): + content = json.dumps(content, indent=indent, **json_dumps_kwargs) + with open(DATA / name, mode=write_mode) as f_out: + f_out.write(content) + return path + + +def load(name: str, **json_kwargs) -> Union[str, Dict]: + """Loads content from a file. If file ends with '.json', call json.load() and return a Dictionary.""" + path = DATA / name + with open(path) as f_in: + if path.suffix == ".json": + return json.load(f_in, **json_kwargs) + return f_in.read() diff --git a/app/utils/populations.py b/app/utils/populations.py index 1d8bd843..f5235fd4 100644 --- a/app/utils/populations.py +++ b/app/utils/populations.py @@ -1,16 +1,23 @@ """app.utils.populations.py""" +import json import logging import requests +import app.io + LOGGER = logging.getLogger(__name__) +GEONAMES_URL = "http://api.geonames.org/countryInfoJSON" +GEONAMES_BACKUP_PATH = "geonames_population_mappings.json" # Fetching of the populations. -def fetch_populations(): +def fetch_populations(save=False): """ Returns a dictionary containing the population of each country fetched from the GeoNames. https://www.geonames.org/ + TODO: only skip writing to the filesystem when deployed with gunicorn, or handle concurent access, or use DB. + :returns: The mapping of populations. :rtype: dict """ @@ -20,12 +27,18 @@ def fetch_populations(): mappings = {} # Fetch the countries. - countries = requests.get("http://api.geonames.org/countryInfoJSON?username=dperic").json()["geonames"] - - # Go through all the countries and perform the mapping. - for country in countries: - mappings.update({country["countryCode"]: int(country["population"]) or None}) - + try: + countries = requests.get(GEONAMES_URL, params={"username": "dperic"}, timeout=1.5).json()["geonames"] + # Go through all the countries and perform the mapping. + for country in countries: + mappings.update({country["countryCode"]: int(country["population"]) or None}) + + if mappings and save: + LOGGER.info(f"Saving population data to {app.io.save(GEONAMES_BACKUP_PATH, mappings)}") + except (json.JSONDecodeError, KeyError, requests.exceptions.Timeout) as err: + LOGGER.warning(f"Error pulling population data. {err.__class__.__name__}: {err}") + mappings = app.io.load(GEONAMES_BACKUP_PATH) + LOGGER.info(f"Using backup data from {GEONAMES_BACKUP_PATH}") # Finally, return the mappings. return mappings diff --git a/requirements-dev.txt b/requirements-dev.txt index e85f4e9c..374fb37c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,14 +7,14 @@ asyncmock==0.4.2 attrs==19.3.0 bandit==1.6.2 black==19.10b0 -certifi==2019.11.28 +certifi==2020.4.5.1 chardet==3.0.4 click==7.1.1 -coverage==5.0.4 -coveralls==1.11.1 +coverage==5.1 +coveralls==2.0.0 docopt==0.6.2 -gitdb==4.0.2 -gitpython==3.1.0 +gitdb==4.0.4 +gitpython==3.1.1 idna==2.9 importlib-metadata==1.6.0 ; python_version < '3.8' invoke==1.4.1 @@ -25,24 +25,25 @@ mock==4.0.2 more-itertools==8.2.0 multidict==4.7.5 packaging==20.3 -pathspec==0.7.0 -pbr==5.4.4 +pathspec==0.8.0 +pbr==5.4.5 pluggy==0.13.1 py==1.8.1 pylint==2.4.4 -pyparsing==2.4.6 +pyparsing==2.4.7 pytest-asyncio==0.10.0 pytest-cov==2.8.1 pytest==5.4.1 pyyaml==5.3.1 -regex==2020.2.20 +regex==2020.4.4 requests==2.23.0 +responses==0.10.12 six==1.14.0 -smmap==3.0.1 +smmap==3.0.2 stevedore==1.32.0 toml==0.10.0 typed-ast==1.4.1 -urllib3==1.25.8 +urllib3==1.25.9 wcwidth==0.1.9 wrapt==1.11.2 zipp==3.1.0 diff --git a/requirements.txt b/requirements.txt index 0d7a2c46..bb9302b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,25 +3,25 @@ aiohttp==3.6.2 async-timeout==3.0.1 asyncache==0.1.1 attrs==19.3.0 -cachetools==4.0.0 -certifi==2019.11.28 +cachetools==4.1.0 +certifi==2020.4.5.1 chardet==3.0.4 click==7.1.1 dataclasses==0.6 ; python_version < '3.7' -fastapi==0.53.2 +fastapi==0.54.1 gunicorn==20.0.4 h11==0.9.0 httptools==0.1.1 ; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy' idna-ssl==1.1.0 ; python_version < '3.7' idna==2.9 multidict==4.7.5 -pydantic==1.4 +pydantic==1.5 python-dateutil==2.8.1 -python-dotenv==0.12.0 +python-dotenv==0.13.0 requests==2.23.0 six==1.14.0 starlette==0.13.2 -urllib3==1.25.8 +urllib3==1.25.9 uvicorn==0.11.3 uvloop==0.14.0 ; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy' websockets==8.1 diff --git a/tests/test_io.py b/tests/test_io.py new file mode 100644 index 00000000..83639cc9 --- /dev/null +++ b/tests/test_io.py @@ -0,0 +1,39 @@ +"""test.test_io.py""" +import string + +import pytest + +import app.io + + +@pytest.mark.parametrize( + "name, content, kwargs", + [ + ("test_file.txt", string.ascii_lowercase, {}), + ("test_json_file.json", {"a": 0, "b": 1, "c": 2}, {}), + ("test_custom_json.json", {"z": -1, "b": 1, "y": -2, "a": 0}, {"indent": 4, "sort_keys": True}), + ], +) +def test_save(tmp_path, name, content, kwargs): + test_path = tmp_path / name + assert not test_path.exists() + + result = app.io.save(test_path, content, **kwargs) + assert result == test_path + assert test_path.exists() + + +@pytest.mark.parametrize( + "name, content, kwargs", + [ + ("test_file.txt", string.ascii_lowercase, {}), + ("test_json_file.json", {"a": 0, "b": 1, "c": 2}, {}), + ("test_custom_json.json", {"z": -1, "b": 1, "y": -2, "a": 0}, {"indent": 4, "sort_keys": True}), + ], +) +def test_round_trip(tmp_path, name, content, kwargs): + test_path = tmp_path / name + assert not test_path.exists() + + app.io.save(test_path, content, **kwargs) + assert app.io.load(test_path) == content diff --git a/tests/test_populations.py b/tests/test_populations.py new file mode 100644 index 00000000..97a33f66 --- /dev/null +++ b/tests/test_populations.py @@ -0,0 +1,77 @@ +"""tests.test_populations.py""" +import pytest +import requests.exceptions +import responses + +import app.io +import app.utils.populations + +NOT_FOUND_HTML = """ + + + +Error + + +
Not Found
+ +""" + +SAMPLE_GEONAMES_JSON = { + "geonames": [ + { + "continent": "EU", + "capital": "Andorra la Vella", + "languages": "ca", + "geonameId": 3041565, + "south": 42.42874300100004, + "isoAlpha3": "AND", + "north": 42.65576500000003, + "fipsCode": "AN", + "population": "77006", + "east": 1.786576000000025, + "isoNumeric": "020", + "areaInSqKm": "468.0", + "countryCode": "AD", + "west": 1.413760001000071, + "countryName": "Andorra", + "continentName": "Europe", + "currencyCode": "EUR", + }, + { + "continent": "AS", + "capital": "Abu Dhabi", + "languages": "ar-AE,fa,en,hi,ur", + "geonameId": 290557, + "south": 22.6315119400001, + "isoAlpha3": "ARE", + "north": 26.0693916590001, + "fipsCode": "AE", + "population": "9630959", + "east": 56.381222289, + "isoNumeric": "784", + "areaInSqKm": "82880.0", + "countryCode": "AE", + "west": 51.5904085340001, + "countryName": "United Arab Emirates", + "continentName": "Asia", + "currencyCode": "AED", + }, + ] +} + + +@responses.activate +@pytest.mark.parametrize( + "body_arg, json_arg", + [ + (None, SAMPLE_GEONAMES_JSON), + (NOT_FOUND_HTML, None), + (None, {"foo": "bar"}), + (requests.exceptions.Timeout("Forced Timeout"), None), + ], +) +def test_fetch_populations(body_arg, json_arg): + responses.add(responses.GET, app.utils.populations.GEONAMES_URL, body=body_arg, json=json_arg) + + assert app.utils.populations.fetch_populations() From 39b445785b31b83fb432f83185b09d6d3531c3de Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 17 Apr 2020 23:30:16 -0400 Subject: [PATCH 07/12] merge model modules --- app/{models/location.py => models.py} | 39 +++++++++++++++++++++++++-- app/models/latest.py | 19 ------------- app/models/timeline.py | 22 --------------- 3 files changed, 37 insertions(+), 43 deletions(-) rename app/{models/location.py => models.py} (56%) delete mode 100644 app/models/latest.py delete mode 100644 app/models/timeline.py diff --git a/app/models/location.py b/app/models.py similarity index 56% rename from app/models/location.py rename to app/models.py index 48fa4d74..8875a92c 100644 --- a/app/models/location.py +++ b/app/models.py @@ -1,9 +1,44 @@ +"""app.models.py""" from typing import Dict, List from pydantic import BaseModel -from .latest import Latest -from .timeline import Timelines + +class Latest(BaseModel): + """ + Latest model. + """ + + confirmed: int + deaths: int + recovered: int + + +class LatestResponse(BaseModel): + """ + Response for latest. + """ + + latest: Latest + + +class Timeline(BaseModel): + """ + Timeline model. + """ + + latest: int + timeline: Dict[str, int] = {} + + +class Timelines(BaseModel): + """ + Timelines model. + """ + + confirmed: Timeline + deaths: Timeline + recovered: Timeline class Location(BaseModel): diff --git a/app/models/latest.py b/app/models/latest.py deleted file mode 100644 index 6dcfd517..00000000 --- a/app/models/latest.py +++ /dev/null @@ -1,19 +0,0 @@ -from pydantic import BaseModel - - -class Latest(BaseModel): - """ - Latest model. - """ - - confirmed: int - deaths: int - recovered: int - - -class LatestResponse(BaseModel): - """ - Response for latest. - """ - - latest: Latest diff --git a/app/models/timeline.py b/app/models/timeline.py deleted file mode 100644 index 453dfb14..00000000 --- a/app/models/timeline.py +++ /dev/null @@ -1,22 +0,0 @@ -from typing import Dict - -from pydantic import BaseModel - - -class Timeline(BaseModel): - """ - Timeline model. - """ - - latest: int - timeline: Dict[str, int] = {} - - -class Timelines(BaseModel): - """ - Timelines model. - """ - - confirmed: Timeline - deaths: Timeline - recovered: Timeline From 6dd7c3c486b36f99893e0348e189906e008e058c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 17 Apr 2020 23:33:57 -0400 Subject: [PATCH 08/12] Combine end router modules move version 2 endpoint definitions to single module move version 1 endpoint definitions to single module --- app/main.py | 3 +- app/router/__init__.py | 8 ---- app/router/v1/__init__.py | 4 -- app/router/v1/all.py | 20 -------- app/router/v1/confirmed.py | 11 ----- app/router/v1/deaths.py | 11 ----- app/router/v1/recovered.py | 11 ----- app/router/v2/__init__.py | 4 -- app/router/v2/latest.py | 21 --------- app/router/v2/sources.py | 11 ----- app/routers/__init__.py | 3 ++ app/routers/v1.py | 47 +++++++++++++++++++ app/{router/v2/locations.py => routers/v2.py} | 40 ++++++++++++---- 13 files changed, 83 insertions(+), 111 deletions(-) delete mode 100644 app/router/__init__.py delete mode 100644 app/router/v1/__init__.py delete mode 100644 app/router/v1/all.py delete mode 100644 app/router/v1/confirmed.py delete mode 100644 app/router/v1/deaths.py delete mode 100644 app/router/v1/recovered.py delete mode 100644 app/router/v2/__init__.py delete mode 100644 app/router/v2/latest.py delete mode 100644 app/router/v2/sources.py create mode 100644 app/routers/__init__.py create mode 100644 app/routers/v1.py rename app/{router/v2/locations.py => routers/v2.py} (62%) diff --git a/app/main.py b/app/main.py index 1224c24e..173a92aa 100644 --- a/app/main.py +++ b/app/main.py @@ -12,8 +12,7 @@ from fastapi.responses import JSONResponse from .data import data_source -from .router.v1 import V1 -from .router.v2 import V2 +from .routers import V1, V2 from .utils.httputils import setup_client_session, teardown_client_session # ############ diff --git a/app/router/__init__.py b/app/router/__init__.py deleted file mode 100644 index 4eda6c21..00000000 --- a/app/router/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""app.router""" -from fastapi import APIRouter - -# pylint: disable=redefined-builtin -from .v1 import all, confirmed, deaths, recovered - -# The routes. -from .v2 import latest, sources, locations # isort:skip diff --git a/app/router/v1/__init__.py b/app/router/v1/__init__.py deleted file mode 100644 index 839bd212..00000000 --- a/app/router/v1/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""app.router.v1""" -from fastapi import APIRouter - -V1 = APIRouter() diff --git a/app/router/v1/all.py b/app/router/v1/all.py deleted file mode 100644 index 91b9e826..00000000 --- a/app/router/v1/all.py +++ /dev/null @@ -1,20 +0,0 @@ -"""app.router.v1.all.py""" -from ...services.location.jhu import get_category -from . import V1 - - -@V1.get("/all") -async def all(): # pylint: disable=redefined-builtin - """Get all the categories.""" - confirmed = await get_category("confirmed") - deaths = await get_category("deaths") - recovered = await get_category("recovered") - - return { - # Data. - "confirmed": confirmed, - "deaths": deaths, - "recovered": recovered, - # Latest. - "latest": {"confirmed": confirmed["latest"], "deaths": deaths["latest"], "recovered": recovered["latest"],}, - } diff --git a/app/router/v1/confirmed.py b/app/router/v1/confirmed.py deleted file mode 100644 index 13365e32..00000000 --- a/app/router/v1/confirmed.py +++ /dev/null @@ -1,11 +0,0 @@ -"""app.router.v1.confirmed.py""" -from ...services.location.jhu import get_category -from . import V1 - - -@V1.get("/confirmed") -async def confirmed(): - """Confirmed cases.""" - confirmed_data = await get_category("confirmed") - - return confirmed_data diff --git a/app/router/v1/deaths.py b/app/router/v1/deaths.py deleted file mode 100644 index fb45498c..00000000 --- a/app/router/v1/deaths.py +++ /dev/null @@ -1,11 +0,0 @@ -"""app.router.v1.deaths.py""" -from ...services.location.jhu import get_category -from . import V1 - - -@V1.get("/deaths") -async def deaths(): - """Total deaths.""" - deaths_data = await get_category("deaths") - - return deaths_data diff --git a/app/router/v1/recovered.py b/app/router/v1/recovered.py deleted file mode 100644 index 3a3a85b7..00000000 --- a/app/router/v1/recovered.py +++ /dev/null @@ -1,11 +0,0 @@ -"""app.router.v1.recovered.py""" -from ...services.location.jhu import get_category -from . import V1 - - -@V1.get("/recovered") -async def recovered(): - """Recovered cases.""" - recovered_data = await get_category("recovered") - - return recovered_data diff --git a/app/router/v2/__init__.py b/app/router/v2/__init__.py deleted file mode 100644 index 62c31905..00000000 --- a/app/router/v2/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""app.router.v2""" -from fastapi import APIRouter - -V2 = APIRouter() diff --git a/app/router/v2/latest.py b/app/router/v2/latest.py deleted file mode 100644 index 105b16fe..00000000 --- a/app/router/v2/latest.py +++ /dev/null @@ -1,21 +0,0 @@ -"""app.router.v2.latest.py""" -from fastapi import Request - -from ...enums.sources import Sources -from ...models.latest import LatestResponse as Latest -from . import V2 - - -@V2.get("/latest", response_model=Latest) -async def get_latest(request: Request, source: Sources = "jhu"): # pylint: disable=unused-argument - """ - Getting latest amount of total confirmed cases, deaths, and recoveries. - """ - locations = await request.state.source.get_all() - return { - "latest": { - "confirmed": sum(map(lambda location: location.confirmed, locations)), - "deaths": sum(map(lambda location: location.deaths, locations)), - "recovered": sum(map(lambda location: location.recovered, locations)), - } - } diff --git a/app/router/v2/sources.py b/app/router/v2/sources.py deleted file mode 100644 index ad906e51..00000000 --- a/app/router/v2/sources.py +++ /dev/null @@ -1,11 +0,0 @@ -"""app.router.v2.sources.py""" -from ...data import DATA_SOURCES -from . import V2 - - -@V2.get("/sources") -async def sources(): - """ - Retrieves a list of data-sources that are availble to use. - """ - return {"sources": list(DATA_SOURCES.keys())} diff --git a/app/routers/__init__.py b/app/routers/__init__.py new file mode 100644 index 00000000..2bdd5ee3 --- /dev/null +++ b/app/routers/__init__.py @@ -0,0 +1,3 @@ +"""app.routers""" +from .v1 import V1 +from .v2 import V2 diff --git a/app/routers/v1.py b/app/routers/v1.py new file mode 100644 index 00000000..65bc4373 --- /dev/null +++ b/app/routers/v1.py @@ -0,0 +1,47 @@ +"""app.router.v1.py""" +from fastapi import APIRouter + +from ..services.location.jhu import get_category + +V1 = APIRouter() + + +@V1.get("/all") +async def all_categories(): + """Get all the categories.""" + confirmed = await get_category("confirmed") + deaths = await get_category("deaths") + recovered = await get_category("recovered") + + return { + # Data. + "confirmed": confirmed, + "deaths": deaths, + "recovered": recovered, + # Latest. + "latest": {"confirmed": confirmed["latest"], "deaths": deaths["latest"], "recovered": recovered["latest"],}, + } + + +@V1.get("/confirmed") +async def get_confirmed(): + """Confirmed cases.""" + confirmed_data = await get_category("confirmed") + + return confirmed_data + + +@V1.get("/deaths") +async def get_deaths(): + """Total deaths.""" + deaths_data = await get_category("deaths") + + return deaths_data + + +@V1.get("/recovered") +async def get_recovered(): + """Recovered cases.""" + recovered_data = await get_category("recovered") + + return recovered_data diff --git a/app/router/v2/locations.py b/app/routers/v2.py similarity index 62% rename from app/router/v2/locations.py rename to app/routers/v2.py index 649f9c9e..853f0964 100644 --- a/app/router/v2/locations.py +++ b/app/routers/v2.py @@ -1,14 +1,30 @@ -"""app.router.v2.locations.py""" -from fastapi import HTTPException, Request +"""app.router.v2""" +from fastapi import APIRouter, HTTPException, Request -from ...enums.sources import Sources -from ...models.location import LocationResponse as Location -from ...models.location import LocationsResponse as Locations -from . import V2 +from ..data import DATA_SOURCES +from ..enums.sources import Sources +from ..models import LatestResponse, LocationResponse, LocationsResponse + +V2 = APIRouter() + + +@V2.get("/latest", response_model=LatestResponse) +async def get_latest(request: Request, source: Sources = "jhu"): + """ + Getting latest amount of total confirmed cases, deaths, and recoveries. + """ + locations = await request.state.source.get_all() + return { + "latest": { + "confirmed": sum(map(lambda location: location.confirmed, locations)), + "deaths": sum(map(lambda location: location.deaths, locations)), + "recovered": sum(map(lambda location: location.recovered, locations)), + } + } # pylint: disable=unused-argument,too-many-arguments,redefined-builtin -@V2.get("/locations", response_model=Locations, response_model_exclude_unset=True) +@V2.get("/locations", response_model=LocationsResponse, response_model_exclude_unset=True) async def get_locations( request: Request, source: Sources = "jhu", @@ -56,10 +72,18 @@ async def get_locations( # pylint: disable=invalid-name -@V2.get("/locations/{id}", response_model=Location) +@V2.get("/locations/{id}", response_model=LocationResponse) async def get_location_by_id(request: Request, id: int, source: Sources = "jhu", timelines: bool = True): """ Getting specific location by id. """ location = await request.state.source.get(id) return {"location": location.serialize(timelines)} + + +@V2.get("/sources") +async def sources(): + """ + Retrieves a list of data-sources that are availble to use. + """ + return {"sources": list(DATA_SOURCES.keys())} From b3ae818a1cbdcc3eacd8aed21cdbb52a2650fe1d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 09:28:21 -0400 Subject: [PATCH 09/12] remove enum subpackage --- app/enums/sources.py | 11 ----------- app/routers/v2.py | 13 ++++++++++++- 2 files changed, 12 insertions(+), 12 deletions(-) delete mode 100644 app/enums/sources.py diff --git a/app/enums/sources.py b/app/enums/sources.py deleted file mode 100644 index 9fc00744..00000000 --- a/app/enums/sources.py +++ /dev/null @@ -1,11 +0,0 @@ -from enum import Enum - - -class Sources(str, Enum): - """ - A source available for retrieving data. - """ - - jhu = "jhu" - csbs = "csbs" - nyt = "nyt" diff --git a/app/routers/v2.py b/app/routers/v2.py index 853f0964..272fffd0 100644 --- a/app/routers/v2.py +++ b/app/routers/v2.py @@ -1,13 +1,24 @@ """app.router.v2""" +import enum + from fastapi import APIRouter, HTTPException, Request from ..data import DATA_SOURCES -from ..enums.sources import Sources from ..models import LatestResponse, LocationResponse, LocationsResponse V2 = APIRouter() +class Sources(str, enum.Enum): + """ + A source available for retrieving data. + """ + + jhu = "jhu" + csbs = "csbs" + nyt = "nyt" + + @V2.get("/latest", response_model=LatestResponse) async def get_latest(request: Request, source: Sources = "jhu"): """ From 82175a72098091e7505da4969706d4eb6733473c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 11:06:12 -0400 Subject: [PATCH 10/12] add back warning suppression --- app/routers/v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/v2.py b/app/routers/v2.py index 272fffd0..a495066b 100644 --- a/app/routers/v2.py +++ b/app/routers/v2.py @@ -20,7 +20,7 @@ class Sources(str, enum.Enum): @V2.get("/latest", response_model=LatestResponse) -async def get_latest(request: Request, source: Sources = "jhu"): +async def get_latest(request: Request, source: Sources = "jhu"): # pylint: disable=unused-argument """ Getting latest amount of total confirmed cases, deaths, and recoveries. """ From 72572d058d608eb8ad0231f4fa9f1b30babb52a1 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 12:22:43 -0400 Subject: [PATCH 11/12] fix module names --- app/routers/v1.py | 2 +- app/routers/v2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/routers/v1.py b/app/routers/v1.py index 65bc4373..662514a0 100644 --- a/app/routers/v1.py +++ b/app/routers/v1.py @@ -1,4 +1,4 @@ -"""app.router.v1.py""" +"""app.routers.v1.py""" from fastapi import APIRouter from ..services.location.jhu import get_category diff --git a/app/routers/v2.py b/app/routers/v2.py index a495066b..de5a5312 100644 --- a/app/routers/v2.py +++ b/app/routers/v2.py @@ -1,4 +1,4 @@ -"""app.router.v2""" +"""app.routers.v2""" import enum from fastapi import APIRouter, HTTPException, Request From 5a243bfba7e3a3d7273d5eea74b9afed6f37cfeb Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Apr 2020 17:45:24 -0400 Subject: [PATCH 12/12] add source info to log message --- app/services/location/csbs.py | 6 +++--- app/services/location/jhu.py | 8 ++++---- app/services/location/nyt.py | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/services/location/csbs.py b/app/services/location/csbs.py index d0de3837..202546b8 100644 --- a/app/services/location/csbs.py +++ b/app/services/location/csbs.py @@ -41,11 +41,11 @@ async def get_locations(): :rtype: dict """ logger = logging.getLogger("services.location.csbs") - logger.info("Requesting data...") + logger.info("csbs Requesting data...") async with httputils.CLIENT_SESSION.get(BASE_URL) as response: text = await response.text() - logger.info("Data received") + logger.info("csbs Data received") data = list(csv.DictReader(text.splitlines())) logger.info("CSV parsed") @@ -83,7 +83,7 @@ async def get_locations(): int(item["Death"] or 0), ) ) - logger.info("Data normalized") + logger.info("csbs Data normalized") # Return the locations. return locations diff --git a/app/services/location/jhu.py b/app/services/location/jhu.py index 39e07ac2..29d7b416 100644 --- a/app/services/location/jhu.py +++ b/app/services/location/jhu.py @@ -57,15 +57,15 @@ async def get_category(category): url = BASE_URL + "time_series_covid19_%s_global.csv" % category # Request the data - logger.info("Requesting data...") + logger.info("jhu Requesting data...") async with httputils.CLIENT_SESSION.get(url) as response: text = await response.text() - logger.info("Data received") + logger.info("jhu Data received") # Parse the CSV. data = list(csv.DictReader(text.splitlines())) - logger.info("CSV parsed") + logger.info("jhu CSV parsed") # The normalized locations. locations = [] @@ -98,7 +98,7 @@ async def get_category(category): "latest": int(latest or 0), } ) - logger.info("Data normalized") + logger.info("jhu Data normalized") # Latest total. latest = sum(map(lambda location: location["latest"], locations)) diff --git a/app/services/location/nyt.py b/app/services/location/nyt.py index 7f12eb62..38ff7ffb 100644 --- a/app/services/location/nyt.py +++ b/app/services/location/nyt.py @@ -75,7 +75,7 @@ async def get_locations(): logger = logging.getLogger("services.location.nyt") # Request the data. - logger.info("Requesting data...") + logger.info("nyt Requesting data...") async with httputils.CLIENT_SESSION.get(BASE_URL) as response: text = await response.text() @@ -83,7 +83,7 @@ async def get_locations(): # Parse the CSV. data = list(csv.DictReader(text.splitlines())) - logger.info("CSV parsed") + logger.info("nyt CSV parsed") # Group together locations (NYT data ordered by dates not location). grouped_locations = get_grouped_locations_dict(data) @@ -125,6 +125,6 @@ async def get_locations(): }, ) ) - logger.info("Data normalized") + logger.info("nyt Data normalized") return locations