Merge pull request #156 from ExpDev07/update-jhu-urls

JHU updates its directory structure and drops tracking of recoveries

Author: ExpDev07

app/services/location/jhu.py

@@ -27,7 +27,7 @@ def get(self, id):
 """
 Base URL for fetching category.
 """
-base_url = 'https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-%s.csv';
+base_url = 'https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/';
 
 @cached(cache=TTLCache(maxsize=1024, ttl=3600))
 def get_category(category):
@@ -39,10 +39,20 @@ def get_category(category):
     """
 
     # Adhere to category naming standard.
-    category = category.lower().capitalize();
+    category = category.lower();
+
+    # URL to request data from.
+    url = base_url + 'time_series_covid19_%s_global.csv' % category
+
+    # Different URL is needed for recoveries.
+    # Read about deprecation here: https://github.com/CSSEGISandData/COVID-19/tree/master/csse_covid_19_data/csse_covid_19_time_series.
+    if category == 'recovered':
+        url = base_url + 'time_series_19-covid-Recovered.csv'
+
+    print (url)
 
     # Request the data
-    request = requests.get(base_url % category)
+    request = requests.get(url)
     text    = request.text
 
     # Parse the CSV.

app/timeline.py

@@ -21,7 +21,15 @@ def latest(self):
         """
         Gets the latest available history value.
         """
-        return list(self.timeline.values())[-1] or 0
+        # Get values in a list.
+        values = list(self.timeline.values())
+
+        # Last item is the latest.
+        if len(values):
+            return values[-1] or 0
+        
+        # Fallback value of 0.
+        return 0
 
     def serialize(self):
         """

…_data/time_series_19-covid-Confirmed.csv …time_series_covid19_confirmed_global.csvtests/example_data/time_series_19-covid-Confirmed.csv renamed to tests/example_data/time_series_covid19_confirmed_global.csv tests/example_data/time_series_19-covid-Confirmed.csv renamed to tests/example_data/time_series_covid19_confirmed_global.csv

@@ -24,7 +24,15 @@ def read_file(self, state):
             """
             Mock HTTP GET-method and return text from file
             """
-            filepath = "tests/example_data/time_series_19-covid-{}.csv".format(state)
+            state = state.lower()
+
+            # Determine filepath.
+            filepath = "tests/example_data/{}.csv".format(state)
+
+            if state == 'recovered':
+                filepath = 'tests/example_data/time_series_19-covid-Recovered.csv'
+
+            # Return fake response.
             print("Try to read {}".format(filepath))
             with open(filepath, "r") as file:
                 return file.read()
@@ -58,76 +66,6 @@ def isoformat(self):
 
     return DateTimeStrpTime(date, strformat)
 
-@pytest.mark.parametrize("category, capitalize_category", [
-                            ("deaths", "Deaths"),
-                            ("recovered", "Recovered"),
-                            ("confirmed", "Confirmed")])
-@mock.patch('app.services.location.jhu.requests.get', side_effect=mocked_requests_get)
-def test_validate_category(mock_request_get, category, capitalize_category):
-    base_url = 'https://raw.githubusercontent.com/CSSEGISandData/2019-nCoV/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-%s.csv'
-    request = app.services.location.jhu.requests.get(base_url % category)
-    
-    assert request.state == capitalize_category
-
-@pytest.mark.parametrize("category, datetime_str, latest_value, country_name, \
-                          country_code, province, latest_country_value, \
-                          coordinate_lat, coordinate_long",
-                         [("deaths", DATETIME_STRING, 1940, "Thailand", "TH", "",
-                           114, "15", "101"),
-                          ("recovered", DATETIME_STRING, 1940, "Thailand", "TH", "",
-                           114, "15", "101"),
-                          ("confirmed", DATETIME_STRING, 1940, "Thailand", "TH", "",
-                           114, "15", "101")])
-@mock.patch('app.services.location.jhu.datetime')
-@mock.patch('app.services.location.jhu.requests.get', side_effect=mocked_requests_get)
-def test_get_category(mock_request_get, mock_datetime, category, datetime_str,
-                      latest_value, country_name, country_code, province, latest_country_value,
-                      coordinate_lat, coordinate_long):
-    #mock app.services.location.jhu.datetime.utcnow().isoformat()
-    mock_datetime.utcnow.return_value.isoformat.return_value = datetime_str
-    output = jhu.get_category(category)
-
-    #simple schema validation
-    assert output["source"] == "https://github.com/ExpDev07/coronavirus-tracker-api"
-
-    assert isinstance(output["latest"], int)
-    assert output["latest"] == latest_value #based on example data
-
-    #check for valid datestring
-    assert date.is_date(output["last_updated"]) is True
-    #ensure date formating
-    assert output["last_updated"] == datetime_str + "Z" #based on example data
-
-    #validate location schema
-    location_entry = output["locations"][0]
-
-    assert isinstance(location_entry["country"], str)
-    assert location_entry["country"] == country_name #based on example data
-
-    assert isinstance(location_entry["country_code"], str)
-    assert len(location_entry["country_code"]) == 2
-    assert location_entry["country_code"] == country_code #based on example data
-
-    assert isinstance(location_entry["province"], str)
-    assert location_entry["province"] == province #based on example data
-
-    assert isinstance(location_entry["latest"], int)
-    assert location_entry["latest"] == latest_country_value #based on example data
-
-    #validate coordinates in location
-    coordinates = location_entry["coordinates"]
-
-    assert isinstance(coordinates["lat"], str)
-    assert coordinates["lat"] == coordinate_lat
-
-    assert isinstance(coordinates["long"], str)
-    assert coordinates["long"] == coordinate_long
-
-    #validate history in location
-    history = location_entry["history"]
-    assert date.is_date(list(history.keys())[0]) is True
-    assert isinstance(list(history.values())[0], int)
-
 @mock.patch('app.services.location.jhu.datetime')
 @mock.patch('app.services.location.jhu.requests.get', side_effect=mocked_requests_get)
 def test_get_locations(mock_request_get, mock_datetime):