Changed to use a gzipped release coverage JSON file, instead of a plain JSON file, if the name given in settings.py ends in .gz. Reduces the current release coverage file size from ~1.6MB to ~200kB. This is a preparation for saving more information in the release coverage file than we currently do.

 - Legacy-Id: 10235

Author: levkowetz

ietf/release/views.py

@@ -2,6 +2,7 @@
 import re
 import json
 import datetime
+import gzip
 
 from django.template import RequestContext
 from django.shortcuts import render_to_response
@@ -43,8 +44,12 @@ def release(request, version=None):
 
     coverage = {}
     if os.path.exists(settings.TEST_COVERAGE_MASTER_FILE):
-        with open(settings.TEST_COVERAGE_MASTER_FILE) as file:
-            coverage_data = json.load(file)
+        if settings.TEST_COVERAGE_MASTER_FILE.endswith(".gz"):
+            with gzip.open(settings.TEST_COVERAGE_MASTER_FILE, "rb") as file:
+                coverage_data = json.load(file)
+        else:
+            with open(settings.TEST_COVERAGE_MASTER_FILE) as file:
+                coverage_data = json.load(file)
         if version in coverage_data:
             coverage = coverage_data[version]
             for key in coverage:

ietf/settings.py

@@ -346,7 +346,7 @@ def skip_unreadable_post(record):
     "ietf/utils/test_runner.py",
 ]
 
-TEST_COVERAGE_MASTER_FILE = os.path.join(BASE_DIR, "../release-coverage.json")
+TEST_COVERAGE_MASTER_FILE = os.path.join(BASE_DIR, "../release-coverage.json.gz")
 TEST_COVERAGE_LATEST_FILE = os.path.join(BASE_DIR, "../latest-coverage.json")
 
 TEST_CODE_COVERAGE_CHECKER = None

ietf/utils/management/commands/coverage_changes.py

@@ -1,6 +1,7 @@
 
 import json
 import codecs
+import gzip
 from optparse import make_option
 
 from django.conf import settings
@@ -25,7 +26,10 @@ class Command(BaseCommand):
     def read_coverage(self, filename, version=None):
         if isinstance(filename, string_types):
             try:
-                file = codecs.open(filename, "r", encoding="utf-8")
+                if filename.endswith(".gz"):
+                    file = gzip.open(filename, "rb")
+                else:
+                    file = codecs.open(filename, "r", encoding="utf-8")
             except IOError as e:
                 self.stderr.write(u"%s" % e)
                 exit(1)

ietf/utils/test_runner.py

@@ -44,6 +44,8 @@
 import warnings
 import datetime
 import codecs
+import gzip
+
 from coverage.report import Reporter
 from coverage.results import Numbers
 from coverage.misc import NotPython
@@ -285,7 +287,7 @@ def code_coverage_test(self):
             checker.stop()
             # Save to the .coverage file
             checker.save()
-            # Apply the confirured and requested omit and include data 
+            # Apply the configured and requested omit and include data 
             checker.config.from_args(ignore_errors=None, omit=settings.TEST_CODE_COVERAGE_EXCLUDE,
                 include=include, file=None)
             # Maybe output a html report
@@ -324,8 +326,12 @@ def setup_test_environment(self, **kwargs):
         ietf.utils.mail.SMTP_ADDR['port'] = 2025
         #
         if self.check_coverage:
-            with codecs.open(self.coverage_file, encoding='utf-8') as file:
-                self.coverage_master = json.load(file)
+            if self.coverage_file.endswith('.gz'):
+                with gzip.open(self.coverage_file, "rb") as file:
+                    self.coverage_master = json.load(file)
+            else:
+                with codecs.open(self.coverage_file, encoding='utf-8') as file:
+                    self.coverage_master = json.load(file)
             self.coverage_data = {
                 "time": datetime.datetime.now(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
                 "template": {
@@ -381,16 +387,20 @@ def teardown_test_environment(self, **kwargs):
         self.smtpd_driver.stop()
         if self.check_coverage:
             latest_coverage_file = os.path.join(self.root_dir, settings.TEST_COVERAGE_LATEST_FILE)
+            coverage_latest = {}
+            coverage_latest["version"] = "latest"
+            coverage_latest["latest"] = self.coverage_data
             with codecs.open(latest_coverage_file, "w", encoding='utf-8') as file:
-                coverage_latest = {}
-                coverage_latest["version"] = "latest"
-                coverage_latest["latest"] = self.coverage_data
                 json.dump(coverage_latest, file, indent=2, sort_keys=True)
             if self.save_version_coverage:
-                with codecs.open(self.coverage_file, "w", encoding="utf-8") as file:
-                    self.coverage_master["version"] = self.save_version_coverage
-                    self.coverage_master[self.save_version_coverage] = self.coverage_data
-                    json.dump(self.coverage_master, file, indent=2, sort_keys=True)
+                self.coverage_master["version"] = self.save_version_coverage
+                self.coverage_master[self.save_version_coverage] = self.coverage_data
+                if self.coverage_file.endswith('.gz'):
+                    with gzip.open(self.coverage_file, "wb") as file:
+                        json.dump(self.coverage_master, file, indent=2, sort_keys=True)
+                else:
+                    with codecs.open(self.coverage_file, "w", encoding="utf-8") as file:
+                        json.dump(self.coverage_master, file, indent=2, sort_keys=True)
         super(IetfTestRunner, self).teardown_test_environment(**kwargs)
 
     def get_test_paths(self, test_labels):