issue2551175 - Make ETag content-encoding aware.

HTTP ETag headers now include a suffix (-gzip, -br, -zstd) indicating
the content-encoding used to send the data per rfc7232. Validate any
form of ETag by stripping a suffix (if present).

Author: rouilj

CHANGES.txt

@@ -47,6 +47,10 @@ Fixed:
   created using 1.2.0 or newer. The fix may have predated the 1.2.0
   release but where the fix actually landed (representing id as a
   string and not as an int) is unknown.
+- issue2551175 - Make ETag content-encoding aware. HTTP ETag headers
+  now include a suffix indicating the content-encoding used to send
+  the data per rfc7232. Properly validate any form of ETag suffixed or
+  non-suffixed for If-Match.
 
 Features:
 

doc/rest.txt

@@ -227,7 +227,12 @@ Performing a ``GET`` on an item or property of an item will return an
 ETag header or an @etag property. This needs to be submitted with
 ``DELETE``, ``PUT`` and ``PATCH`` operations on the item using an
 ``If-Match`` header or an ``"@etag`` property in the data payload if
-the method supports a payload.
+the method supports a payload. The ETag header value will include a
+suffix (starting with '-') indicating the Content-Encoding used to
+respond to the request. If the response was uncompressed, there will
+be no suffix. The ``@etag`` property never includes the suffix. Any
+ETag value suffixed or not can be sent in an ``If-Match`` header as
+the suffix is ignored during comparison.
 
 The exact details of returned data is determined by the value of the
 ``@verbose`` query parameter.  The various supported values and their

roundup/cgi/client.py

@@ -2166,6 +2166,14 @@ def compress_encode(self, byte_content, quality=4):
             self.additional_headers['Content-Length'] = str(len(new_content))
             self.additional_headers['Content-Encoding'] = encoder
             self.setVary('Accept-Encoding')
+            try:
+                current_etag = self.additional_headers['ETag']
+            except KeyError:
+                pass  # etag not set for non-rest endpoints
+            else:
+                etag_end = current_etag.rindex('"')
+                self.additional_headers['ETag'] = ( current_etag[:etag_end] +
+                                    '-' + encoder + current_etag[etag_end:])
 
         return new_content
 

roundup/rest.py

@@ -218,8 +218,20 @@ def check_etag(node, key, etags, classname="Missing", id="0",
                                repr_format=repr_format)
 
     for etag in etags:
-        if etag is not None:
-            if etag != node_etag:
+        # etag includes doublequotes around tag:
+        #   '"a46a5572190e4fad63958c135f3746fa"'
+        # but can include content-encoding suffix like:
+        #   '"a46a5572190e4fad63958c135f3746fa-gzip"'
+        # turn the latter into the former as we don't care what
+        # encoding was used to send the body with the etag.
+        try:
+            suffix_start = etag.rindex('-')
+            clean_etag = etag[:suffix_start] + '"'
+        except (ValueError, AttributeError):
+            # - not in etag or etag is None
+            clean_etag = etag
+        if clean_etag is not None:
+            if clean_etag != node_etag:
                 return False
             have_etag_match = True
 

test/rest_common.py

@@ -1180,8 +1180,8 @@ def testEtagProcessing(self):
         each one broke and no etag. Use the put command
         to trigger the etag checking code.
         '''
-        for mode in ('header', 'etag', 'both',
-                     'brokenheader', 'brokenetag', 'none'):
+        for mode in ('header', 'header-gzip', 'etag', 'etag-br',
+                     'both', 'brokenheader', 'brokenetag', 'none'):
             try:
                 # clean up any old header
                 del(self.headers)
@@ -1198,9 +1198,17 @@ def testEtagProcessing(self):
             if mode == 'header':
                 print("Mode = %s"%mode)
                 self.headers = {'if-match': etag}
+            elif mode == 'header-gzip':
+                print("Mode = %s"%mode)
+                gzip_etag = etag[:-1] + "-gzip" + etag[-1:]
+                self.headers = {'if-match': gzip_etag}
             elif mode == 'etag':
                 print("Mode = %s"%mode)
                 form.list.append(cgi.MiniFieldStorage('@etag', etag))
+            elif mode == 'etag-br':
+                print("Mode = %s"%mode)
+                br_etag = etag[:-1] + "-br" + etag[-1:]
+                form.list.append(cgi.MiniFieldStorage('@etag', br_etag))
             elif mode == 'both':
                 print("Mode = %s"%mode)
                 self.headers = {'etag': etag}
@@ -1216,7 +1224,7 @@ def testEtagProcessing(self):
             elif mode == 'none':
                 print( "Mode = %s"%mode)
             else:
-                self.fail("unknown mode found")
+                self.fail("unknown mode '%s' found"%mode)
 
             results = self.server.put_attribute(
                 'user', self.joeid, 'realname', form

test/test_liveserver.py

@@ -439,6 +439,13 @@ def test_compression_gzip(self):
 
         self.assertDictEqual(json_dict, content)
 
+        # verify that ETag header ends with -gzip
+        try:
+            self.assertRegex(f.headers['ETag'], r'^"[0-9a-f]{32}-gzip"$')
+        except AttributeError:
+            # python2 no assertRegex so try substring match
+            self.assertEqual(33, f.headers['ETag'].rindex('-gzip"'))
+
         # use dict comprehension to remove fields like date,
         # content-length etc. from f.headers.
         self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
@@ -568,6 +575,13 @@ def test_compression_br(self):
 
         self.assertDictEqual(json_dict, content)
 
+        # verify that ETag header ends with -br
+        try:
+            self.assertRegex(f.headers['ETag'], r'^"[0-9a-f]{32}-br"$')
+        except AttributeError:
+            # python2 no assertRegex so try substring match
+            self.assertEqual(33, f.headers['ETag'].rindex('-br"'))
+
         # use dict comprehension to remove fields like date,
         # content-length etc. from f.headers.
         self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
@@ -714,6 +728,13 @@ def test_compression_zstd(self):
 
         self.assertDictEqual(json_dict, content)
 
+        # verify that ETag header ends with -zstd
+        try:
+            self.assertRegex(f.headers['ETag'], r'^"[0-9a-f]{32}-zstd"$')
+        except AttributeError:
+            # python2 no assertRegex so try substring match
+            self.assertEqual(33, f.headers['ETag'].rindex('-zstd"'))
+
         # use dict comprehension to remove fields like date,
         # content-length etc. from f.headers.
         self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)