flake8 - remove re.compile from method arg + test + doc

changed 2 methods defined like:

 def method(..., dre=re.compile(r'...')):

moved re.compile to module variables and passed the var name

  def method(..., dre=var_name):

while doing this I found out that a url of .../issue0001 will behave
like .../issue1. Who knew. Documented in customizing.

Tested same in test_liveserver. Added msg1 as well so I could verify
msg0001 worked.

Also added some range tests as well.

Author: rouilj

doc/customizing.txt

@@ -2232,6 +2232,12 @@ for specific object of issue class with id 1 will look like:
 
 8. ``/issue1``
 
+.. _strip_zeros:
+
+Note that a leading string of 0's will be stripped from the id part of
+the object designator in the URL. E.G. ``/issue001`` is the same as
+``/issue1``.  Similarly for ``/file01`` etc. However you should
+generate URL's without the extra zeros.
 
 Determining web context
 -----------------------
@@ -2262,6 +2268,7 @@ c. if there is something in the path (as in example 1, "issue"), it
    identifies the tracker class to display.
 d. if the path is an item designator (as in examples 2 and 4, "issue1"
    and "file1"), then we're to display a specific item.
+   :ref:`Note. <strip_zeros>`
 e. if the path starts with an item designator and is longer than one
    entry (as in example 5, "file1/kitten.png"), then we're assumed to be
    handling an item of a ``FileClass``, and the extra path information

roundup/cgi/client.py

@@ -1179,7 +1179,8 @@ def determine_user(self):
                             raise LoginError("Token roles are invalid.")
 
                     # will be used later to override the get_roles method
-                    override_get_roles = lambda self: iter_roles(
+                    # having it defined as truthy allows it to be used.
+                    override_get_roles = lambda self: iter_roles(  # noqa: E731
                         ','.join(token['roles']))
 
         # if user was not set by http authorization, try session lookup
@@ -1632,7 +1633,13 @@ def opendb(self, username):
                 # we can no longer use it.
                 self.session_api = Session(self)
 
-    def determine_context(self, dre=re.compile(r'([^\d]+)0*(\d+)')):
+    # match designator in URL stripping leading 0's. So:
+    # https://issues.roundup-tracker.org/issue002551190 is the same as
+    # https://issues.roundup-tracker.org/issue2551190
+    # Note: id's are strings not numbers so "02" != "2" but 02 == 2
+    dre_url = re.compile(r'([^\d]+)0*(\d+)')
+
+    def determine_context(self, dre=dre_url):
         """Determine the context of this page from the URL:
 
         The URL path after the instance identifier is examined. The path
@@ -1743,7 +1750,11 @@ def determine_context(self, dre=re.compile(r'([^\d]+)0*(\d+)')):
         if template_override is not None:
             self.template = template_override
 
-    def serve_file(self, designator, dre=re.compile(r'([^\d]+)(\d+)')):
+    # re for splitting designator, see also dre_url above this one
+    # doesn't strip leading 0's from the id. Why not??
+    dre = re.compile(r'([^\d]+)(\d+)')
+
+    def serve_file(self, designator, dre=dre):
         """ Serve the file from the content property of the designated item.
         """
         m = dre.match(str(designator))
@@ -2514,6 +2525,8 @@ def handle_range_header(self, length, etag):
         first = self.http_strip(pos[0])
         last = self.http_strip(pos[1])
         # We do not handle suffix ranges.
+        #   Note this also captures atempts to make first
+        #   element of range a negative number.
         if not first:
             return None
         # Convert the first and last positions to integers.
@@ -2527,6 +2540,8 @@ def handle_range_header(self, length, etag):
             # The positions could not be parsed as integers.
             return None
         # Check that the range makes sense.
+        # Note, if range is -1-10, first = '', so this code will never
+        # be reached. if range = 1--10, this code is reached.
         if (first < 0 or last < 0 or last < first):
             return None
         if last >= length:

test/test_liveserver.py

@@ -1,5 +1,6 @@
 import shutil, errno, pytest, json, gzip, mimetypes, os, re
 
+from roundup import date as rdate
 from roundup import i18n
 from roundup import password
 from roundup.anypy.strings import b2s
@@ -94,6 +95,11 @@ def setup_class(cls):
         # also used for text searching.
         result = cls.db.issue.create(title="foo bar RESULT")
 
+        # add a message to allow retrieval
+        result = cls.db.msg.create(author = "1",
+                                   content = "a message foo bar RESULT",
+                                   date=rdate.Date(),
+                                   messageid="test-msg-id")
         cls.db.commit()
         cls.db.close()
 
@@ -191,12 +197,14 @@ def test_start_in_german(self):
         self.assertTrue(b'dauerhaft anmelden?' in f.content)
 
     def test_byte_Ranges(self):
-        """ Roundup only handles one simple two number range.
+        """ Roundup only handles one simple two number range, or
+            a single number to start from:
             Range: 10-20
+            Range: 10-
 
-            The following are not supported.
+            The following is not supported.
             Range: 10-20, 25-30
-            Range: 10-
+            Range: -10
 
             Also If-Range only supports strong etags not dates or weak etags.
 
@@ -220,7 +228,27 @@ def test_byte_Ranges(self):
         self.assertEqual(f.headers['content-range'],
                          "bytes 0-10/%s"%expected_length)
 
+        # get bytes 11-21 unconditionally (0 index really??)
+        hdrs = {"Range": "bytes=10-20"}
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 206)
+        self.assertEqual(f.content, b"ge styles *")
+        # compression disabled for length < 100, so we can use 11 here
+        self.assertEqual(f.headers['content-length'], '11')
+        self.assertEqual(f.headers['content-range'],
+                         "bytes 10-20/%s"%expected_length)
+
+        # get all bytest starting from 11
+        hdrs = {"Range": "bytes=11-"}
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 206)
+        self.assertEqual(f.headers['content-range'],
+                         "bytes 11-%s/%s"%(int(expected_length) - 1,
+                                           expected_length))
+        self.assertIn("SHA:", f.content)  # detect sha sum at end of file
+
         # conditional request 11 bytes since etag matches 206 code
+        hdrs = {"Range": "bytes=0-10"}
         hdrs['If-Range'] = etag
         f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
         self.assertEqual(f.status_code, 206)
@@ -262,7 +290,98 @@ def test_byte_Ranges(self):
         self.assertEqual(f.status_code, 416)
         self.assertEqual(f.headers['content-range'],
                          "bytes */%s"%expected_length)
-        
+
+        # invalid range multiple ranges
+        hdrs['Range'] = "bytes=0-10, 20-45"
+        print(hdrs)
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 200)
+        self.assertNotIn('content-range', f.headers,
+                         'content-range should not be present')
+        self.assertIn("SHA:", f.content)  # detect sha sum at end of file
+
+        # invalid range is single number not number followed by -
+        hdrs['Range'] = "bytes=1"
+        print(hdrs)
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 200)
+        self.assertNotIn('content-range', f.headers,
+                         'content-range should not be present')
+        self.assertIn("SHA:", f.content)  # detect sha sum at end of file
+
+        # range is invalid first number not a number
+        hdrs['Range'] = "bytes=boom-99" # bad first value
+        print(hdrs)
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 200)
+        self.assertNotIn('content-range', f.headers,
+                         'content-range should not be present')
+        self.assertIn("SHA:", f.content)  # detect sha sum at end of file
+
+        # range is invalid last number not a number
+        hdrs['Range'] = "bytes=1-boom" # bad last value
+        print(hdrs)
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 200)
+        self.assertNotIn('content-range', f.headers,
+                         'content-range should not be present')
+        self.assertIn("SHA:", f.content)  # detect sha sum at end of file
+
+        # range is invalid first position empty
+        hdrs['Range'] = "bytes=-11" # missing first value
+        print(hdrs)
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 200)
+        self.assertNotIn('content-range', f.headers,
+                         'content-range should not be present')
+        self.assertIn("SHA:", f.content)  # detect sha sum at end of file
+
+        # range is invalid #2 < #1
+        hdrs['Range'] = "bytes=11-1" # inverted range
+        print(hdrs)
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 200)
+        self.assertNotIn('content-range', f.headers,
+                         'content-range should not be present')
+        self.assertIn("SHA:", f.content)  # detect sha sum at end of file
+
+        # range is invalid negative first number
+        hdrs['Range'] = "bytes=-1-11" # negative first number
+        print(hdrs)
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 200)
+        self.assertNotIn('content-range', f.headers,
+                         'content-range should not be present')
+        self.assertIn("SHA:", f.content)  # detect sha sum at end of file
+
+        # range is invalid negative second number
+        hdrs['Range'] = "bytes=1--11" # negative second number
+        print(hdrs)
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 200)
+        self.assertNotIn('content-range', f.headers,
+                         'content-range should not be present')
+        self.assertIn("SHA:", f.content)  # detect sha sum at end of file
+
+        # range is unsupported units
+        hdrs['Range'] = "badunits=1-11"
+        print(hdrs)
+        f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
+        self.assertEqual(f.status_code, 200)
+        self.assertNotIn('content-range', f.headers,
+                         'content-range should not be present')
+        self.assertIn("SHA:", f.content)  # detect sha sum at end of file
+
+
+        # valid range, invalid file
+        hdrs['Range'] = "bytes=0-11"
+        print(hdrs)
+        f = requests.get(self.url_base() + "/@@file/style_nope.css",
+                         headers=hdrs)
+        self.assertEqual(f.status_code, 404)
+        self.assertNotIn('content-range', f.headers,
+                         'content-range should not be present')
+
     def test_rest_preflight_collection(self):
         # no auth for rest csrf preflight
         f = requests.options(self.url_base() + '/rest/data/user',
@@ -562,7 +681,22 @@ def test_ims(self):
 
 
     def test_load_issue1(self):
-        f = requests.get(self.url_base() + '/issue1>',
+        import pdb; pdb.set_trace()
+        for tail in [
+                '/issue1',      # normal url
+                '/issue00001',  # leading 0's should be stripped from id
+                '/issue1>'      # surprise this works too, should it??
+        ]:
+            f = requests.get(self.url_base() + tail,
+                             headers = { 'Accept-Encoding': 'gzip',
+                                         'Accept': '*/*'})
+
+            self.assertIn(b'foo bar RESULT', f.content)
+            self.assertEqual(f.status_code, 200)
+
+    def test_load_msg1(self):
+        # leading 0's should be stripped from id
+        f = requests.get(self.url_base() + '/msg0001',
                          headers = { 'Accept-Encoding': 'gzip',
                                      'Accept': '*/*'})
 
@@ -1173,6 +1307,12 @@ def setup_class(cls):
 
         result = cls.db.issue.create(title="foo bar RESULT")
 
+        # add a message to allow retrieval
+        result = cls.db.msg.create(author = "1",
+                                   content = "a message foo bar RESULT",
+                                   date=rdate.Date(),
+                                   messageid="test-msg-id")
+
         cls.db.commit()
         cls.db.close()