Fix issue with retreiving raw template files using the @@file mechanism.

This changes the static_files option in config.ini from supporting a
single directory to support multiple directories. If one of the
directory elements is '-' (i.e. a lone hyphen) the search is stopped
and the TEMPLATES directory is not searched. Since the TEMPLATES
directory is not searched the raw templates aren't accessed.

See: https://sourceforge.net/p/roundup/mailman/message/35773357/
Message subject: showing template sources to all
for details.

Also check in CHANGES.txt that mentions a couple of other small
improvements in the roundup-admin command.

Author: rouilj

CHANGES.txt

@@ -192,6 +192,10 @@ Features:
   session cookie. Default is lax, but there is a settable
   option in config.ini file to change to strict or
   suppress it entirely. See ``upgrading.txt``. (John Rouillard)
+- Added a new roundup-admin command: updateconfig. Similar to
+  genconfig but it uses values from an existing config.ini
+  rather than default values. Use to update an existing
+  config.ini with new options and help text. (John Rouillard)
 
 Fixed:
 
@@ -427,6 +431,14 @@ Fixed:
 - issue2550937: fix crash by verifying that sendto is not null before
   calling mailer.smtp_send. Discovered and patched by Trent Gamblin.
   Applied by John Rouillard.
+- removed old code from roundup-admin that implemented the obsolete
+  config (do_config) command. (John Rouillard)
+- Modified configuration option static_files to be a space separated
+  list of directories to search for static files in the web interface.
+  If one of the elements is -, the search stops and the TEMPLATES
+  directory is not searched. See:
+    https://sourceforge.net/p/roundup/mailman/message/35773357/
+  subject is "showing template sources to all".
 
 2016-01-11: 1.5.1
 

doc/customizing.txt

@@ -87,6 +87,8 @@ section "ConfigParser -- Configuration file parser":
 
 __ http://docs.python.org/lib/module-ConfigParser.html
 
+Example configuration settings are below.
+ 
 Section **main**
  database -- ``db``
   Database directory path. The path may be either absolute or relative
@@ -97,12 +99,13 @@ Section **main**
   or relative to the directory containing this config file.
 
  static_files -- default *blank*
-  Path to directory holding additional static files available via Web
-  UI.  This directory may contain sitewide images, CSS stylesheets etc.
-  and is searched for these files prior to the TEMPLATES directory
-  specified above.  If this option is not set, all static files are
-  taken from the TEMPLATES directory The path may be either absolute or
-  relative to the directory containig this config file.
+  A list of space separated directory paths (or a single directory).
+  These directories hold additional static files available via Web UI.
+  These directories may contain sitewide images, CSS stylesheets etc. If
+  a '-' is included, the list processing ends and the TEMPLATES
+  directory is not searched after the specified directories.  If this
+  option is not set, all static files are taken from the TEMPLATES
+  directory.
 
  admin_email -- ``roundup-admin``
   Email address that roundup will complain to if it runs into trouble. If
@@ -459,7 +462,8 @@ Section **nosy**
 
 
 You may generate a new default config file using the ``roundup-admin
-genconfig`` command.
+genconfig`` command. You can generate a new config file merging in
+existing settings using the ``roundup-admin updateconfig`` command.
 
 Configuration variables may be referred to in lower or upper case. In code,
 variables not in the "main" section are referred to using their section and

roundup/cgi/client.py

@@ -1439,13 +1439,33 @@ def serve_static_file(self, file):
             prefix = self.instance.config[dir_option]
             if not prefix:
                 continue
-            # ensure the load doesn't try to poke outside
-            # of the static files directory
-            prefix = os.path.normpath(prefix)
-            filename = os.path.normpath(os.path.join(prefix, file))
-            if os.path.isfile(filename) and filename.startswith(prefix):
+            if type(prefix) is str:
+                # prefix can be a string or list depending on
+                # option. Make it a list to iterate over.
+                prefix = [ prefix ]
+
+            for p in prefix:
+                # if last element of STATIC_FILES ends with '/-',
+                # we failed to find the file and we should
+                # not look in TEMPLATES. So raise exception.
+                if dir_option == 'STATIC_FILES' and p[-2:] == '/-':
+                    raise NotFound(file)
+
+                # ensure the load doesn't try to poke outside
+                # of the static files directory
+                p = os.path.normpath(p)
+                filename = os.path.normpath(os.path.join(p, file))
+                if os.path.isfile(filename) and filename.startswith(p):
+                    break # inner loop over list of directories
+                else:
+                    # reset filename to None as sentinel for use below.
+                    filename = None
+
+            # break out of outer loop over options
+            if filename:
                 break
-        else:
+
+        if filename is None: # we didn't find a filename
             raise NotFound(file)
 
         # last-modified time

roundup/configuration.py

@@ -379,6 +379,31 @@ def get(self):
             _val = os.path.join(self.config["HOME"], _val)
         return _val
 
+class MultiFilePathOption(Option):
+
+    """List of space seperated File or directory path name
+
+    Paths may be either absolute or relative to the HOME. None
+    is returned if there are no elements.
+
+    """
+
+    class_description = "The space separated paths may be either absolute or\n" \
+        "relative to the directory containing this config file."
+
+    def get(self):
+        pathlist = []
+        _val = Option.get(self)
+        for elem in _val.split():
+            if elem and not os.path.isabs(elem):
+                pathlist.append(os.path.join(self.config["HOME"], elem))
+            else:
+                pathlist.append(elem)
+        if pathlist:
+            return pathlist
+        else:
+            return None
+
 class FloatNumberOption(Option):
 
     """Floating point numbers"""
@@ -526,13 +551,15 @@ def str2value(self, value):
             "ported from Zope, or 'chameleon' for Chameleon."),
         (FilePathOption, "templates", "html",
             "Path to the HTML templates directory."),
-        (NullableFilePathOption, "static_files", "",
-            "Path to directory holding additional static files\n"
-            "available via Web UI.  This directory may contain\n"
-            "sitewide images, CSS stylesheets etc. and is searched\n"
-            "for these files prior to the TEMPLATES directory\n"
-            "specified above.  If this option is not set, all static\n"
-            "files are taken from the TEMPLATES directory"),
+        (MultiFilePathOption, "static_files", "",
+            "A list of space separated directory paths (or a single\n"
+            "directory).  These directories hold additional static\n"
+            "files available via Web UI.  These directories may\n"
+            "contain sitewide images, CSS stylesheets etc. If a '-'\n"
+            "is included, the list processing ends and the TEMPLATES\n"
+            "directory is not searched after the specified\n"
+            "directories.  If this option is not set, all static\n"
+            "files are taken from the TEMPLATES directory."),
         (MailAddressOption, "admin_email", "roundup-admin",
             "Email address that roundup will complain to if it runs\n"
             "into trouble.\n"

test/test_cgi.py

@@ -11,7 +11,7 @@
 import unittest, os, shutil, errno, sys, difflib, cgi, re, StringIO
 
 from roundup.cgi import client, actions, exceptions
-from roundup.cgi.exceptions import FormError
+from roundup.cgi.exceptions import FormError, NotFound
 from roundup.exceptions import UsageError
 from roundup.cgi.templating import HTMLItem, HTMLRequest, NoTemplate, anti_csrf_nonce
 from roundup.cgi.templating import HTMLProperty, _HTMLItem
@@ -1390,6 +1390,105 @@ def testEditCSV(self):
         k = self.db.keyword.getnode('1')
         self.assertEqual(k.name, u'\xe4\xf6\xfc'.encode('utf-8'))
 
+    def testserve_static_files(self):
+        # make a client instance
+        cl = self._make_client({})
+
+        # hijack _serve_file so I can see what is found
+        output = []
+        def my_serve_file(a, b, c, d):
+            output.append((a,b,c,d))
+        cl._serve_file = my_serve_file
+
+        # check case where file is not found.
+        self.assertRaises(NotFound,
+                          cl.serve_static_file,"missing.css")
+
+        # TEMPLATES dir is searched by default. So this file exists.
+        # Check the returned values.
+        cl.serve_static_file("issue.index.html")
+        self.assertEquals(output[0][1], "text/html")
+        self.assertEquals(output[0][3], "_test_cgi_form/html/issue.index.html")
+        del output[0] # reset output buffer
+
+        # stop searching TEMPLATES for the files.
+        cl.instance.config['STATIC_FILES'] = '-'
+        # previously found file should not be found
+        self.assertRaises(NotFound,
+                          cl.serve_static_file,"issue.index.html")
+
+        # explicitly allow html directory
+        cl.instance.config['STATIC_FILES'] = 'html -'
+        cl.serve_static_file("issue.index.html")
+        self.assertEquals(output[0][1], "text/html")
+        self.assertEquals(output[0][3], "_test_cgi_form/html/issue.index.html")
+        del output[0] # reset output buffer
+
+        # set the list of files and do not look at the templates directory
+        cl.instance.config['STATIC_FILES'] = 'detectors   extensions -	'
+
+        # find file in first directory
+        cl.serve_static_file("messagesummary.py")
+        self.assertEquals(output[0][1], "text/x-python")
+        self.assertEquals(output[0][3], "_test_cgi_form/detectors/messagesummary.py")
+        del output[0] # reset output buffer
+
+        # find file in second directory
+        cl.serve_static_file("README.txt")
+        self.assertEquals(output[0][1], "text/plain")
+        self.assertEquals(output[0][3], "_test_cgi_form/extensions/README.txt")
+        del output[0] # reset output buffer
+
+        # make sure an embedded - ends the searching.
+        cl.instance.config['STATIC_FILES'] = ' detectors - extensions '
+        self.assertRaises(NotFound, cl.serve_static_file, "README.txt")
+
+        cl.instance.config['STATIC_FILES'] = ' detectors - extensions   '
+        self.assertRaises(NotFound, cl.serve_static_file, "issue.index.html")
+
+        # create an empty README.txt in the first directory
+        f = open('_test_cgi_form/detectors/README.txt', 'a').close()
+        # find file now in first directory
+        cl.serve_static_file("README.txt")
+        self.assertEquals(output[0][1], "text/plain")
+        self.assertEquals(output[0][3], "_test_cgi_form/detectors/README.txt")
+        del output[0] # reset output buffer
+
+        cl.instance.config['STATIC_FILES'] = ' detectors extensions '
+        # make sure lack of trailing - allows searching TEMPLATES
+        cl.serve_static_file("issue.index.html")
+        self.assertEquals(output[0][1], "text/html")
+        self.assertEquals(output[0][3], "_test_cgi_form/html/issue.index.html")
+        del output[0] # reset output buffer
+
+        # Make STATIC_FILES a single element.
+        cl.instance.config['STATIC_FILES'] = 'detectors'
+        # find file now in first directory
+        cl.serve_static_file("messagesummary.py")
+        self.assertEquals(output[0][1], "text/x-python")
+        self.assertEquals(output[0][3], "_test_cgi_form/detectors/messagesummary.py")
+        del output[0] # reset output buffer
+
+        # make sure files found in subdirectory
+        os.mkdir('_test_cgi_form/detectors/css')
+        f = open('_test_cgi_form/detectors/css/README.css', 'a').close()
+        # use subdir in filename
+        cl.serve_static_file("css/README.css")
+        self.assertEquals(output[0][1], "text/css")
+        self.assertEquals(output[0][3], "_test_cgi_form/detectors/css/README.css")
+        del output[0] # reset output buffer
+
+
+        # use subdir in static files path
+        cl.instance.config['STATIC_FILES'] = 'detectors html/css'
+        os.mkdir('_test_cgi_form/html/css')
+        f = open('_test_cgi_form/html/css/README1.css', 'a').close()
+        cl.serve_static_file("README1.css")
+        self.assertEquals(output[0][1], "text/css")
+        self.assertEquals(output[0][3], "_test_cgi_form/html/css/README1.css")
+        del output[0] # reset output buffer
+
+
     def testRoles(self):
         cl = self._make_client({})
         self.db.user.set('1', roles='aDmin,    uSer')