escape HTML tags in markdown content

enabled fenced code blocks for markdown
allow mistune to be used as a markdown parser
test all installed markdown parsers and fallback code

Author: cmeerw

.travis.yml

@@ -94,6 +94,8 @@ install:
   - pip install gpg pytz whoosh pyjwt
   - pip install pytest-cov codecov
   - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* ]]; then pip install docutils; fi
+  - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* ]]; then pip install mistune; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == "3.[5-9]"* ]]; then pip install Markdown; fi
   - pip install markdown2
 
 before_script:

doc/installation.txt

@@ -120,9 +120,9 @@ docutils
   To use ReStructuredText rendering you need to have the docutils
   package installed.
 
-markdown or markdown2
-  To use markdown rendering you need to either have the markdown or
-  markdown2 package installed.
+markdown, markdown2 or mistune
+  To use markdown rendering you need to have the markdown, markdown2
+  or mistune package installed.
 
 Windows Service
   You can run Roundup as a Windows service if pywin32_ is installed.
@@ -139,6 +139,7 @@ Windows Service
 .. _docutils: https://docutils.sourceforge.io/
 .. _markdown: https://python-markdown.github.io/
 .. _markdown2: https://github.com/trentm/python-markdown2
+.. _mistune: https://github.com/lepture/mistune
 
 
 Getting Roundup

roundup/cgi/templating.py

@@ -54,16 +54,49 @@
 except ImportError:
     ReStructuredText = None
 try:
-    from markdown2 import markdown
+    from itertools import zip_longest
 except ImportError:
+    from itertools import izip_longest as zip_longest
+
+def _import_markdown2():
     try:
-        from markdown import markdown
+        import markdown2, re
+        class Markdown(markdown2.Markdown):
+            # don't restrict protocols in links
+            _safe_protocols = re.compile('', re.IGNORECASE)
+
+        markdown = lambda s: Markdown(safe_mode='escape', extras={ 'fenced-code-blocks' : True }).convert(s)
     except ImportError:
         markdown = None
-try:
-    from itertools import zip_longest
-except ImportError:
-    from itertools import izip_longest as zip_longest
+
+    return markdown
+
+def _import_markdown():
+    try:
+        from markdown import markdown as markdown_impl
+        from markdown.extensions import Extension as MarkdownExtension
+
+        # make sure any HTML tags get escaped
+        class EscapeHtml(MarkdownExtension):
+            def extendMarkdown(self, md):
+                md.preprocessors.deregister('html_block')
+                md.inlinePatterns.deregister('html')
+
+        markdown = lambda s: markdown_impl(s, extensions=[EscapeHtml(), 'fenced_code'])
+    except ImportError:
+        markdown = None
+
+    return markdown
+
+def _import_mistune():
+    try:
+        from mistune import markdown
+    except ImportError:
+        markdown = None
+
+    return markdown
+
+markdown = _import_markdown2() or _import_markdown() or _import_mistune()
 
 # bring in the templating support
 from roundup.cgi import TranslationService, ZTUtils

test/test_templating.py

@@ -9,36 +9,36 @@
 import pytest
 from .pytest_patcher import mark_class
 
-try:
-    from docutils.core import publish_parts as ReStructuredText
+if ReStructuredText:
     skip_rst = lambda func, *args, **kwargs: func
-except ImportError:
-    ReStructuredText = None
+else:
     skip_rst = mark_class(pytest.mark.skip(
         reason='ReStructuredText not available'))
 
-try:
-    from StructuredText.StructuredText import HTML as StructuredText
+if StructuredText:
     skip_stext = lambda func, *args, **kwargs: func
-except ImportError:
-    try: # older version
-        import StructuredText
-        skip_stext = lambda func, *args, **kwargs: func
-    except ImportError:
-        StructuredText = None
-        skip_stext = mark_class(pytest.mark.skip(
-                reason='StructuredText not available'))
-
-try:
+else:
+    skip_stext = mark_class(pytest.mark.skip(
+        reason='StructuredText not available'))
+
+import roundup.cgi.templating
+if roundup.cgi.templating._import_mistune():
+    skip_mistune = lambda func, *args, **kwargs: func
+else:
+    skip_mistune = mark_class(pytest.mark.skip(
+        reason='mistune not available'))
+
+if roundup.cgi.templating._import_markdown2():
+    skip_markdown2 = lambda func, *args, **kwargs: func
+else:
+    skip_markdown2 = mark_class(pytest.mark.skip(
+        reason='markdown2 not available'))
+
+if roundup.cgi.templating._import_markdown():
     skip_markdown = lambda func, *args, **kwargs: func
-    from markdown2 import markdown
-except ImportError:
-    try:
-        from markdown import markdown
-    except ImportError:
-        markdown = None
-        skip_markdown = mark_class(pytest.mark.skip(
-                reason='markdown not available'))
+else:
+    skip_markdown = mark_class(pytest.mark.skip(
+        reason='markdown not available'))
 
 from roundup.anypy.strings import u2s, s2u
 
@@ -247,21 +247,7 @@ def test_string_plain_or_hyperlinked(self):
 
         self.assertEqual(p.hyperlinked(), 'A string &lt;b&gt; with <a href="mailto:[email protected]">[email protected]</a> embedded &amp;lt; html&lt;/b&gt;')
 
-    @skip_markdown
-    def test_string_markdown_installed(self):
-        pass # just so we have a record of a skipped test
-
-    def test_string_markdown(self):
-        p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string http://localhost with [email protected] *embedded* \u00df'))
-        if markdown:
-            self.assertEqual(p.markdown().strip(), u2s(u'<p>A string <a href="http://localhost">http://localhost</a> with <a href="mailto:[email protected]">[email protected]</a> <em>embedded</em> \u00df</p>'))
-        else:
-            self.assertEqual(p.markdown(), u2s(u'A string <a href="http://localhost" rel="nofollow noopener">http://localhost</a> with <a href="mailto:[email protected]">[email protected]</a> *embedded* \u00df'))
-
     @skip_rst
-    def test_string_rst_installed(self):
-        pass # just so we have a record of a skipped test
-
     def test_string_rst(self):
         p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string with [email protected] *embedded* \u00df'))
 
@@ -295,23 +281,14 @@ def test_string_rst(self):
 </div>
 </div>
 '''
-        if ReStructuredText:
-            self.assertEqual(p.rst(), u2s(u'<div class="document">\n<p>A string with <a class="reference external" href="mailto:cmeerw&#64;example.com">cmeerw&#64;example.com</a> <em>embedded</em> \u00df</p>\n</div>\n'))
-            self.assertEqual(q.rst(), u2s(q_result))
-            self.assertEqual(r.rst(), u2s(r_result))
-        else:
-            self.assertEqual(p.rst(), u2s(u'A string with <a href="mailto:[email protected]">[email protected]</a> *embedded* \u00df'))
+        self.assertEqual(p.rst(), u2s(u'<div class="document">\n<p>A string with <a class="reference external" href="mailto:cmeerw&#64;example.com">cmeerw&#64;example.com</a> <em>embedded</em> \u00df</p>\n</div>\n'))
+        self.assertEqual(q.rst(), u2s(q_result))
+        self.assertEqual(r.rst(), u2s(r_result))
 
     @skip_stext
-    def test_string_stext_installed(self):
-        pass # just so we have a record of a skipped test
-
     def test_string_stext(self):
         p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string with [email protected] *embedded* \u00df'))
-        if StructuredText:
-            self.assertEqual(p.stext(), u2s(u'<p>A string with <a href="mailto:[email protected]">[email protected]</a> <em>embedded</em> \u00df</p>\n'))
-        else:
-            self.assertEqual(p.stext(), u2s(u'A string with <a href="mailto:[email protected]">[email protected]</a> *embedded* \u00df'))
+        self.assertEqual(p.stext(), u2s(u'<p>A string with <a href="mailto:[email protected]">[email protected]</a> <em>embedded</em> \u00df</p>\n'))
 
     def test_string_field(self):
         p = StringHTMLProperty(self.client, 'test', '1', None, 'test', 'A string <b> with [email protected] embedded &lt; html</b>')
@@ -436,6 +413,105 @@ def test_input_xhtml(self):
         input=input_xhtml(**attrs)
         self.assertEqual(input, '<input class="required" disabled="disabled" size="30" type="text"/>')
 
+# common markdown test cases
+class MarkdownTests:
+    def test_string_markdown(self):
+        p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string http://localhost with [email protected] <br> *embedded* \u00df'))
+        self.assertEqual(p.markdown().strip(), u2s(u'<p>A string <a href="http://localhost">http://localhost</a> with <a href="mailto:[email protected]">[email protected]</a> &lt;br&gt; <em>embedded</em> \u00df</p>'))
+
+    def test_string_markdown_code_block(self):
+        p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'embedded code block\n\n```\nline 1\nline 2\n```\n\nnew paragraph'))
+        self.assertEqual(p.markdown().strip().replace('\n\n', '\n'), u2s(u'<p>embedded code block</p>\n<pre><code>line 1\nline 2\n</code></pre>\n<p>new paragraph</p>'))
+
+@skip_mistune
+class MistuneTestCase(TemplatingTestCase, MarkdownTests) :
+    def setUp(self):
+        TemplatingTestCase.setUp(self)
+
+        from roundup.cgi import templating
+        self.__markdown = templating.markdown
+        templating.markdown = templating._import_mistune()
+
+    def tearDown(self):
+        from roundup.cgi import templating
+        templating.markdown = self.__markdown
+
+@skip_markdown2
+class Markdown2TestCase(TemplatingTestCase, MarkdownTests) :
+    def setUp(self):
+        TemplatingTestCase.setUp(self)
+
+        from roundup.cgi import templating
+        self.__markdown = templating.markdown
+        templating.markdown = templating._import_markdown2()
+
+    def tearDown(self):
+        from roundup.cgi import templating
+        templating.markdown = self.__markdown
+
+@skip_markdown
+class MarkdownTestCase(TemplatingTestCase, MarkdownTests) :
+    def setUp(self):
+        TemplatingTestCase.setUp(self)
+
+        from roundup.cgi import templating
+        self.__markdown = templating.markdown
+        templating.markdown = templating._import_markdown()
+
+    def tearDown(self):
+        from roundup.cgi import templating
+        templating.markdown = self.__markdown
+
+
+class NoMarkdownTestCase(TemplatingTestCase) :
+    def setUp(self):
+        TemplatingTestCase.setUp(self)
+
+        from roundup.cgi import templating
+        self.__markdown = templating.markdown
+        templating.markdown = None
+
+    def tearDown(self):
+        from roundup.cgi import templating
+        templating.markdown = self.__markdown
+
+    def test_string_markdown(self):
+        p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string http://localhost with [email protected] <br> *embedded* \u00df'))
+        self.assertEqual(p.markdown(), u2s(u'A string <a href="http://localhost" rel="nofollow noopener">http://localhost</a> with <a href="mailto:[email protected]">[email protected]</a> &lt;br&gt; *embedded* \u00df'))
+
+class NoRstTestCase(TemplatingTestCase) :
+    def setUp(self):
+        TemplatingTestCase.setUp(self)
+
+        from roundup.cgi import templating
+        self.__ReStructuredText = templating.ReStructuredText
+        templating.ReStructuredText = None
+
+    def tearDown(self):
+        from roundup.cgi import templating
+        templating.ReStructuredText = self.__ReStructuredText
+
+    def test_string_rst(self):
+        p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string with [email protected] *embedded* \u00df'))
+        self.assertEqual(p.rst(), u2s(u'A string with <a href="mailto:[email protected]">[email protected]</a> *embedded* \u00df'))
+
+class NoStextTestCase(TemplatingTestCase) :
+    def setUp(self):
+        TemplatingTestCase.setUp(self)
+
+        from roundup.cgi import templating
+        self.__StructuredText = templating.StructuredText
+        templating.StructuredText = None
+
+    def tearDown(self):
+        from roundup.cgi import templating
+        templating.StructuredText = self.__StructuredText
+
+    def test_string_stext(self):
+        p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string with [email protected] *embedded* \u00df'))
+        self.assertEqual(p.stext(), u2s(u'A string with <a href="mailto:[email protected]">[email protected]</a> *embedded* \u00df'))
+
+
 r'''
 class HTMLPermissions:
     def is_edit_ok(self):