Merged in [16922] from sasha@dashcare.nl:

    Fix ietf-tools#2584 - Add additional content validation for uploaded texts.
Permitted MIME types are now text/plain, text/markdown and text/x-rst.
This applies to all usages of get_cleaned_text_file_content(),
including reviews, but also other similar places where text can either
be written either into a textarea or uploaded.
 - Legacy-Id: 16930
Note: SVN reference [16922] has been migrated to Git commit fd53f98

Author: levkowetz

ietf/settings.py

@@ -678,6 +678,10 @@ def skip_unreadable_post(record):
     "bluesheets": "https://www.ietf.org/proceedings/{meeting.number}/bluesheets/{doc.uploaded_filename}",
 }
 
+# Valid MIME types for cases where text is uploaded and immediately extracted,
+# e.g. a charter or a review. Must be a tuple, not a list.
+DOC_TEXT_FILE_VALID_UPLOAD_MIME_TYPES = ('text/plain', 'text/markdown', 'text/x-rst')
+
 # Override this in settings_local.py if needed
 CACHE_MIDDLEWARE_SECONDS = 300
 CACHE_MIDDLEWARE_KEY_PREFIX = ''

ietf/utils/test_textupload.py

@@ -0,0 +1,37 @@
+# Copyright The IETF Trust 2015-2019, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from django.core.exceptions import ValidationError
+from django.core.files.uploadedfile import SimpleUploadedFile
+
+from .textupload import get_cleaned_text_file_content
+from ietf.utils.test_utils import TestCase
+
+
+class GetCleanedTextFileContentTest(TestCase):
+    def test_no_file(self):
+        self.assertEqual(get_cleaned_text_file_content(None), "")
+
+    def test_valid_file(self):
+        data = 'testing 👾'
+        uploaded_file = SimpleUploadedFile('data.txt', data.encode('utf-8'))
+        self.assertEqual(get_cleaned_text_file_content(uploaded_file), data)
+
+    def test_invalid_mime_type_gif(self):
+        data = 'GIF89a

;'
+        uploaded_file = SimpleUploadedFile('data.txt', data.encode('utf-8'))
+        with self.assertRaises(ValidationError) as context:
+            get_cleaned_text_file_content(uploaded_file)
+        self.assertIn('does not appear to be a text file', context.exception.message)
+        self.assertIn('image/gif', context.exception.message)
+
+    def test_invalid_mime_type_rst(self):
+        data = r'{\rtf1}'
+        uploaded_file = SimpleUploadedFile('data.txt', data.encode('utf-8'))
+        with self.assertRaises(ValidationError) as context:
+            get_cleaned_text_file_content(uploaded_file)
+        self.assertIn('does not appear to be a text file', context.exception.message)
+        self.assertIn('text/rtf', context.exception.message)

ietf/utils/textupload.py

@@ -6,6 +6,7 @@
 
 import re
 
+from django.conf import settings
 from django.core.exceptions import ValidationError
 
 import debug                            # pyflakes:ignore
@@ -36,8 +37,10 @@ def get_cleaned_text_file_content(uploaded_file):
         magic.magic_load(m.cookie, None)
         filetype = m.from_buffer(content)
 
-    if not filetype.startswith("text"):
-        raise ValidationError("Uploaded file does not appear to be a text file.")
+    if not filetype.startswith(settings.DOC_TEXT_FILE_VALID_UPLOAD_MIME_TYPES):
+        raise ValidationError("Uploaded file does not appear to be a text file. "
+                              "Permitted MIME types are {}, this file is {}"
+                              .format(', '.join(settings.DOC_TEXT_FILE_VALID_UPLOAD_MIME_TYPES), filetype))
 
     match = re.search(r"charset=([\w-]+)", filetype)
     if not match: