Merged in support of xml-source only drafts submissions from personal/henrik/submitxml/.

 - Legacy-Id: 9859

Author: levkowetz

ietf/settings.py

@@ -432,18 +432,23 @@ def skip_unreadable_post(record):
 IDSUBMIT_DEFAULT_CUTOFF_TIME_UTC = datetime.timedelta(hours=23, minutes=59, seconds=59)
 IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21)
 
-MEETING_MATERIALS_SUBMISSION_START_DAYS = -90
-MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26
-MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 50
-
-INTERNET_DRAFT_DAYS_TO_EXPIRE = 185
-
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
 IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
 IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
 
-IDSUBMIT_MAX_PLAIN_DRAFT_SIZE = 6291456  # Max size of the txt draft in bytes
+IDSUBMIT_FILE_TYPES = (
+    'txt',
+    'xml',
+    'pdf',
+    'ps',
+)
+IDSUBMIT_MAX_DRAFT_SIZE =  {
+    'txt':  6*1024*1024,  # Max size of txt draft file in bytes
+    'xml': 10*1024*1024,  # Max size of xml draft file in bytes
+    'pdf': 10*1024*1024,
+    'ps' : 10*1024*1024,
+}
 
 IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME = 20
 IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE = 50 # in MB
@@ -454,6 +459,14 @@ def skip_unreadable_post(record):
 IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000
 IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB
 
+XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/"
+
+MEETING_MATERIALS_SUBMISSION_START_DAYS = -90
+MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26
+MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 50
+
+INTERNET_DRAFT_DAYS_TO_EXPIRE = 185
+
 DOT_BINARY = '/usr/bin/dot'
 UNFLATTEN_BINARY= '/usr/bin/unflatten'
 PS2PDF_BINARY = '/usr/bin/ps2pdf'

ietf/submit/forms.py

@@ -1,6 +1,9 @@
 import os
+import re
 import datetime
 import pytz
+import xml2rfc
+import tempfile
 
 from django import forms
 from django.conf import settings
@@ -23,14 +26,14 @@
 from ietf.utils.draft import Draft
 
 
-class UploadForm(forms.Form):
-    txt = forms.FileField(label=u'.txt format', required=True)
+class SubmissionUploadForm(forms.Form):
+    txt = forms.FileField(label=u'.txt format', required=False)
     xml = forms.FileField(label=u'.xml format', required=False)
     pdf = forms.FileField(label=u'.pdf format', required=False)
-    ps = forms.FileField(label=u'.ps format', required=False)
+    ps  = forms.FileField(label=u'.ps format', required=False)
 
     def __init__(self, request, *args, **kwargs):
-        super(UploadForm, self).__init__(*args, **kwargs)
+        super(SubmissionUploadForm, self).__init__(*args, **kwargs)
 
         self.remote_ip = request.META.get('REMOTE_ADDR', None)
 
@@ -41,7 +44,13 @@ def __init__(self, request, *args, **kwargs):
         self.set_cutoff_warnings()
 
         self.group = None
+        self.filename = None
+        self.revision = None
+        self.title = None
+        self.abstract = None
+        self.authors = []
         self.parsed_draft = None
+        self.file_types = []
 
     def set_cutoff_warnings(self):
         now = datetime.datetime.now(pytz.utc)
@@ -93,15 +102,14 @@ def clean_file(self, field_name, parser_class):
 
         return f
 
-
     def clean_txt(self):
         return self.clean_file("txt", PlainParser)
 
     def clean_pdf(self):
         return self.clean_file("pdf", PDFParser)
 
     def clean_ps(self):
-        return self.clean_file("ps", PSParser)
+        return self.clean_file("ps",  PSParser)
 
     def clean_xml(self):
         return self.clean_file("xml", XMLParser)
@@ -116,37 +124,100 @@ def clean(self):
             if not os.path.exists(getattr(settings, s)):
                 raise forms.ValidationError('%s defined in settings.py does not exist' % s)
 
+        for ext in ['txt', 'pdf', 'xml', 'ps']:
+            f = self.cleaned_data.get(ext, None)
+            if not f:
+                continue
+            self.file_types.append('.%s' % ext)
+        if not ('.txt' in self.file_types or '.xml' in self.file_types):
+            raise forms.ValidationError('You must submit either a .txt or an .xml file; didn\'t find either.')
+
+        #debug.show('self.cleaned_data["xml"]')
+        if self.cleaned_data.get('xml'):
+            #if not self.cleaned_data.get('txt'):
+            xml_file = self.cleaned_data.get('xml')
+            tfh, tfn = tempfile.mkstemp(suffix='.xml')
+            try:
+                # We need to write the xml file to disk in order to hand it
+                # over to the xml parser.  XXX FIXME: investigate updating
+                # xml2rfc to be able to work with file handles to in-memory
+                # files.
+                with open(tfn, 'wb+') as tf:
+                    for chunk in xml_file.chunks():
+                        tf.write(chunk)
+                os.environ["XML_LIBRARY"] = settings.XML_LIBRARY
+                parser = xml2rfc.XmlRfcParser(tfn, quiet=True)
+                self.xmltree = parser.parse()
+                ok, errors = self.xmltree.validate()
+                if not ok:
+                    raise forms.ValidationError(errors)
+                self.xmlroot = self.xmltree.getroot()
+                draftname = self.xmlroot.attrib.get('docName')
+                revmatch = re.search("-[0-9][0-9]$", draftname)
+                if revmatch:
+                    self.revision = draftname[-2:]
+                    self.filename = draftname[:-3]
+                else:
+                    self.revision = None
+                    self.filename = draftname
+                self.title = self.xmlroot.find('front/title').text
+                self.abstract = self.xmlroot.find('front/abstract').text
+                self.author_list = []
+                author_info = self.xmlroot.findall('front/author')
+                for author in author_info:
+                    author_dict = dict(
+                        company = author.find('organization').text,
+                        last_name = author.attrib.get('surname'),
+                        full_name = author.attrib.get('fullname'),
+                        email = author.find('address/email').text,
+                    )
+                    self.author_list.append(author_dict)
+                    line = "%(full_name)s <%(email)s>" % author_dict
+                    self.authors.append(line)
+            except Exception as e:
+                raise forms.ValidationError("Exception: %s" % e)
+            finally:
+                os.close(tfh)
+                os.unlink(tfn)
+
         if self.cleaned_data.get('txt'):
             # try to parse it
             txt_file = self.cleaned_data['txt']
             txt_file.seek(0)
             self.parsed_draft = Draft(txt_file.read(), txt_file.name)
+            self.filename = self.parsed_draft.filename
+            self.revision = self.parsed_draft.revision
+            self.title    = self.parsed_draft.get_title()
             txt_file.seek(0)
 
-            if not self.parsed_draft.filename:
-                raise forms.ValidationError("Draft parser could not extract a valid draft name from the .txt file")
+            if not self.filename:
+                raise forms.ValidationError("Draft parser could not extract a valid draft name from the upload")
+
+            if not self.revision:
+                raise forms.ValidationError("Draft parser could not extract a valid draft revision from the upload")
 
-            if not self.parsed_draft.get_title():
-                raise forms.ValidationError("Draft parser could not extract a valid title from the .txt file")
+            if not self.title:
+                raise forms.ValidationError("Draft parser could not extract a valid title from the upload")
 
+        if self.cleaned_data.get('txt') or self.cleaned_data.get('xml'):
             # check group
             self.group = self.deduce_group()
 
             # check existing
-            existing = Submission.objects.filter(name=self.parsed_draft.filename, rev=self.parsed_draft.revision).exclude(state__in=("posted", "cancel"))
+            existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel"))
             if existing:
                 raise forms.ValidationError(mark_safe('Submission with same name and revision is currently being processed. <a href="%s">Check the status here.</a>' % urlreverse("submit_submission_status", kwargs={ 'submission_id': existing[0].pk })))
 
             # cut-off
-            if self.parsed_draft.revision == '00' and self.in_first_cut_off:
+            if self.revision == '00' and self.in_first_cut_off:
                 raise forms.ValidationError(mark_safe(self.cutoff_warning))
 
             # check thresholds
             today = datetime.date.today()
 
             self.check_submissions_tresholds(
-                "for the draft %s" % self.parsed_draft.filename,
-                dict(name=self.parsed_draft.filename, rev=self.parsed_draft.revision, submission_date=today),
+                "for the draft %s" % self.filename,
+                dict(name=self.filename, rev=self.revision, submission_date=today),
                 settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
             )
             self.check_submissions_tresholds(
@@ -166,19 +237,19 @@ def clean(self):
                 settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
             )
 
-        return super(UploadForm, self).clean()
+        return super(SubmissionUploadForm, self).clean()
 
     def check_submissions_tresholds(self, which, filter_kwargs, max_amount, max_size):
         submissions = Submission.objects.filter(**filter_kwargs)
 
         if len(submissions) > max_amount:
             raise forms.ValidationError("Max submissions %s has been reached for today (maximum is %s submissions)." % (which, max_amount))
-        if sum(s.file_size for s in submissions) > max_size * 1024 * 1024:
+        if sum(s.file_size for s in submissions if s.file_size) > max_size * 1024 * 1024:
             raise forms.ValidationError("Max uploaded amount %s has been reached for today (maximum is %s MB)." % (which, max_size))
 
     def deduce_group(self):
         """Figure out group from name or previously submitted draft, returns None if individual."""
-        name = self.parsed_draft.filename
+        name = self.filename
         existing_draft = Document.objects.filter(name=name, type="draft")
         if existing_draft:
             group = existing_draft[0].group

ietf/submit/parsers/base.py

@@ -1,4 +1,11 @@
+import os
 import re
+import magic
+import datetime
+import debug                            # pyflakes:ignore
+
+from django.conf import settings
+from django.template.defaultfilters import filesizeformat
 
 class MetaData(object):
     rev = None
@@ -42,6 +49,8 @@ def __init__(self, fd):
     # no other file parsing is recommended
     def critical_parse(self):
         self.parse_invalid_chars_in_filename()
+        self.parse_max_size();
+        self.parsed_info.metadata.submission_date = datetime.date.today()
         return self.parsed_info
 
     def parse_invalid_chars_in_filename(self):
@@ -50,3 +59,22 @@ def parse_invalid_chars_in_filename(self):
         chars = regexp.findall(name)
         if chars:
             self.parsed_info.add_error('Invalid characters were found in the name of the file which was just submitted: %s' % ', '.join(set(chars)))
+
+    def parse_max_size(self):
+        __, ext = os.path.splitext(self.fd.name)
+        ext = ext.lstrip('.')
+        max_size = settings.IDSUBMIT_MAX_DRAFT_SIZE[ext]
+        if self.fd.size > max_size:
+            self.parsed_info.add_error('File size is larger than the permitted maximum of %s' % filesizeformat(max_size))
+        self.parsed_info.metadata.file_size = self.fd.size
+
+    def parse_filename_extension(self, ext):
+        if not self.fd.name.lower().endswith('.'+ext):
+            self.parsed_info.add_error('Expected the %s file to have extension ".%s", found "%s"' % (ext.upper(), ext, self.fd.name))
+
+    def parse_file_type(self, ext, expected):
+        self.fd.file.seek(0)
+        content = self.fd.file.read(4096)
+        mimetype = magic.from_buffer(content, mime=True)
+        if not mimetype == expected:
+            self.parsed_info.add_error('Expected an %s file of type "%s", found one of type "%s"' % (expected, mimetype))

ietf/submit/parsers/pdf_parser.py

@@ -7,9 +7,6 @@ class PDFParser(FileParser):
     # no other file parsing is recommended
     def critical_parse(self):
         super(PDFParser, self).critical_parse()
-        self.parse_filename_extension()
+        self.parse_filename_extension('pdf')
+        self.parse_file_type('pdf', 'application/pdf')
         return self.parsed_info
-
-    def parse_filename_extension(self):
-        if not self.fd.name.endswith('.pdf'):
-            self.parsed_info.add_error('Format of this document must be PDF')

ietf/submit/parsers/plain_parser.py

@@ -1,8 +1,5 @@
-import datetime
 import re
 
-from django.conf import settings
-from django.template.defaultfilters import filesizeformat
 from ietf.submit.parsers.base import FileParser
 
 
@@ -15,17 +12,12 @@ def __init__(self, fd):
     # no other file parsing is recommended
     def critical_parse(self):
         super(PlainParser, self).critical_parse()
-        self.parse_max_size()
+        self.parse_filename_extension('txt')
+        self.parse_file_type('txt', 'text/plain')
         self.parse_file_charset()
         self.parse_name()
         return self.parsed_info
 
-    def parse_max_size(self):
-        if self.fd.size > settings.IDSUBMIT_MAX_PLAIN_DRAFT_SIZE:
-            self.parsed_info.add_error('File size is larger than %s' % filesizeformat(settings.IDSUBMIT_MAX_PLAIN_DRAFT_SIZE))
-        self.parsed_info.metadata.file_size = self.fd.size
-        self.parsed_info.metadata.submission_date = datetime.date.today()
-
     def parse_file_charset(self):
         import magic
         self.fd.file.seek(0)

ietf/submit/parsers/ps_parser.py

@@ -7,9 +7,6 @@ class PSParser(FileParser):
     # no other file parsing is recommended
     def critical_parse(self):
         super(PSParser, self).critical_parse()
-        self.parse_filename_extension()
+        self.parse_filename_extension('ps')
+        self.parse_file_type('ps', 'application/postscript')
         return self.parsed_info
-
-    def parse_filename_extension(self):
-        if not self.fd.name.endswith('.ps'):
-            self.parsed_info.add_error('Format of this document must be PS')

ietf/submit/parsers/xml_parser.py

@@ -7,9 +7,7 @@ class XMLParser(FileParser):
     # no other file parsing is recommended
     def critical_parse(self):
         super(XMLParser, self).critical_parse()
-        self.parse_filename_extension()
+        self.parse_filename_extension('xml')
+        self.parse_file_type('xml', 'application/xml')
         return self.parsed_info
-
-    def parse_filename_extension(self):
-        if not self.fd.name.endswith('.xml'):
-            self.parsed_info.add_error('Format of this document must be XML')
+