Added conversion of version 3 XML draft submissions to v3 HTML, in addition to text. Added a link to HTML versions (when present) to the document status page, and renamed the link to the htmlized version to 'htmlized'. Converted the submission XML test-file to version 3. Added test code to check that HTML is generated for version 3 XML submissions.

 - Legacy-Id: 16737

Author: levkowetz

ietf/doc/views_doc.py

@@ -206,7 +206,7 @@ def document_main(request, name, rev=None):
 
             # file types
             base_path = os.path.join(settings.RFC_PATH, name + ".")
-            possible_types = ["txt", "pdf", "ps"]
+            possible_types = settings.RFC_FILE_TYPES
             found_types = [t for t in possible_types if os.path.exists(base_path + t)]
 
             base = "https://www.rfc-editor.org/rfc/"
@@ -220,7 +220,7 @@ def document_main(request, name, rev=None):
                 file_urls.append(("pdf", base + "pdfrfc/" + name + ".txt.pdf"))
 
             if "txt" in found_types:
-                file_urls.append(("html", settings.TOOLS_ID_HTML_URL + name))
+                file_urls.append(("htmlized", settings.TOOLS_ID_HTML_URL + name))
                 if doc.tags.filter(slug="errata"):
                     file_urls.append(("with errata", settings.RFC_EDITOR_INLINE_ERRATA_URL.format(rfc_number=rfc_number)))
 
@@ -236,8 +236,8 @@ def document_main(request, name, rev=None):
 
             # file types
             base_path = os.path.join(settings.INTERNET_DRAFT_PATH, doc.name + "-" + doc.rev + ".")
-            possible_types = ["pdf", "xml", "ps"]
-            found_types = ["txt"] + [t for t in possible_types if os.path.exists(base_path + t)]
+            possible_types = settings.IDSUBMIT_FILE_TYPES
+            found_types = [t for t in possible_types if os.path.exists(base_path + t)]
 
             if not snapshot and doc.get_state_slug() == "active":
                 base = settings.IETF_ID_URL
@@ -251,7 +251,7 @@ def document_main(request, name, rev=None):
 
             if "pdf" not in found_types:
                 file_urls.append(("pdf", settings.TOOLS_ID_PDF_URL + doc.name + "-" + doc.rev + ".pdf"))
-            file_urls.append(("html", settings.TOOLS_ID_HTML_URL + doc.name + "-" + doc.rev))
+            file_urls.append(("htmlized", settings.TOOLS_ID_HTML_URL + doc.name + "-" + doc.rev))
 
             # latest revision
             latest_revision = doc.latest_event(NewRevisionDocEvent, type="new_revision")

ietf/settings.py

@@ -782,13 +782,17 @@ def skip_unreadable_post(record):
 
 IDSUBMIT_FILE_TYPES = (
     'txt',
+    'html',
     'xml',
     'pdf',
     'ps',
 )
+RFC_FILE_TYPES = IDSUBMIT_FILE_TYPES
+
 IDSUBMIT_MAX_DRAFT_SIZE =  {
     'txt':  2*1024*1024,  # Max size of txt draft file in bytes
     'xml':  3*1024*1024,  # Max size of xml draft file in bytes
+    'html': 4*1024*1024,
     'pdf':  6*1024*1024,
     'ps' :  6*1024*1024,
 }

ietf/submit/forms.py

@@ -352,12 +352,16 @@ def deduce_group(self):
 class SubmissionManualUploadForm(SubmissionBaseUploadForm):
     xml = forms.FileField(label='.xml format', required=False) # xml field with required=False instead of True
     txt = forms.FileField(label='.txt format', required=False)
+    # We won't permit html upload until we can verify that the content
+    # reasonably matches the text and/or xml upload.  Till then, we generate
+    # html for version 3 xml submissions.
+    # html = forms.FileField(label='.html format', required=False)
     pdf = forms.FileField(label='.pdf format', required=False)
     ps  = forms.FileField(label='.ps format', required=False)
 
     def __init__(self, request, *args, **kwargs):
         super(SubmissionManualUploadForm, self).__init__(request, *args, **kwargs)
-        self.formats = ['txt', 'pdf', 'xml', 'ps', ]
+        self.formats = settings.IDSUBMIT_FILE_TYPES
         self.base_formats = ['txt', 'xml', ]
 
     def clean_txt(self):

ietf/submit/test_submission.xml

@@ -1,12 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE rfc SYSTEM "rfc2629.dtd" []>
+<!DOCTYPE rfc SYSTEM "rfc2629-xhtml.ent">
 <?rfc toc="yes"?>
-<rfc category="info" docName="%(name)s" ipr="trust200902">
+<rfc xmlns:xi="http://www.w3.org/2001/XInclude" category="info" docName="%(name)s" ipr="trust200902" obsoletes="" updates="" submissionType="IETF" xml:lang="en" version="3">
   <front>
     <title>%(title)s</title>
+    <seriesInfo name="Internet-Draft" value="%(name)s"/>
     <author fullname="%(author)s" initials="%(initials)s" surname="%(surname)s">
       <organization>Test Centre Inc.</organization>
-
       <address>
         <postal>
           <street>42 Some Road</street>
@@ -16,26 +16,24 @@
         <email>%(email)s</email>
       </address>
     </author>
-    <date month="%(month)s" year="%(year)s" />
+    <date day="%(day)s" month="%(month)s" year="%(year)s"/>
     <workgroup>%(group)s</workgroup>
     <abstract>
       <t>
          This document describes how to test tests.
       </t>
     </abstract>
   </front>
-
   <middle>
-    <section title="Introduction">
+    <section numbered="true" toc="default">
+      <name>Introduction</name>
       <t>
          This document describes a protocol for testing tests.
       </t>
     </section>
-    <section title="Yang">
-       <figure>
-          <artwork>
-             <![CDATA[
-<CODE BEGINS> file "ietf-yang-metadata@2016-08-05.yang"
+    <section numbered="true" toc="default">
+      <name>Yang</name>
+      <sourcecode name="ietf-yang-metadata@2016-08-05.yang" type="" markers="true"><![CDATA[
 
 module ietf-yang-metadata {
 
@@ -122,13 +120,10 @@ module ietf-yang-metadata {
  }
 }
 
-<CODE ENDS>
-             ]]>
-          </artwork>
-       </figure>
+]]></sourcecode>
     </section>
-
-    <section anchor="JSON" title="JSON example">
+    <section anchor="JSON" numbered="true" toc="default">
+      <name>JSON example</name>
       <t>
    The JSON object should look like this:
 
@@ -137,17 +132,18 @@ module ietf-yang-metadata {
    }
       </t>
     </section>
-    <section anchor="Security" title="Security Considerations">
+    <section anchor="Security" numbered="true" toc="default">
+      <name>Security Considerations</name>
       <t>
          There are none.
       </t>
     </section>
-    <section anchor="IANA" title="IANA Considerations">
+    <section anchor="IANA" numbered="true" toc="default">
+      <name>IANA Considerations</name>
       <t>
          No new registrations for IANA.
       </t>
     </section>
   </middle>
-  <back>
-  </back>
+  <back/>
 </rfc>

ietf/submit/tests.py

@@ -64,6 +64,7 @@ def submission_file(name, rev, group, format, templatename, author=None, email=N
             expiration=(datetime.date.today() + datetime.timedelta(days=100)).strftime("%d %B, %Y"),
             year=year,
             month=datetime.date.today().strftime("%B"),
+            day=datetime.date.today().strftime("%d"),
             name="%s-%s" % (name, rev),
             group=group or "",
             author=author.ascii if ascii else author.name,
@@ -151,6 +152,8 @@ def do_submission(self, name, rev, group=None, formats=["txt",], author=None):
         status_url = r["Location"]
         for format in formats:
             self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, format))))
+            if format == 'xml':
+                self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, 'html'))))
         self.assertEqual(Submission.objects.filter(name=name).count(), 1)
         submission = Submission.objects.get(name=name)
         if len(submission.authors) != 1:
@@ -1684,7 +1687,7 @@ def test_api_submit_no_title(self):
         self.assertContains(r, expected, status_code=400)
 
     def test_api_submit_failed_idnits(self):
-        r, author, name = self.post_submission('00', year="1900")
+        r, author, name = self.post_submission('00', year="2010")
         expected = "Document date must be within 3 days of submission date"
         self.assertContains(r, expected, status_code=400)
 

ietf/submit/utils.py

@@ -528,16 +528,14 @@ def cancel_submission(submission):
     remove_submission_files(submission)
 
 def rename_submission_files(submission, prev_rev, new_rev):
-    from ietf.submit.forms import SubmissionManualUploadForm
-    for ext in list(SubmissionManualUploadForm.base_fields.keys()):
+    for ext in settings.IDSUBMIT_FILE_TYPES:
         source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, prev_rev, ext))
         dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, new_rev, ext))
         if os.path.exists(source):
             os.rename(source, dest)
 
 def move_files_to_repository(submission):
-    from ietf.submit.forms import SubmissionManualUploadForm
-    for ext in list(SubmissionManualUploadForm.base_fields.keys()):
+    for ext in settings.IDSUBMIT_FILE_TYPES:
         source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext))
         dest = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext))
         if os.path.exists(source):
@@ -622,18 +620,24 @@ def get_draft_meta(form, saved_files):
     file_name = saved_files
     abstract = None
     file_size = None
+    xml2rfc.log.write_out = open(os.devnull, "w")
+    xml2rfc.log.write_err = open(os.devnull, "w")
     if form.cleaned_data['xml']:
+        try:
+            xmlroot = form.xmltree.getroot()
+            xml_version = xmlroot.get('version', '2')
+            if xml_version == '3':
+                prep = xml2rfc.PrepToolWriter(form.xmltree, quiet=True)
+                form.xmltree.tree = prep.prep()
+        except Exception as e:
+            raise ValidationError("Error from xml2rfc (prep): %s" % e)
         if not ('txt' in form.cleaned_data and form.cleaned_data['txt']):
             file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (form.filename, form.revision))
             try:
-                xmlroot = form.xmltree.getroot()
-                xml_version = xmlroot.get('version', '2')
                 if xml_version != '3':
                     pagedwriter = xml2rfc.PaginatedTextRfcWriter(form.xmltree, quiet=True)
                     pagedwriter.write(file_name['txt'])
                 else:
-                    prep = xml2rfc.PrepToolWriter(form.xmltree, quiet=True)
-                    form.xmltree.tree = prep.prep()
                     writer = xml2rfc.TextWriter(form.xmltree, quiet=True)
                     writer.write(file_name['txt'])
                 log.log("In %s: xml2rfc %s generated %s from %s (version %s)" %
@@ -643,8 +647,22 @@ def get_draft_meta(form, saved_files):
                             os.path.basename(file_name['xml']),
                             xml_version))
             except Exception as e:
-                raise ValidationError("Error from xml2rfc: %s" % e)
+                raise ValidationError("Error from xml2rfc (text): %s" % e) 
             file_size = os.stat(file_name['txt']).st_size
+        if xml_version == '3':
+            try:
+                file_name['html'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.html' % (form.filename, form.revision))
+                writer = xml2rfc.HtmlWriter(form.xmltree, quiet=True)
+                writer.write(file_name['html'])
+                form.file_types.append('.html')
+                log.log("In %s: xml2rfc %s generated %s from %s (version %s)" %
+                    (   os.path.dirname(file_name['xml']),
+                        xml2rfc.__version__,
+                        os.path.basename(file_name['html']),
+                        os.path.basename(file_name['xml']),
+                        xml_version))
+            except Exception as e:
+                raise ValidationError("Error from xml2rfc (html): %s" % e) 
         # Some meta-information, such as the page-count, can only
         # be retrieved from the generated text file.  Provide a
         # parsed draft object to get at that kind of information.