Added a management command to generate draft bibxml files, and also a trial version of datatracker draft bibxml pages.

 - Legacy-Id: 16962

Author: levkowetz

ietf/doc/factories.py

@@ -30,6 +30,7 @@ class Meta:
         model = Document
 
     title = factory.Faker('sentence',nb_words=5)
+    abstract = factory.Faker('paragraph', nb_sentences=5)
     rev = '00'
     std_level_id = None                 # type: Optional[str]
     intended_std_level_id = None

ietf/doc/management/commands/generate_draft_bibxml_files.py

@@ -1,44 +1,95 @@
-# Copyright The IETF Trust 2012-2019, All Rights Reserved
+# Copyright The IETF Trust 2018-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-
 from __future__ import absolute_import, print_function, unicode_literals
 
+import datetime
 import io
-import sys
 import os
+import re
+import sys
 
-from django.core.management.base import BaseCommand
 from django.conf import settings
+from django.core.management.base import BaseCommand
 from django.template.loader import render_to_string
 
-from ietf.doc.models import Document
-
-def write(fn, new):
-    try:
-        f = io.open(fn)
-        old = f.read().decode('utf-8')
-        f.close
-    except IOError:
-        old = ""
-    if old.strip() != new.strip():
-        sys.stdout.write(os.path.basename(fn)+'\n')
-        f = io.open(fn, "wb")
-        f.write(new.encode('utf-8'))
-        f.close()
+import debug                            # pyflakes:ignore
+
+from ietf.doc.models import NewRevisionDocEvent
+
+DEFAULT_DAYS = 7
 
 class Command(BaseCommand):
-    help = ('Generate draft bibxml files, for xml2rfc references')
+    help = ('Generate draft bibxml files for xml2rfc references, placing them in the '
+            'directory configured in settings.BIBXML_BASE_PATH: %s.  '
+            'By default, generate files as needed for new draft revisions from the '
+            'last %s days.' % (settings.BIBXML_BASE_PATH, DEFAULT_DAYS))
+
+    def add_arguments(self, parser):
+        parser.add_argument('--all', action='store_true', default=False, help="Process all documents, not only recent submissions")
+        parser.add_argument('--days', type=int, default=DEFAULT_DAYS, help="Look submissions from the last DAYS days, instead of %s" % DEFAULT_DAYS)
+
+    def say(self, msg):
+        if self.verbosity > 0:
+            sys.stdout.write(msg)
+            sys.stdout.write('\n')
+
+    def note(self, msg):
+        if self.verbosity > 1:
+            sys.stdout.write(msg)
+            sys.stdout.write('\n')
+
+    def mutter(self, msg):
+        if self.verbosity > 2:
+            sys.stdout.write(msg)
+            sys.stdout.write('\n')
+
+    def write(self, fn, new):
+        # normalize new
+        new = re.sub(r'\r\n?', r'\n', new)
+        try:
+            with io.open(fn, encoding='utf-8') as f:
+                old = f.read()
+        except IOError:
+            old = ""
+        if old.strip() != new.strip():
+            self.note('Writing %s' % os.path.basename(fn))
+            with io.open(fn, "w", encoding='utf-8') as f:
+                f.write(new)
 
     def handle(self, *args, **options):
-        documents = Document.objects.filter(type__slug='draft')
+        self.verbosity = options.get("verbosity", 1)
+        process_all = options.get("all")
+        days = options.get("days")
+        #
         bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml3')
         if not os.path.exists(bibxmldir):
             os.makedirs(bibxmldir)
-        for doc in documents:
-            ref_text = render_to_string('doc/bibxml.xml', {'doc': doc, 'doc_bibtype':'I-D'})
-            ref_file_name = os.path.join(bibxmldir, 'reference.I-D.%s.xml' % (doc.name, ))
-            ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (doc.name, doc.rev))
-            write(ref_file_name, ref_text)
-            write(ref_rev_file_name, ref_text)
-                
+        #
+        if process_all:
+            doc_events = NewRevisionDocEvent.objects.filter(type='new_revision', doc__type_id='draft')
+        else:
+            start = datetime.datetime.now() - datetime.timedelta(days=days)
+            doc_events = NewRevisionDocEvent.objects.filter(type='new_revision', doc__type_id='draft', time__gte=start)
+        doc_events = doc_events.order_by('time')
+
+        for e in doc_events:
+            self.mutter('%s %s' % (e.time, e.doc.name))
+            try:
+                e.doc.date = e.time.date()
+                doc = e.doc
+                if e.rev != doc.rev:
+                    for h in doc.history_set.order_by("-time"):
+                        if e.rev == h.rev:
+                            doc = h
+                            break
+                ref_text = '%s' % render_to_string('doc/bibxml.xml', {'doc': doc, 'doc_bibtype':'I-D'})
+                if e.rev == e.doc.rev:
+                    ref_file_name = os.path.join(bibxmldir, 'reference.I-D.%s.xml' % (doc.name[6:], ))
+                    self.write(ref_file_name, ref_text)
+                else:
+                    self.note("Skipping %s; outdated revision: %s" % (os.path.basename(ref_file_name), e.rev))
+                ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (doc.name, doc.rev))
+                self.write(ref_rev_file_name, ref_text)
+            except Exception as ee:
+                sys.stderr.write('\n%s-%s: %s\n' % (doc.name, doc.rev, ee))

ietf/doc/tests.py

@@ -8,6 +8,7 @@
 import shutil
 import datetime
 import io
+import lxml
 import sys
 import bibtexparser
 
@@ -46,6 +47,7 @@
 from ietf.utils.mail import outbox
 from ietf.utils.test_utils import login_testing_unauthorized, unicontent
 from ietf.utils.test_utils import TestCase
+from ietf.utils.text import normalize_text
 
 class SearchTests(TestCase):
     def test_search(self):
@@ -972,6 +974,22 @@ def test_document_bibtex(self):
         #
         self.assertNotIn('doi', entry)
 
+    def test_document_bibxml(self):
+
+        draft = IndividualDraftFactory.create()
+        docname = '%s-%s' % (draft.name, draft.rev)
+        url = urlreverse('ietf.doc.views_doc.document_bibxml', kwargs=dict(name=draft.name))
+        r = self.client.get(url)
+        entry = lxml.etree.fromstring(r.content)
+        self.assertEqual(entry.find('./front/title').text, draft.title)
+        date = entry.find('./front/date')
+        self.assertEqual(date.get('year'),     str(draft.pub_date().year))
+        self.assertEqual(date.get('month'),    draft.pub_date().strftime('%B'))
+        self.assertEqual(date.get('day'),      str(draft.pub_date().day))
+        self.assertEqual(normalize_text(entry.find('./front/abstract/t').text), normalize_text(draft.abstract))
+        self.assertEqual(entry.find('./seriesInfo').get('value'), docname)
+        self.assertEqual(entry.find('./seriesInfo[@name="DOI"]'), None)
+
 class AddCommentTestCase(TestCase):
     def test_add_comment(self):
         draft = WgDraftFactory(name='draft-ietf-mars-test',group__acronym='mars')

ietf/doc/urls.py

@@ -74,6 +74,7 @@
 
     url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main),
     url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex),
+    url(r'^bibxml3/%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml),
     url(r'^%(name)s/history/$' % settings.URL_REGEXPS, views_doc.document_history),
     url(r'^%(name)s/writeup/$' % settings.URL_REGEXPS, views_doc.document_writeup),
     url(r'^%(name)s/email/$' % settings.URL_REGEXPS, views_doc.document_email),

ietf/doc/views_doc.py

@@ -824,7 +824,37 @@ def document_bibtex(request, name, rev=None):
                                    latest_revision=latest_revision,
                                    doi=doi,
                                ),
-                              content_type="text/plain; charset=utf-8",
+                              content_type="application/xml; charset=utf-8",
+                          )
+
+
+def document_bibxml(request, name, rev=None):
+    # This only deals with drafts, as bibxml entries for RFCs should come from
+    # the RFC-Editor.
+    doc = get_object_or_404(Document, docalias__name=name, name__startswith='draft-', type_id='draft')
+
+    latest_revision = doc.latest_event(NewRevisionDocEvent, type="new_revision")
+    latest_rev = latest_revision.rev if latest_revision else None
+        
+    if rev != None and rev != doc.rev:
+        # find the entry in the history
+        for h in doc.history_set.order_by("-time"):
+            if rev == h.rev:
+                doc = h
+                break
+
+    try:
+        doc_event = NewRevisionDocEvent.objects.get(doc__name=doc.name, rev=(rev or latest_rev))
+        doc.date = doc_event.time.date()
+    except DocEvent.DoesNotExist:
+        doc.date = doc.time.date()      # Even if this may be incoreect, what would be better?
+
+    return render(request, "doc/bibxml.xml",
+                              dict(
+                                  doc=doc,
+                                  doc_bibtype='I-D',
+                               ),
+                              content_type="application/xml; charset=utf-8",
                           )
 
 

ietf/templates/doc/bibxml.xml

@@ -1,15 +1,20 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<reference anchor='{{doc_bibtype}}.{{doc.name|slice:"6:"}}'>
+<?xml version="1.0" encoding="UTF-8"?>
+<reference anchor="{{doc_bibtype}}.{{doc.name|slice:"6:"}}">
    <front>
-      <title>{{doc.title}}</title>
-      {% for author in doc.documentauthor_set.all %}
-      <author initials='{{ author.person.initials }}' surname='{{ author.person.last_name }}' fullname='{{ author.person.name }}'>
-         <organization>{{ author.affiliation }}</organization>
-      </author>
-      {% endfor %}
-      <date month='{{doc.time|date:"F"}}' day='{{doc.time.day}}' year='{{doc.time.year}}' />
-      <abstract><t>{{doc.abstract}}</t></abstract>
+      <title>{{doc.title}}</title>{% if doc.submission %}{% for author in doc.submission.authors %}
+      <author fullname="{{ author.name }}">
+	 {% if author.affiliation %}<organization>{{ author.affiliation }}</organization>
+      {% endif %}</author>{% endfor %}{% else %}{% for author in doc.documentauthor_set.all %}
+      <author initials="{{ author.person.initials }}" surname="{{ author.person.last_name }}" fullname="{{ author.person.name }}">
+         {% if author.affiliation %}<organization>{{ author.affiliation }}</organization>
+      {% endif %}</author>{% endfor %}{% endif %}
+      <date month="{{doc.date|date:"F"}}" day="{{doc.date.day}}" year="{{doc.date.year}}" />
+      <abstract>
+	 <t>{{doc.abstract}}
+	 </t>
+      </abstract>
    </front>
-   <seriesInfo name='Internet-Draft' value='{{doc.name}}-{{doc.rev}}' />
-   <format type='TXT' target='https://www.ietf.org/internet-drafts/{{doc.name}}-{{doc.rev}}.txt' />
+   <seriesInfo name="Internet-Draft" value="{{doc.name}}-{{doc.rev}}" />
+   {% if doi %}<seriesInfo name="DOI" value="{{doi}}" />
+   {% endif %}<format type="TXT" target="https://www.ietf.org/internet-drafts/{{doc.name}}-{{doc.rev}}.txt" />
 </reference>

ietf/utils/text.py

@@ -194,3 +194,5 @@ def texescape(s):
 def unwrap(s):
     return s.replace('\n', ' ')
 
+def normalize_text(s):
+    return s.replace(r'\s+', ' ').strip()