Add ability to import session minutes from notes.ietf.org. Mock out calls to the requests library in tests. Call markdown library through a util method. Fixes ietf-tools#3489. Commit ready for merge.

 - Legacy-Id: 19763

Author: jennifer-richards

ietf/doc/views_bofreq.py

@@ -3,7 +3,6 @@
 import debug    # pyflakes:ignore
 
 import io 
-import markdown
 
 from django import forms
 from django.contrib.auth.decorators import login_required
@@ -20,6 +19,7 @@
 from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible
 from ietf.ietfauth.utils import has_role, role_required
 from ietf.person.fields import SearchablePersonsField
+from ietf.utils import markdown
 from ietf.utils.response import permission_denied
 from ietf.utils.text import xslugify
 from ietf.utils.textupload import get_cleaned_text_file_content
@@ -64,7 +64,7 @@ def require_field(f):
             if require_field("bofreq_file"):
                 content = get_cleaned_text_file_content(self.cleaned_data["bofreq_file"])
         try:
-            _ = markdown.markdown(content, extensions=['extra'])
+            _ = markdown.markdown(content)
         except Exception as e:
            raise forms.ValidationError(f'Markdown processing failed: {e}')
 

ietf/doc/views_doc.py

@@ -40,7 +40,6 @@
 import json
 import os
 import re
-import markdown
 
 from urllib.parse import quote
 
@@ -80,7 +79,7 @@
 from ietf.review.models import ReviewAssignment
 from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
-from ietf.utils import markup_txt, log
+from ietf.utils import markup_txt, log, markdown
 from ietf.utils.draft import Draft
 from ietf.utils.response import permission_denied
 from ietf.utils.text import maybe_split
@@ -550,7 +549,7 @@ def document_main(request, name, rev=None):
                                        ))
 
     if doc.type_id == "bofreq":
-        content = markdown.markdown(doc.text_or_error(),extensions=['extra'])
+        content = markdown.markdown(doc.text_or_error())
         editors = bofreq_editors(doc)
         responsible = bofreq_responsible(doc)
         can_manage = has_role(request.user,['Secretariat', 'Area Director', 'IAB'])
@@ -661,7 +660,7 @@ def document_main(request, name, rev=None):
                 content = doc.text_or_error()
                 t = "plain text"
             elif extension == ".md":
-                content = markdown.markdown(doc.text_or_error(), extensions=['extra'])
+                content = markdown.markdown(doc.text_or_error())
                 content_is_html = True
                 t = "markdown"
             other_types.append((t, url))

ietf/group/views.py

@@ -38,7 +38,6 @@
 import datetime
 import itertools
 import io
-import markdown
 import math
 import os
 import re
@@ -121,7 +120,7 @@
 from ietf.utils.pipe import pipe
 from ietf.utils.response import permission_denied
 from ietf.utils.text import strip_suffix
-
+from ietf.utils import markdown
 
 
 # --- Helpers ----------------------------------------------------------
@@ -581,7 +580,7 @@ def group_about_rendertest(request, acronym, group_type=None):
     if group.charter:
         charter = get_charter_text(group)
     try:
-        rendered = markdown.markdown(charter, extensions=['extra'])
+        rendered = markdown.markdown(charter)
     except Exception as e:
         rendered = f'Markdown rendering failed: {e}'
     return render(request, 'group/group_about_rendertest.html', {'group':group, 'charter':charter, 'rendered':rendered})

ietf/meeting/forms.py

@@ -436,6 +436,10 @@ def clean_title(self):
         return title
 
 
+class ImportMinutesForm(forms.Form):
+    markdown_text = forms.CharField(strip=False, widget=forms.HiddenInput)
+
+
 class RequestMinutesForm(forms.Form):
     to = MultiEmailField()
     cc = MultiEmailField(required=False)

ietf/meeting/models.py

@@ -14,6 +14,7 @@
 
 from collections import namedtuple
 from pathlib import Path
+from urllib.parse import urljoin
 
 import debug                            # pyflakes:ignore
 
@@ -1260,6 +1261,13 @@ def jabber_room_name(self):
         else:
             return self.group.acronym
 
+    def notes_id(self):
+        note_id_fragment = 'plenary' if self.type.slug == 'plenary' else self.group.acronym
+        return f'notes-ietf-{self.meeting.number}-{note_id_fragment}'
+
+    def notes_url(self):
+        return urljoin(settings.IETF_NOTES_URL, self.notes_id())
+
 class SchedulingEvent(models.Model):
     session = ForeignKey(Session)
     time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened")

ietf/meeting/tests_views.py

@@ -8,6 +8,8 @@
 import re
 import shutil
 import pytz
+import requests.exceptions
+import requests_mock
 
 from unittest import skipIf
 from mock import patch, PropertyMock
@@ -19,7 +21,6 @@
 from PIL import Image
 from pathlib import Path
 
-
 from django.urls import reverse as urlreverse
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -5405,12 +5406,16 @@ def test_iphone_app_json(self):
             self.assertTrue(msessions.filter(group__acronym=s['group']['acronym']).exists())
 
 class FinalizeProceedingsTests(TestCase):
-    @patch('urllib.request.urlopen')
-    def test_finalize_proceedings(self, mock_urlopen):
-        mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
+    @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
+    @requests_mock.Mocker()
+    def test_finalize_proceedings(self, mock):
         make_meeting_test_data()
         meeting = Meeting.objects.filter(type_id='ietf').order_by('id').last()
         meeting.session_set.filter(group__acronym='mars').first().sessionpresentation_set.create(document=Document.objects.filter(type='draft').first(),rev=None)
+        mock.get(
+            settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
+            text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
+        )
 
         url = urlreverse('ietf.meeting.views.finalize_proceedings',kwargs={'num':meeting.number})
         login_testing_unauthorized(self,"secretary",url)
@@ -5605,8 +5610,10 @@ def test_upload_minutes_agenda(self):
             self.assertEqual(doc.rev,'02')
 
             # Verify that we don't have dead links
-            url = url=urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
+            url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
             top = '/meeting/%s/' % session.meeting.number
+            self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes')
+            self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'}))
             self.crawl_materials(url=url, top=top)
 
     def test_upload_minutes_agenda_unscheduled(self):
@@ -5653,8 +5660,10 @@ def test_upload_minutes_agenda_interim(self):
             self.assertEqual(doc.rev,'00')
 
             # Verify that we don't have dead links
-            url = url=urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
+            url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
             top = '/meeting/%s/' % session.meeting.number
+            self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes')
+            self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'}))
             self.crawl_materials(url=url, top=top)
 
     def test_upload_slides(self):
@@ -5928,6 +5937,151 @@ def test_submit_and_approve_multiple_versions(self):
         self.assertIn('third version', contents)
 
 
+@override_settings(IETF_NOTES_URL='https://notes.ietf.org/')
+class ImportNotesTests(TestCase):
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH']
+
+    def setUp(self):
+        super().setUp()
+        self.session = SessionFactory(meeting__type_id='ietf')
+        self.meeting = self.session.meeting
+
+    def test_retrieves_note(self):
+        """Can import and preview a note from notes.ietf.org"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        with requests_mock.Mocker() as mock:
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='markdown text')
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
+                     text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            iframe = q('iframe#preview')
+            self.assertEqual('<p>markdown text</p>', iframe.attr('srcdoc'))
+            markdown_text_input = q('form #id_markdown_text')
+            self.assertEqual(markdown_text_input.val(), 'markdown text')
+
+    def test_retrieves_with_broken_metadata(self):
+        """Can import and preview a note even if it has a metadata problem"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        with requests_mock.Mocker() as mock:
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='markdown text')
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info', text='this is not valid json {]')
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            iframe = q('iframe#preview')
+            self.assertEqual('<p>markdown text</p>', iframe.attr('srcdoc'))
+            markdown_text_input = q('form #id_markdown_text')
+            self.assertEqual(markdown_text_input.val(), 'markdown text')
+
+    def test_redirects_on_success(self):
+        """Redirects to session details page after import"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        r = self.client.post(url, {'markdown_text': 'markdown text'})
+        self.assertRedirects(
+            r,
+            urlreverse(
+                'ietf.meeting.views.session_details',
+                kwargs={
+                    'num': self.meeting.number,
+                    'acronym': self.session.group.acronym,
+                },
+            ),
+        )
+
+    def test_imports_previewed_text(self):
+        """Import text that was shown as preview even if notes site is updated"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        with requests_mock.Mocker() as mock:
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='updated markdown text')
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
+                     text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
+            r = self.client.post(url, {'markdown_text': 'original markdown text'})
+        self.assertEqual(r.status_code, 302)
+        minutes_path = Path(self.meeting.get_materials_path()) / 'minutes'
+        with (minutes_path / self.session.minutes().uploaded_filename).open() as f:
+            self.assertEqual(f.read(), 'original markdown text')
+
+    def test_refuses_identical_import(self):
+        """Should not be able to import text identical to the current revision"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        r = self.client.post(url, {'markdown_text': 'original markdown text'})  # create a rev
+        self.assertEqual(r.status_code, 302)
+        with requests_mock.Mocker() as mock:
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='original markdown text')
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
+                     text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
+            r = self.client.get(url)  # try to import the same text
+            self.assertContains(r, "This document is identical", status_code=200)
+            q = PyQuery(r.content)
+            self.assertEqual(len(q('button:disabled[type="submit"]')), 1)
+            self.assertEqual(len(q('button:not(:disabled)[type="submit"]')), 0)
+
+    def test_handles_missing_previous_revision_file(self):
+        """Should still allow import if the file for the previous revision is missing"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        r = self.client.post(url, {'markdown_text': 'original markdown text'})  # create a rev
+        # remove the file uploaded for the first rev
+        minutes_docs = self.session.sessionpresentation_set.filter(document__type='minutes')
+        self.assertEqual(minutes_docs.count(), 1)
+        Path(minutes_docs.first().document.get_file_name()).unlink()
+
+        self.assertEqual(r.status_code, 302)
+        with requests_mock.Mocker() as mock:
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='original markdown text')
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
+                     text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            iframe = q('iframe#preview')
+            self.assertEqual('<p>original markdown text</p>', iframe.attr('srcdoc'))
+            markdown_text_input = q('form #id_markdown_text')
+            self.assertEqual(markdown_text_input.val(), 'original markdown text')
+
+    def test_handles_note_does_not_exist(self):
+        """Should not try to import a note that does not exist"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        with requests_mock.Mocker() as mock:
+            mock.get(requests_mock.ANY, status_code=404)
+            r = self.client.get(url, follow=True)
+        self.assertContains(r, 'Could not import', status_code=200)
+
+    def test_handles_notes_server_failure(self):
+        """Problems communicating with the notes server should be handled gracefully"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+        self.client.login(username='secretary', password='secretary+password')
+
+        with requests_mock.Mocker() as mock:
+            mock.get(re.compile(r'.+/download'), exc=requests.exceptions.ConnectTimeout)
+            mock.get(re.compile(r'.+//info'), text='{}')
+            r = self.client.get(url, follow=True)
+        self.assertContains(r, 'Could not reach the notes server', status_code=200)
+
+
 class SessionTests(TestCase):
 
     def test_meeting_requests(self):
@@ -6911,27 +7065,34 @@ def test_proceedings_acknowledgements_link(self):
             0,
         )
 
-    @patch('ietf.meeting.utils.requests.get')
-    def test_proceedings_attendees(self, mockobj):
-        mockobj.return_value.text = b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]'
-        mockobj.return_value.json = lambda: json.loads(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
+    @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
+    @requests_mock.Mocker()
+    def test_proceedings_attendees(self, mock):
         make_meeting_test_data()
         meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="97")
+        mock.get(
+            settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
+            text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
+        )
         finalize(meeting)
         url = urlreverse('ietf.meeting.views.proceedings_attendees',kwargs={'num':97})
         response = self.client.get(url)
         self.assertContains(response, 'Attendee List')
         q = PyQuery(response.content)
         self.assertEqual(1,len(q("#id_attendees tbody tr")))
 
-    @patch('urllib.request.urlopen')
-    def test_proceedings_overview(self, mock_urlopen):
+    @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
+    @requests_mock.Mocker()
+    def test_proceedings_overview(self, mock):
         '''Test proceedings IETF Overview page.
         Note: old meetings aren't supported so need to add a new meeting then test.
         '''
-        mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
         make_meeting_test_data()
         meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="97")
+        mock.get(
+            settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
+            text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
+        )
         finalize(meeting)
         url = urlreverse('ietf.meeting.views.proceedings_overview',kwargs={'num':97})
         response = self.client.get(url)

ietf/meeting/urls.py

@@ -13,6 +13,7 @@
     url(r'^session/(?P<session_id>\d+)/bluesheets$', views.upload_session_bluesheets),
     url(r'^session/(?P<session_id>\d+)/minutes$', views.upload_session_minutes),
     url(r'^session/(?P<session_id>\d+)/agenda$', views.upload_session_agenda),
+    url(r'^session/(?P<session_id>\d+)/import/minutes$', views.import_session_minutes),
     url(r'^session/(?P<session_id>\d+)/propose_slides$', views.propose_session_slides),
     url(r'^session/(?P<session_id>\d+)/slides(?:/%(name)s)?$' % settings.URL_REGEXPS, views.upload_session_slides),
     url(r'^session/(?P<session_id>\d+)/add_to_session$', views.ajax_add_slides_to_session),