fix: keep the ftp dir in sync when the datatracker creates artifacts (ietf-tools#8401)

* chore: fix typo in log message

* fix: write new conflict reviews to the ftp directory

* fix: log exception description

* chore: avoid reusing a name for mypy

* fix: write non-meeting-related slides to ftp/slides

* fix: write status changes to ftp dir

* fix: copy draft yang modules to the ftp dir

the last commit above (see ietf-tools#8401) has no tests :(. There aren't tests of this yang scaffolding to add to. Adding a set of tests should be an independent larger effort.

Author: rjsparks

ietf/doc/tests_conflict_review.py

@@ -4,6 +4,7 @@
 
 import io
 import os
+from pathlib import Path
 
 from pyquery import PyQuery
 from textwrap import wrap
@@ -387,7 +388,7 @@ def setUp(self):
 
 
 class ConflictReviewSubmitTests(TestCase):
-    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CONFLICT_REVIEW_PATH']
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CONFLICT_REVIEW_PATH','FTP_PATH']
     def test_initial_submission(self):
         doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
         url = urlreverse('ietf.doc.views_conflict_review.submit',kwargs=dict(name=doc.name))
@@ -403,16 +404,23 @@ def test_initial_submission(self):
         # Right now, nothing to test - we let people put whatever the web browser will let them put into that textbox
 
         # sane post using textbox
-        path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.name, doc.rev))
+        basename = f"{doc.name}-{doc.rev}.txt"
+        path = Path(settings.CONFLICT_REVIEW_PATH) / basename
+        ftp_dir = Path(settings.FTP_DIR) / "conflict-reviews"
+        if not ftp_dir.exists():
+            ftp_dir.mkdir()
+        ftp_path = ftp_dir / basename
         self.assertEqual(doc.rev,'00')
-        self.assertFalse(os.path.exists(path))
+        self.assertFalse(path.exists())
+        self.assertFalse(ftp_path.exists())
         r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1"))
         self.assertEqual(r.status_code,302)
         doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
         self.assertEqual(doc.rev,'00')
         with io.open(path) as f:
             self.assertEqual(f.read(),"Some initial review text\n")
             f.close()
+        self.assertTrue(ftp_path.exists())
         self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc)
 
     def test_subsequent_submission(self):

ietf/doc/tests_material.py

@@ -28,7 +28,7 @@
 
 
 class GroupMaterialTests(TestCase):
-    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH']
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH', 'FTP_DIR']
     def setUp(self):
         super().setUp()
         self.materials_dir = self.tempdir("materials")
@@ -37,6 +37,10 @@ def setUp(self):
             self.slides_dir.mkdir()
         self.saved_document_path_pattern = settings.DOCUMENT_PATH_PATTERN
         settings.DOCUMENT_PATH_PATTERN = self.materials_dir + "/{doc.type_id}/"
+        self.assertTrue(Path(settings.FTP_DIR).exists())
+        ftp_slides_dir = Path(settings.FTP_DIR) / "slides"
+        if not ftp_slides_dir.exists():
+            ftp_slides_dir.mkdir()
 
         self.meeting_slides_dir = Path(settings.AGENDA_PATH) / "42" / "slides"
         if not self.meeting_slides_dir.exists():
@@ -112,7 +116,12 @@ def test_upload_slides(self):
         self.assertEqual(doc.title, "Test File - with fancy title")
         self.assertEqual(doc.get_state_slug(), "active")
 
-        with io.open(os.path.join(self.materials_dir, "slides", doc.name + "-" + doc.rev + ".pdf")) as f:
+        basename=f"{doc.name}-{doc.rev}.pdf"
+        filepath=Path(self.materials_dir) / "slides" / basename
+        with filepath.open() as f:
+            self.assertEqual(f.read(), content)
+        ftp_filepath=Path(settings.FTP_DIR) / "slides" / basename
+        with ftp_filepath.open() as f:
             self.assertEqual(f.read(), content)
 
         # check that posting same name is prevented

ietf/doc/tests_status_change.py

@@ -4,6 +4,7 @@
 
 import io
 import os
+from pathlib import Path
 
 import debug    # pyflakes:ignore
 
@@ -540,7 +541,7 @@ def setUp(self):
         DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review',notify='notify@example.org')
 
 class StatusChangeSubmitTests(TestCase):
-    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['STATUS_CHANGE_PATH']
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['STATUS_CHANGE_PATH', 'FTP_PATH']
     def test_initial_submission(self):
         doc = Document.objects.get(name='status-change-imaginary-mid-review')
         url = urlreverse('ietf.doc.views_status_change.submit',kwargs=dict(name=doc.name))
@@ -556,14 +557,19 @@ def test_initial_submission(self):
         # Right now, nothing to test - we let people put whatever the web browser will let them put into that textbox
 
         # sane post using textbox
-        path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.name, doc.rev))
         self.assertEqual(doc.rev,'00')
-        self.assertFalse(os.path.exists(path))
+        basename = f"{doc.name}-{doc.rev}.txt"
+        filepath = Path(settings.STATUS_CHANGE_PATH) / basename
+        ftp_filepath = Path(settings.FTP_DIR) / "status-changes" / basename
+        self.assertFalse(filepath.exists())
+        self.assertFalse(ftp_filepath.exists())
         r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1"))
         self.assertEqual(r.status_code,302)
         doc = Document.objects.get(name='status-change-imaginary-mid-review')
         self.assertEqual(doc.rev,'00')
-        with io.open(path) as f:
+        with filepath.open() as f:
+            self.assertEqual(f.read(),"Some initial review text\n")
+        with ftp_filepath.open() as f:
             self.assertEqual(f.read(),"Some initial review text\n")
         self.assertTrue( "mid-review-00" in doc.latest_event(NewRevisionDocEvent).desc)
 
@@ -628,3 +634,6 @@ def test_subsequent_submission(self):
     def setUp(self):
         super().setUp()
         DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review',notify='notify@example.org')
+        ftp_subdir=Path(settings.FTP_DIR)/"status-changes"
+        if not ftp_subdir.exists():
+            ftp_subdir.mkdir()

ietf/doc/utils_charter.py

@@ -112,10 +112,10 @@ def fix_charter_revision_after_approval(charter, by):
     )
     try:
         os.link(new, ftp_filepath)
-    except IOError:
+    except IOError as ex:
         log(
-            "There was an error creating a harlink at %s pointing to %s"
-            % (ftp_filepath, new)
+            "There was an error creating a hardlink at %s pointing to %s: %s"
+            % (ftp_filepath, new, ex)
         )
 
     events = []

ietf/doc/views_conflict_review.py

@@ -5,6 +5,7 @@
 import datetime
 import io
 import os
+from pathlib import Path
 
 from django import forms
 from django.shortcuts import render, get_object_or_404, redirect
@@ -181,12 +182,21 @@ def clean_txt(self):
         return get_cleaned_text_file_content(self.cleaned_data["txt"])
 
     def save(self, review):
-        filename = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (review.name, review.rev))
-        with io.open(filename, 'w', encoding='utf-8') as destination:
+        basename = f"{review.name}-{review.rev}.txt"
+        filepath = Path(settings.CONFLICT_REVIEW_PATH) / basename
+        with filepath.open('w', encoding='utf-8') as destination:
             if self.cleaned_data['txt']:
                 destination.write(self.cleaned_data['txt'])
             else:
                 destination.write(self.cleaned_data['content'])
+        ftp_filepath = Path(settings.FTP_DIR) / "conflict-reviews" / basename
+        try:
+            os.link(filepath, ftp_filepath) # Path.hardlink_to is not available until 3.10
+        except IOError as e:
+            log.log(
+                "There was an error creating a hardlink at %s pointing to %s: %s"
+                % (ftp_filepath, filepath, e)
+            )
 
 #This is very close to submit on charter - can we get better reuse?
 @role_required('Area Director','Secretariat')

ietf/doc/views_material.py

@@ -3,8 +3,8 @@
 
 
 # views for managing group materials (slides, ...)
-import io
 import os
+from pathlib import Path
 import re
 
 from django import forms
@@ -162,9 +162,21 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None):
                 f = form.cleaned_data["material"]
                 file_ext = os.path.splitext(f.name)[1]
 
-                with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + file_ext), 'wb+') as dest:
+                basename = f"{doc.name}-{doc.rev}{file_ext}" # Note the lack of a . before file_ext - see os.path.splitext
+                filepath = Path(doc.get_file_path()) / basename
+                with filepath.open('wb+') as dest:
                     for chunk in f.chunks():
                         dest.write(chunk)
+                if not doc.meeting_related():
+                    log.assertion('doc.type_id == "slides"')
+                    ftp_filepath = Path(settings.FTP_DIR) / doc.type_id / basename
+                    try:
+                        os.link(filepath, ftp_filepath) # Path.hardlink is not available until 3.10
+                    except IOError as ex:
+                        log.log(
+                            "There was an error creating a hardlink at %s pointing to %s: %s"
+                            % (ftp_filepath, filepath, ex)
+                        )
 
             if prev_rev != doc.rev:
                 e = NewRevisionDocEvent(type="new_revision", doc=doc, rev=doc.rev)

ietf/doc/views_status_change.py

@@ -5,6 +5,7 @@
 import datetime
 import io
 import os
+from pathlib import Path
 import re
 
 from typing import Dict             # pyflakes:ignore
@@ -33,6 +34,7 @@
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.name.models import DocRelationshipName, StdLevelName
 from ietf.person.models import Person
+from ietf.utils.log import log
 from ietf.utils.mail import send_mail_preformatted
 from ietf.utils.textupload import get_cleaned_text_file_content
 from ietf.utils.timezone import date_today, DEADLINE_TZINFO
@@ -154,12 +156,21 @@ def clean_txt(self):
         return get_cleaned_text_file_content(self.cleaned_data["txt"])
 
     def save(self, doc):
-       filename = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.name, doc.rev))
-       with io.open(filename, 'w', encoding='utf-8') as destination:
-           if self.cleaned_data['txt']:
-               destination.write(self.cleaned_data['txt'])
-           else:
-               destination.write(self.cleaned_data['content'])
+        basename = f"{doc.name}-{doc.rev}.txt"
+        filename = Path(settings.STATUS_CHANGE_PATH) / basename
+        with io.open(filename, 'w', encoding='utf-8') as destination:
+            if self.cleaned_data['txt']:
+                destination.write(self.cleaned_data['txt'])
+            else:
+                destination.write(self.cleaned_data['content'])
+        try:
+            ftp_filename = Path(settings.FTP_DIR) / "status-changes" / basename
+            os.link(filename, ftp_filename) # Path.hardlink is not available until 3.10
+        except IOError as ex:
+            log(
+                "There was an error creating a hardlink at %s pointing to %s: %s"
+                % (ftp_filename, filename, ex)
+            )
 
 #This is very close to submit on charter - can we get better reuse?
 @role_required('Area Director','Secretariat')

ietf/submit/checkers.py

@@ -4,6 +4,7 @@
 
 import io
 import os
+from pathlib import Path
 import re
 import shutil
 import sys
@@ -280,6 +281,15 @@ def check_file_txt(self, path):
 
             dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model)
             shutil.move(path, dest)
+            ftp_dest = Path(settings.FTP_DIR) / "yang" / "draftmod" / model
+            try:
+                os.link(dest, ftp_dest)
+            except IOError as ex:
+                log(
+                    "There was an error creating a hardlink at %s pointing to %s: %s"
+                    % (ftp_dest, dest, ex)
+                )
+
 
             # summary result
             results.append({

ietf/submit/utils.py

@@ -8,6 +8,7 @@
 import os
 import pathlib
 import re
+import subprocess
 import sys
 import time
 import traceback
@@ -1596,3 +1597,6 @@ def active(dirent):
                         modfile.unlink()
         except UnicodeDecodeError as e:
             log.log(f"Error processing {item.name}: {e}")
+
+    ftp_moddir = Path(settings.FTP_DIR) / "yang" / "draftmod"
+    subprocess.call(("/usr/bin/rsync", "-aq", "--delete", moddir, ftp_moddir))