Changed handling of yang modeules from drafts with errors, to not place them in separate directories, due to possible follow-on effects. Added use of IANA-held modules when validating, and regular rsyncing from IANA.

 - Legacy-Id: 14389

Author: levkowetz

bin/daily

@@ -22,6 +22,10 @@ source $DTDIR/env/bin/activate
 # Update our information about the current version of some commands we use
 $DTDIR/ietf/manage.py update_external_command_info
 
+# Get IANA-registered yang models
+YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
+rsync -avz --delete rsync.ietf.org::iana/yang-parameters $YANG_IANA_DIR/
+
 # Populate the yang repositories
 $DTDIR/ietf/manage.py populate_yang_model_dirs
 

ietf/checks.py

@@ -145,7 +145,7 @@ def check_yang_model_directories(app_configs, **kwargs):
         return []
     #
     errors = []
-    for s in ("SUBMIT_YANG_RFC_MODEL_DIR", "SUBMIT_YANG_DRAFT_MODEL_DIR", "SUBMIT_YANG_INVAL_MODEL_DIR", ):
+    for s in ("SUBMIT_YANG_RFC_MODEL_DIR", "SUBMIT_YANG_DRAFT_MODEL_DIR", "SUBMIT_YANG_IANA_MODEL_DIR", ):
         p = getattr(settings, s)
         if not os.path.exists(p):
             errors.append(checks.Critical(

ietf/settings.py

@@ -677,14 +677,14 @@ def skip_unreadable_post(record):
 # they (mainly) belong.  So here, SUBMIT_, rather than IDSUBMIT_
 SUBMIT_YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
 SUBMIT_YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
-SUBMIT_YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/'
+SUBMIT_YANG_IANA_MODEL_DIR = '/a/www/ietf-ftp/yang/ianamod/'
 
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
 IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
 IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
 SUBMIT_PYANG_COMMAND = 'pyang --verbose --ietf -p {libs} {model}'
-SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {tmplib} -p {rfclib} -p {draftlib} -p {invallib} {model} -i'
+SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {tmplib} -p {rfclib} -p {draftlib} -p {ianalib} {model} -i'
 SUBMIT_YANGLINT_COMMAND = None        # use the value above if you have yanglint installed
 
 SUBMIT_YANG_CATALOG_MODULEARG = "modules[]={module}"

ietf/submit/checkers.py

@@ -192,7 +192,7 @@ def check_file_txt(self, path):
                                 workdir,
                                 settings.SUBMIT_YANG_RFC_MODEL_DIR,
                                 settings.SUBMIT_YANG_DRAFT_MODEL_DIR,
-                                settings.SUBMIT_YANG_INVAL_MODEL_DIR,
+                                settings.SUBMIT_YANG_IANA_MODEL_DIR,
                             ])
             if os.path.exists(path):
                 with open(path) as file:
@@ -230,7 +230,7 @@ def check_file_txt(self, path):
                     command = [ w for w in cmd_template.split() if not '=' in w ][0]
                     cmd_version = VersionInfo.objects.get(command=command).version
                     cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir,
-                        draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, invallib=settings.SUBMIT_YANG_INVAL_MODEL_DIR, )
+                        draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, )
                     code, out, err = pipe(cmd)
                     if code > 0 or len(err.strip()) > 0:
                         error_lines = err.splitlines()
@@ -248,14 +248,9 @@ def check_file_txt(self, path):
             else:
                 errors += 1
                 message += "No such file: %s\nPossible mismatch between extracted xym file name and returned module name?\n" % (path)
-                    
 
-            if errors==0 and warnings==0:
-                dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model)
-                shutil.move(path, dest)
-            else:
-                dest = os.path.join(settings.SUBMIT_YANG_INVAL_MODEL_DIR, model)
-                shutil.move(path, dest)
+            dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model)
+            shutil.move(path, dest)
 
             # summary result
             results.append({

ietf/submit/tests.py

@@ -92,24 +92,24 @@ def setUp(self):
         self.yang_draft_model_dir = self.tempdir('yang-draft-model')
         settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.yang_draft_model_dir
 
-        self.saved_yang_inval_model_dir = settings.SUBMIT_YANG_INVAL_MODEL_DIR
-        self.yang_inval_model_dir = self.tempdir('yang-inval-model')
-        settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.yang_inval_model_dir
+        self.saved_yang_iana_model_dir = settings.SUBMIT_YANG_IANA_MODEL_DIR
+        self.yang_iana_model_dir = self.tempdir('yang-iana-model')
+        settings.SUBMIT_YANG_IANA_MODEL_DIR = self.yang_iana_model_dir
 
     def tearDown(self):
         shutil.rmtree(self.staging_dir)
         shutil.rmtree(self.repository_dir)
         shutil.rmtree(self.archive_dir)
         shutil.rmtree(self.yang_rfc_model_dir)
         shutil.rmtree(self.yang_draft_model_dir)
-        shutil.rmtree(self.yang_inval_model_dir)
+        shutil.rmtree(self.yang_iana_model_dir)
         settings.IDSUBMIT_STAGING_PATH = self.saved_idsubmit_staging_path
         settings.INTERNET_DRAFT_PATH = self.saved_internet_draft_path
         settings.IDSUBMIT_REPOSITORY_PATH = self.saved_idsubmit_repository_path
         settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir
         settings.SUBMIT_YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir
         settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir
-        settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir
+        settings.SUBMIT_YANG_IANA_MODEL_DIR = self.saved_yang_iana_model_dir
 
 
     def do_submission(self, name, rev, group=None, formats=["txt",], author=None):

ietf/utils/management/commands/populate_yang_model_dirs.py

@@ -20,8 +20,7 @@ class Command(BaseCommand):
 
     Extracts yang models from RFCs (found in settings.RFC_PATH and places
     them in settings.SUBMIT_YANG_RFC_MODEL_DIR, and from active drafts, placed in
-    settings.SUBMIT_YANG_DRAFT_MODEL_DIR if valid and settings.SUBMIT_YANG_INVAL_MODEL_DIR
-    if not.
+    settings.SUBMIT_YANG_DRAFT_MODEL_DIR.
 
     """
 
@@ -142,14 +141,14 @@ def active(item):
         for item in draftdir.iterdir():
             try:
                 if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
-                    model_list = extract_from(item, moddir)
+                    model_list = extract_from(item, moddir, strict=False)
                     for name in model_list:
                         if not name.startswith('example'):
                             if verbosity > 1:
-                                print("  Extracted valid module from %s: %s" % (item, name))
+                                print("  Extracted module from %s: %s" % (item, name))
                             else:
-                                    sys.stdout.write('.')
-                                    sys.stdout.flush()
+                                sys.stdout.write('.')
+                                sys.stdout.flush()
                         else:
                             modfile = moddir / name
                             modfile.unlink()
@@ -161,39 +160,3 @@ def active(item):
                 sys.stderr.write('\n')
         print("")
 
-        # Extract invalid modules from drafts
-        valdir = moddir
-        moddir = Path(settings.SUBMIT_YANG_INVAL_MODEL_DIR)
-        if not moddir.exists():
-            moddir.mkdir(parents=True)
-        print("Emptying %s ..." % moddir)
-        for item in moddir.iterdir():
-            item.unlink()
-
-        print("Extracting to %s ..." % moddir)
-        for item in draftdir.iterdir():
-            try:
-                if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
-                    model_list = extract_from(item, moddir, strict=False)
-                    for name in model_list:
-                        modfile = moddir / name                    
-                        if (valdir/name).exists():
-                            modfile.unlink()
-                            if verbosity > 1:
-                                print("  Skipped valid module from %s: %s" % (item, name))
-                        elif not name.startswith('example'):
-                            if verbosity > 1:
-                                print("  Extracted invalid module from %s: %s" % (item, name))
-                            else:
-                                    sys.stdout.write('.')
-                                    sys.stdout.flush()
-                        else:
-                            modfile.unlink()
-                            if verbosity > 1:
-                                print("  Skipped module from %s: %s" % (item, name))
-            except UnicodeDecodeError as e:
-                sys.stderr.write('\nError: %s\n' % (e, ))
-                sys.stderr.write(item.name)
-                sys.stderr.write('\n')
-
-        print("")