Added support for pulling yang modules from yangcatalog.org.

 - Legacy-Id: 18141

Author: levkowetz

bin/daily

@@ -32,6 +32,9 @@ $DTDIR/ietf/manage.py update_external_command_info
 # Hardcode the rsync target to avoid any unwanted deletes:
 rsync -avzq --delete rsync.ietf.org::iana/yang-parameters/ /a/www/ietf-ftp/yang/ianamod/
 
+# Get Yang models from Yangcatalog.
+rsync -avzq rsync://yangcatalog.org:10873/yangdeps /a/www/ietf-ftp/yang/catalogmod/
+
 # Populate the yang repositories
 $DTDIR/ietf/manage.py populate_yang_model_dirs -v0
 

ietf/checks.py

@@ -152,7 +152,7 @@ def check_yang_model_directories(app_configs, **kwargs):
         return []
     #
     errors = []
-    for s in ("SUBMIT_YANG_RFC_MODEL_DIR", "SUBMIT_YANG_DRAFT_MODEL_DIR", "SUBMIT_YANG_IANA_MODEL_DIR", ):
+    for s in ("SUBMIT_YANG_RFC_MODEL_DIR", "SUBMIT_YANG_DRAFT_MODEL_DIR", "SUBMIT_YANG_IANA_MODEL_DIR", "SUBMIT_YANG_CATALOG_MODEL_DIR",):
         p = getattr(settings, s)
         if not os.path.exists(p):
             errors.append(checks.Critical(

ietf/settings.py

@@ -819,13 +819,14 @@ def skip_unreadable_post(record):
 SUBMIT_YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
 SUBMIT_YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
 SUBMIT_YANG_IANA_MODEL_DIR = '/a/www/ietf-ftp/yang/ianamod/'
+SUBMIT_YANG_CATALOG_MODEL_DIR = '/a/www/ietf-ftp/yang/catalogmod/'
 
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
 IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
 IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
 SUBMIT_PYANG_COMMAND = 'pyang --verbose --ietf -p {libs} {model}'
-SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {tmplib} -p {rfclib} -p {draftlib} -p {ianalib} {model} -i'
+SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {tmplib} -p {rfclib} -p {draftlib} -p {ianalib} -p {cataloglib} {model} -i'
 SUBMIT_YANGLINT_COMMAND = ''        # use the value above if you have yanglint installed
 
 SUBMIT_YANG_CATALOG_MODULEARG = "modules[]={module}"

ietf/submit/checkers.py

@@ -201,6 +201,7 @@ def check_file_txt(self, path):
                                 settings.SUBMIT_YANG_RFC_MODEL_DIR,
                                 settings.SUBMIT_YANG_DRAFT_MODEL_DIR,
                                 settings.SUBMIT_YANG_IANA_MODEL_DIR,
+                                settings.SUBMIT_YANG_CATALOG_MODEL_DIR,
                             ])
             if os.path.exists(path):
                 with io.open(path) as file:
@@ -242,7 +243,8 @@ def check_file_txt(self, path):
                     command = [ w for w in cmd_template.split() if not '=' in w ][0]
                     cmd_version = VersionInfo.objects.get(command=command).version
                     cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir,
-                        draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, )
+                        draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR,
+                        cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, )
                     code, out, err = pipe(cmd)
                     out = out.decode('utf-8')
                     err = err.decode('utf-8')

ietf/submit/tests.py

@@ -103,6 +103,10 @@ def setUp(self):
         self.yang_iana_model_dir = self.tempdir('yang-iana-model')
         settings.SUBMIT_YANG_IANA_MODEL_DIR = self.yang_iana_model_dir
 
+        self.saved_yang_catalog_model_dir = settings.SUBMIT_YANG_CATALOG_MODEL_DIR
+        self.yang_catalog_model_dir = self.tempdir('yang-catalog-model')
+        settings.SUBMIT_YANG_CATALOG_MODEL_DIR = self.yang_catalog_model_dir
+
         # Submit views assume there is a "next" IETF to look for cutoff dates against
         MeetingFactory(type_id='ietf', date=datetime.date.today()+datetime.timedelta(days=180))
 
@@ -113,13 +117,15 @@ def tearDown(self):
         shutil.rmtree(self.yang_rfc_model_dir)
         shutil.rmtree(self.yang_draft_model_dir)
         shutil.rmtree(self.yang_iana_model_dir)
+        shutil.rmtree(self.yang_catalog_model_dir)
         settings.IDSUBMIT_STAGING_PATH = self.saved_idsubmit_staging_path
         settings.INTERNET_DRAFT_PATH = self.saved_internet_draft_path
         settings.IDSUBMIT_REPOSITORY_PATH = self.saved_idsubmit_repository_path
         settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir
         settings.SUBMIT_YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir
         settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir
         settings.SUBMIT_YANG_IANA_MODEL_DIR = self.saved_yang_iana_model_dir
+        settings.SUBMIT_YANG_CATALOG_MODEL_DIR = self.saved_yang_catalog_model_dir
 
 
     def do_submission(self, name, rev, group=None, formats=["txt",], author=None):