Added a new yang checker, 'yanglint', to the existing Yang checker class, in

addition to the existing 'pyang' checker.

Added modal overlay displays showing the yang check results every place the
yin/yang symbol is shown (red or green) to indicate the presencee and result
of yang checks.  Added a Yang Validation: line in the document
meta-information section on the document's page in the datatracker.

Added the result of the xym extaction to the yang check results, to make
extration failures visible.

Added the version of the used xym, pyang, and yanglint commands to the check
results.

Added an action to move successfully extracted and validated modules to the
module library directories immediately on submission.

Added the xym and pyang repositories as svn:external components, rather than
listing them in requirements.txt, as there has been delays of many months
between essential features in the repositories, and an actual release.  We may
get occasional buildbot failures if broken code is pulled in from the
repository, but better that than the functionality failure of severely
outdated componets.

Added a new management command to re-run yang validation for active drafts for
which yang modules were found at submission time, in order to pick up imported
models which may have arrived in the model libraries after the draft's
submission.  Run daily from bin/daily.

Added a table to hold version information for external commands.  The yang
checker output should include the version information of the used checkers,
but seems unnecessary to run each command with its --version switch every
time we check a module...

Added a new management command to collect version information for external
commands on demand.  To be run daily from bin/daily.

Added tests to verify that xym, pyang and yanglint information is available
on the submission confirmation page, and updated the yang module contained in
the test document to validate under both pyang and yanglint.

Updated admin.py and resource.py files as needed.
 - Legacy-Id: 13630

Author: levkowetz

.gitignore

@@ -39,3 +39,4 @@
 /static
 /testresult
 /unix.tag
+/tmp-nomcom-public-keys-dir

bin/daily

@@ -18,9 +18,16 @@ logger -p user.info -t cron "Running $DTDIR/bin/daily"
 # Set up the virtual environment
 source $DTDIR/env/bin/activate
 
+
+# Update our information about the current version of some commands we use
+$DTDIR/ietf/manage.py update_external_command_info
+
 # Populate the yang repositories
 $DTDIR/ietf/manage.py populate_yang_model_dirs
 
+# Re-run yang checks on active documents
+$DTDIR/ietf/manage.py run_yang_model_checks
+
 # Expire internet drafts
 # Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner:
 $DTDIR/ietf/bin/expire-ids
@@ -36,5 +43,3 @@ $DTDIR/ietf/bin/rfc-editor-index-updates -d 1969-01-01
 
 # Fetch meeting attendance data from ietf.org/registration/attendees
 $DTDIR/ietf/manage.py fetch_meeting_attendance --latest
-
-

ietf/doc/templatetags/ietf_filters.py

@@ -533,3 +533,6 @@ def comma_separated_list(seq, end_word="and"):
 def role_names(roles):
     return list(set([ "%s %s" % (r.group.name, r.name.name) for r in roles ]))
 
+@register.filter()
+def zaptmp(s):
+    return re.sub(r'/tmp/tmp[^/]+/', '', s)

ietf/doc/views_doc.py

@@ -1277,3 +1277,11 @@ def all_presentations(request, name):
         'in_progress': in_progress,
         'past' : past,
         })
+
+def document_checks(request, name):
+    doc = get_object_or_404(Document, docalias__name=name, type_id='draft')
+
+    checks = doc.submission.latest_checks()
+    debug.show('checks')
+
+    return render(request, 'doc/document_checks.html', {'doc': doc, 'checks': checks})

ietf/name/fixtures/names.json

@@ -9708,5 +9708,38 @@
     },
     "model": "name.topicaudiencename",
     "pk": "nominees"
+  },
+  {
+    "fields": {
+      "command": "xym",
+      "switch": "--version",
+      "time": "2017-06-15T06:24:58.869",
+      "used": true,
+      "version": "xym 0.3.2"
+    },
+    "model": "utils.versioninfo",
+    "pk": 1
+  },
+  {
+    "fields": {
+      "command": "pyang",
+      "switch": "--version",
+      "time": "2017-06-15T06:24:59.516",
+      "used": true,
+      "version": "pyang 1.7.2"
+    },
+    "model": "utils.versioninfo",
+    "pk": 2
+  },
+  {
+    "fields": {
+      "command": "yanglint",
+      "switch": "--version",
+      "time": "2017-06-15T06:24:59.531",
+      "used": true,
+      "version": "yanglint 0.12.183"
+    },
+    "model": "utils.versioninfo",
+    "pk": 3
   }
 ]

ietf/name/generate_fixtures.py

@@ -47,5 +47,8 @@ def output(name, seq):
 objects += ietf.mailtrigger.models.Recipient.objects.all()
 objects += ietf.mailtrigger.models.MailTrigger.objects.all()
 
+import ietf.utils.models
+objects += ietf.utils.models.VersionInfo.objects.all()
+
 output("names", objects)
 

ietf/settings.py

@@ -660,11 +660,19 @@ def skip_unreadable_post(record):
 IDSUBMIT_DEFAULT_CUTOFF_TIME_UTC = datetime.timedelta(hours=23, minutes=59, seconds=59)
 IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21)
 
+# 14 Jun 2017: New convention: prefix settings with the app name to which
+# they (mainly) belong.  So here, SUBMIT_, rather than IDSUBMIT_
+SUBMIT_YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
+SUBMIT_YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
+SUBMIT_YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/'
+
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
 IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
 IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
-IDSUBMIT_PYANG_COMMAND = 'pyang -p %(modpath)s --verbose --ietf  %(model)s'
+SUBMIT_PYANG_COMMAND = 'pyang --verbose --ietf -p {libs} {model}'
+SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {rfclib} -p {draftlib} {model}'
+SUBMIT_YANGLINT_COMMAND = None        # use the value above if you have yanglint installed
 
 IDSUBMIT_CHECKER_CLASSES = (
     "ietf.submit.checkers.DraftIdnitsChecker",
@@ -696,10 +704,6 @@ def skip_unreadable_post(record):
 IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000
 IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB
 
-YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
-YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
-YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/'
-
 XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/"
 
 # === Meeting Related Settings =================================================

ietf/static/ietf/css/ietf.css

@@ -865,3 +865,9 @@ blockquote {
 #debug-query-table .code .current {
   background-color: #ddd;
 }
+
+.checker-warning,
+.checker-success {
+  line-height: 1.0;
+  cursor: pointer;
+}

ietf/submit/admin.py

@@ -33,7 +33,7 @@ def rev(self, instance):
 admin.site.register(SubmissionEvent, SubmissionEventAdmin)
 
 class SubmissionCheckAdmin(admin.ModelAdmin):
-    list_display = ['submission', 'time', 'checker', 'passed', 'errors', 'warnings', 'items']
+    list_display = ['submission', 'time', 'checker', 'passed', 'errors', 'warnings', 'message']
     raw_id_fields = ['submission']
     search_fields = ['submission__name']
 admin.site.register(SubmissionCheck, SubmissionCheckAdmin)

ietf/submit/checkers.py

@@ -2,16 +2,19 @@
 
 import os
 import re
+import sys
 from xym import xym
 import shutil
 import tempfile
+import StringIO
 
 from django.conf import settings
 
 import debug                            # pyflakes:ignore
 
-from ietf.utils.pipe import pipe
 from ietf.utils.log import log
+from ietf.utils.models import VersionInfo
+from ietf.utils.pipe import pipe
 
 class DraftSubmissionChecker():
     name = ""
@@ -47,6 +50,9 @@ class DraftIdnitsChecker(object):
     # start using this when we provide more in the way of warnings during
     # submission checking:
     # symbol = '<span class="fa fa-check-square"></span>'
+    # symbol = u'<span class="large">\ua17d</span>' # Yi syllable 'nit'
+    # symbol = u'<span class="large">\ub2e1</span>' # Hangul syllable 'nit'
+
     symbol = ""
 
     def __init__(self, options=["--submitcheck", "--nitcount", ]):
@@ -123,39 +129,66 @@ class DraftYangChecker(object):
     def check_file_txt(self, path):
         name = os.path.basename(path)
         workdir = tempfile.mkdtemp()
-        errors = []
-        warnings = []
-        results = {}
+        errors = 0
+        warnings = 0
+        message = ""
+        results = []
+        passed = True                   # Used by the submission tool.  Yang checks always pass.
 
-        extractor = xym.YangModuleExtractor(path, workdir, strict=True, debug_level = 0)
+        extractor = xym.YangModuleExtractor(path, workdir, strict=True, strict_examples=False, debug_level=0)
         if not os.path.exists(path):
             return None, "%s: No such file or directory: '%s'"%(name.capitalize(), path), errors, warnings, results
         with open(path) as file:
+            out = ""
+            err = ""
+            code = 0
             try:
                 # This places the yang models as files in workdir
+                saved_stdout = sys.stdout
+                saved_stderr = sys.stderr
+                sys.stdout = StringIO.StringIO()
+                sys.stderr = StringIO.StringIO()
                 extractor.extract_yang_model(file.readlines())
+                out = sys.stdout.getvalue()
+                err = sys.stderr.getvalue()
+                sys.stdout = saved_stdout
+                sys.stderr = saved_stderr
                 model_list = extractor.get_extracted_models()
             except Exception as exc:
-                passed  = False
-                message = exc
-                errors  = [ (name, None, None, exc) ]
-                warnings = []
-                return passed, message, errors, warnings
+                code = 1
+                err = '\n'.join( [ m for m in [out, err, exc] if m ] )
+        if err:
+            code += 1
+        command = "xym"
+        cmd_version = VersionInfo.objects.get(command=command).version
+        message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err)
+
+        results.append({
+            "name": name,
+            "passed":  passed,
+            "message": message,
+            "warnings": 0,
+            "errors":  code,
+            "items": [],
+        })
 
         for model in model_list:
             path = os.path.join(workdir, model)
+            message = ""
             modpath = ':'.join([
                                 workdir,
-                                settings.YANG_RFC_MODEL_DIR,
-                                settings.YANG_DRAFT_MODEL_DIR,
-                                settings.YANG_INVAL_MODEL_DIR,
+                                settings.SUBMIT_YANG_RFC_MODEL_DIR,
+                                settings.SUBMIT_YANG_DRAFT_MODEL_DIR,
+                                settings.SUBMIT_YANG_INVAL_MODEL_DIR,
                             ])
             with open(path) as file:
                 text = file.readlines()
-            cmd = settings.IDSUBMIT_PYANG_COMMAND % {"modpath": modpath, "model": path, }
+            # pyang
+            cmd_template = settings.SUBMIT_PYANG_COMMAND
+            command = cmd_template.split()[0]
+            cmd_version = VersionInfo.objects.get(command=command).version
+            cmd = cmd_template.format(libs=modpath, model=path)
             code, out, err = pipe(cmd)
-            errors = 0
-            warnings = 0
             items = []
             if code > 0:
                 error_lines = err.splitlines()
@@ -175,26 +208,54 @@ def check_file_txt(self, path):
                                 warnings += 1
                         except ValueError:
                             pass
-            results[model] = {
-                "passed":  code == 0,
-                "message": out+"No validation errors\n" if code == 0 else err,
+            #passed = passed and code == 0 # For the submission tool.  Yang checks always pass
+            message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if code == 0 else err)
+
+            # yanglint
+            cmd_template = settings.SUBMIT_YANGLINT_COMMAND
+            command = cmd_template.split()[0]
+            cmd_version = VersionInfo.objects.get(command=command).version
+            cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
+            code, out, err = pipe(cmd)
+            if code > 0:
+                error_lines = err.splitlines()
+                for line in error_lines:
+                    if line.strip():
+                        try:
+                            if 'err : ' in line:
+                                errors += 1
+                            if 'warn: ' in line:
+                                warnings += 1
+                        except ValueError:
+                            pass
+            #passed = passed and code == 0 # For the submission tool.  Yang checks always pass
+            message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if code == 0 else err)
+
+            if errors==0 and warnings==0:
+                dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model)
+                shutil.move(path, dest)
+            else:
+                dest = os.path.join(settings.SUBMIT_YANG_INVAL_MODEL_DIR, model)
+                shutil.move(path, dest)
+
+            # summary result
+            results.append({
+                "name": model,
+                "passed":  passed,
+                "message": message,
                 "warnings": warnings,
                 "errors":  errors,
                 "items": items,
-            }
+            })
+
 
         shutil.rmtree(workdir)
 
-        ## For now, never fail because of failed yang validation.
-        if len(model_list):
-            passed = True
-        else:
-            passed = None
-        #passed  = all( res["passed"] for res in results.values() )
-        message = "\n\n".join([ "\n".join([model+':', res["message"]]) for model, res in results.items() ])
-        errors  = sum(res["errors"] for res in results.values() )
-        warnings  = sum(res["warnings"] for res in results.values() )
-        items  = [ e for res in results.values() for e in res["items"] ]
+        passed  = all( res["passed"] for res in results )
+        message = "\n".join([ "\n".join([res['name']+':', res["message"]]) for res in results ])
+        errors  = sum(res["errors"] for res in results )
+        warnings  = sum(res["warnings"] for res in results )
+        items  = [ e for res in results for e in res["items"] ]
 
         return passed, message, errors, warnings, items