Merged in ^/personal/henrik/6.98.2-py3@16468, containing code converted to work with both Python-2.7 and Python-3.7.

 - Legacy-Id: 16470

Author: levkowetz

bin/check-copyright

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# -*- python -*-
+#!/usr/bin/env python3.7
+# -*- mode: python; coding: utf-8 -*-
 # Copyright The IETF Trust 2019, All Rights Reserved
 """
 NAME
@@ -10,11 +10,17 @@ SYNOPSIS
 
 DESCRIPTION
 	Given a list of files or filename wildcard patterns, check all for
-	an IETF Trust copyright notice with the current year.
+	an IETF Trust copyright notice with the current year.  Optionally
+        generate a diff on standard out which can be used by 'patch'.
 
-%(options)s
+        An invocation similar to the following can be particularly useful with
+        a set of changed version-controlled files, as it will fix up the
+        Copyright statements of any python files with pending changes:
+
+        $ check-copyright -p $(svn st | cut -c 9- | grep '\.py$' ) | patch -p0
 
-FILES
+
+%(options)s
 
 AUTHOR
 	Written by Henrik Levkowetz, <henrik@tools.ietf.org>
@@ -43,10 +49,12 @@ import pytz
 import tzparse
 import debug
 
-version = "0.10"
+version = "1.0.0"
 program = os.path.basename(sys.argv[0])
 progdir = os.path.dirname(sys.argv[0])
 
+debug.debug = True
+
 # ----------------------------------------------------------------------
 # Parse options
 
@@ -63,8 +71,8 @@ if len(sys.argv) < 1:
     sys.exit(1)
 
 try:
-    opts, files = getopt.gnu_getopt(sys.argv[1:], "hvV", ["help", "version", "verbose",])
-except Exception, e:
+    opts, files = getopt.gnu_getopt(sys.argv[1:], "hC:pvV", ["help", "copyright=", "patch", "version", "verbose",])
+except Exception as e:
     print( "%s: %s" % (program, e))
     sys.exit(1)
 
@@ -73,16 +81,22 @@ except Exception, e:
 
 # set default values, if any
 opt_verbose = 0
+opt_patch = False
+opt_copyright = "Copyright The IETF Trust {years}, All Rights Reserved"
 
 # handle individual options
 for opt, value in opts:
     if   opt in ["-h", "--help"]: # Output this help, then exit
         print( __doc__ % locals() )
         sys.exit(1)
+    elif opt in ["-p", "--patch"]: # Generate patch output rather than error messages
+        opt_patch = True
+    elif opt in ["-C", "--copyright"]: # Copyright line pattern using {years} for years
+        opt_copyright = value
     elif opt in ["-V", "--version"]: # Output version information, then exit
         print( program, version )
         sys.exit(0)
-    elif opt in ["-v", "--verbose"]: # Output version information, then exit
+    elif opt in ["-v", "--verbose"]: # Be more verbose
         opt_verbose += 1
 
 # ----------------------------------------------------------------------
@@ -107,7 +121,7 @@ def pipe(cmd, inp=None):
     args = shlex.split(cmd)
     bufsize = 4096
     stdin = PIPE if inp else None
-    pipe = Popen(args, stdin=stdin, stdout=PIPE, stderr=PIPE, bufsize=bufsize)
+    pipe = Popen(args, stdin=stdin, stdout=PIPE, stderr=PIPE, bufsize=bufsize, encoding='utf-8', universal_newlines=True)
     out, err = pipe.communicate(inp)
     code = pipe.returncode
     if code != 0:
@@ -156,9 +170,6 @@ import json
 
 cwd = os.getcwd()
 
-if cwd.split(os.path.sep)[-1] != 'trunk':
-    die("Expected to run this operation in trunk, but the current\ndirectory is '%s'" % cwd)
-
 # Get current initinfo from cache and svn
 cachefn = os.path.join(os.environ.get('HOME', '.'), '.initinfo')
 
@@ -177,24 +188,67 @@ write_cache = False
 loginfo_format = r'^r[0-9]+ \| [^@]+@[^@]+ \| \d\d\d\d-\d\d-\d\d '
 
 year = time.strftime('%Y')
+copyright_re = "(?i)"+opt_copyright.format(years=r"(\d+-)?\d+")
 for path in files:
-    note("Checking path %s" % path)
-    if not path in initinfo:
-        initinfo.update(get_first_commit(path))
-        write_cache = True
-    date = initinfo[path]['date']
-    init = date[:4]
-    copyright = "(?i)Copyright The IETF Trust (%s-)?%s, All Rights Reserved" % (init, year)
-    with open(path) as file:
-        chunk = file.read(4000)
-        if os.path.basename(path) == '__init__.py' and len(chunk)==0:
+    try:
+        if not os.path.exists(path):
+            note("File does not exist: %s" % path)
             continue
-        if not re.search(copyright, chunk):
-            sys.stdout.write("%s(1): Error: Missing or bad copyright.  " % path)
-            if year == init:
-                print("  Expected: Copyright The IETF Trust %s, All Rights Reserved" % year)
-            else:
-                print("  Expected: Copyright The IETF Trust %s-%s, All Rights Reserved" % (init, year))
+        note("Checking path %s" % path)
+        if not path in initinfo:
+            initinfo.update(get_first_commit(path))
+            write_cache = True
+        date = initinfo[path]['date']
+        init = date[:4]
+        
+        copyright_year_re = "(?i)"+opt_copyright.format(years=r"({init}-)?{year}".format(init=init, year=year))
+        with open(path) as file:
+            try:
+                chunk = file.read(4000)
+            except UnicodeDecodeError as e:
+                sys.stderr.write(f'Error when reading {file.name}: {e}\n')
+                raise
+            if os.path.basename(path) == '__init__.py' and len(chunk)==0:
+                continue
+            if not re.search(copyright_year_re, chunk):
+                if year == init:
+                    copyright = opt_copyright.format(years=year)
+                else:
+                    copyright = opt_copyright.format(years=f"{init}-{year}")
+                if opt_patch:
+                    print(f"--- {file.name}\t(original)")
+                    print(f"+++ {file.name}\t(modified)")
+                    if not re.search(copyright_re, chunk):
+                        # Simple case, just insert copyright at the top
+                        print( "@@ -1,3 +1,4 @@")
+                        print(f"+# {copyright}")
+                        for i, line in list(enumerate(chunk.splitlines()))[:3]:
+                            print(f" {line}")
+                    else:
+                        # Find old copyright, then emit preceding lines,
+                        # change, and following lines.
+                        pos = None
+                        for i, line in enumerate(chunk.splitlines(), start=1):
+                            if re.search(copyright_re, line):
+                                pos = i
+                                break
+                        if not pos:
+                            raise RuntimeError("Unexpected state: Expected a copyright line, but found none")
+                        print(f"@@ -1,{pos+3} +1,{pos+3} @@")
+                        for i, line in list(enumerate(chunk.splitlines(), start=1))[:pos+3]:
+                            if i == pos:
+                                print(f"-{line}")
+                                print(f"+# {copyright}")
+                            else:
+                                print(f" {line}")
+                else:
+                    sys.stderr.write(f"{path}(1): Error: Missing or bad copyright.  Expected: {copyright}")
+    except Exception:
+        if write_cache:
+            cache = initinfo
+            with open(cachefn, "w") as file:
+                json.dump(cache, file, indent=2, sort_keys=True)
+        raise
 
 if write_cache:
     cache = initinfo

bin/test-crawl

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# Copyright The IETF Trust 2013-2019, All Rights Reserved
 
 import os, sys, re, datetime, argparse, traceback, json, subprocess
 import html5lib
@@ -62,6 +63,7 @@ import debug    # pyflakes:ignore
 
 from ietf.name.models import DocTypeName
 from ietf.utils.html import unescape
+from ietf.utils.test_utils import unicontent
 
 # --- Constants ---
 
@@ -387,7 +389,7 @@ if __name__ == "__main__":
                 if ctype == "text/html":
                     try:
                         if args.follow and not skip_extract_from(url):
-                            for u in extract_html_urls(r.content):
+                            for u in extract_html_urls(unicontent(r)):
                                 if u not in visited and u not in urls:
                                     urls[u] = url
                                     referrers[u] = url
@@ -403,7 +405,7 @@ if __name__ == "__main__":
                 elif ctype == "application/json":
                     try:
                         if args.follow:
-                            for u in extract_tastypie_urls(r.content):
+                            for u in extract_tastypie_urls(unicontent(r)):
                                 if u not in visited and u not in urls:
                                     urls[u] = url
                                     referrers[u] = url

changelog.py

@@ -1,4 +1,9 @@
+# Copyright The IETF Trust 2012-2019, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, print_function, unicode_literals
 import re
+import six
 from tzparse import tzparse
 from datetime import datetime as Datetime
 
@@ -38,12 +43,12 @@ class ChangeLogEntry:
     title = ""
 
 def parse(logfile):
-    ver_line = "^(\w+) \((\S+)\) (\S+;)? (?:urgency=(\S+))?$"
-    sig_line = "^ -- ([^<]+) <([^>]+)>  (.*?) *$"
+    ver_line = r"^(\w+) \((\S+)\) (\S+;)? (?:urgency=(\S+))?$"
+    sig_line = r"^ -- ([^<]+) <([^>]+)>  (.*?) *$"
     inf_line = r"^  \*\*(.*)\*\* *"
 
     entries = []
-    if type(logfile) == type(''):
+    if isinstance(logfile, six.string_types):
         logfile = open(logfile)
     entry = None
     for line in logfile:
@@ -66,5 +71,5 @@ def parse(logfile):
         elif entry:
             entry.logentry += line
         else:
-            print "Unexpected line: '%s'" % line
+            print("Unexpected line: '%s'" % line)
     return entries

debug.py

@@ -1,3 +1,4 @@
+# Copyright The IETF Trust 2012-2019, All Rights Reserved
 import os
 import sys
 import time as timeutils

docker/install-extras

@@ -5,3 +5,4 @@ su - -c "apt-get update \
 	&& apt-get install -qy graphviz ghostscript apache2-utils \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/*"
+