Merged in changes from trunk up to r17584.

 - Legacy-Id: 17593

Author: levkowetz

.gitignore

@@ -26,6 +26,7 @@
 /attic
 /bin
 /etc
+/env
 /ghostdriver.log
 /htmlcov
 /include
@@ -48,3 +49,5 @@
 /trunk37
 /unix.tag
 /tmp-nomcom-public-keys-dir
+*.pyc
+__pycache__

bin/mergeready

@@ -87,16 +87,17 @@ for opt, value in opts:
     if   opt in ["-h", "--help"]: # Output this help, then exit
         print( __doc__ % locals() )
         sys.exit(1)
-    elif opt in ["-v", "--version"]: # Output version information, then exit
+    elif opt in ["-V", "--version"]: # Output version information, then exit
         print( program, version )
         sys.exit(0)
-    elif opt in ["-V", "--verbose"]: # Output version information, then exit
+    elif opt in ["-v", "--verbose"]: # Output version information, then exit
         opt_verbose += 1
 
 # ----------------------------------------------------------------------
 def say(s):
     sys.stderr.write("%s\n" % (s))
 
+
 # ----------------------------------------------------------------------
 def note(s):
     if opt_verbose:
@@ -204,6 +205,7 @@ for line in pipe('svn propget svn:mergeinfo .').splitlines():
                 write_cache = True
         mergeinfo[line] = merged
     merged_revs.update(merged)
+note('')
 
 if write_cache:
     cache[repo] = mergeinfo
@@ -227,7 +229,7 @@ def get_changeset_list_from_file(repo, filename):
             if line.startswith('#') or line == "":
                 continue
             try:
-                note("   '%s'" % line)
+                #note("   '%s'" % line)
                 parts = line.split()
                 if len(parts) >1 and parts[1] == '@':
                     branch, rev = parts[0], parts[2]
@@ -253,7 +255,7 @@ def get_changeset_list_from_file(repo, filename):
 def get_ready_commits(repo, tree):
     list = []
     note("Getting ready commits from '%s'" % tree)
-    cmd = 'svn log -v -r %s:HEAD %s/%s/' % ((head-500), repo, tree)
+    cmd = 'svn log -v -r %s:HEAD %s/%s/' % ((head-200), repo, tree)
     if opt_verbose > 1:
         note("Running '%s' ..." % cmd)
     commit_log = pipe(cmd)
@@ -273,7 +275,7 @@ def get_ready_commits(repo, tree):
                 note("   %s  %s: %s@%s" % (when.strftime("%Y-%m-%d %H:%MZ"), who, branch, rev))
                 list += [(rev, repo, branch),]
             elif rev in merged_revs and not branch == merged_revs[rev]:
-                sys.stderr.write('Rev %s: %s != %s' % (rev, branch, merged_revs[rev]))                
+                sys.stderr.write('Rev %s: %s != %s\n' % (rev, branch, merged_revs[rev]))                
             else:
                 pass
         else:
@@ -286,10 +288,8 @@ ready += get_changeset_list_from_file(repo, '../ready-for-merge')
 hold   = get_changeset_list_from_file(repo, 'hold-for-merge')
 hold  += get_changeset_list_from_file(repo, '../hold-for-merge')
 ready += get_ready_commits(repo, 'personal')
-ready += get_ready_commits(repo, 'branch/amsl')
 ready += get_ready_commits(repo, 'branch/iola')
 ready += get_ready_commits(repo, 'branch/dash')
-ready += get_ready_commits(repo, 'branch/proceedings')
 
 ready_commits = {}
 all_commits = {}
@@ -328,7 +328,9 @@ for entry in ready:
         else:
             raise
     # 
-    merge_path = os.path.join(*path.split(os.path.sep)[:4])
+    dirs = path.split(os.path.sep)
+    dirs = dirs[:dirs.index('ietf')] if 'ietf' in dirs else dirs[:4]
+    merge_path = os.path.join(*dirs)
     if not (rev, repo, merge_path) in hold:
         output_line = "%s  %-24s ^/%s@%s" % (when.strftime("%Y-%m-%d_%H:%MZ"), who+":", merge_path, rev)
         all_commits[when] = (rev, repo, branch, who, merge_path)
@@ -388,10 +390,11 @@ for key in keys:
 keys = list(not_passed.keys())
 keys.sort()
 if len(keys) > 0:
-    sys.stderr.write("Commits marked ready which haven't passed the test suite:\n")
+    print("")
+    print("Commits marked ready which haven't passed the test suite:\n")
     for key in keys:
-        sys.stderr.write(not_passed[key]+'\n')
-    sys.stderr.write('\n')
+        print(not_passed[key])
+    print('')
 
 keys = list(ready_commits.keys())
 keys.sort()

bin/mkdevbranch

@@ -78,12 +78,13 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)";
 # Option parsing
 
 # Options
-shortopts=hm:M:vV
-longopts=help,meeting=,message=,verbose,version
+shortopts=hm:M:nvV
+longopts=help,meeting=,message=,dry-run,verbose,version
 
 # Default values
 num=""
 msg=""
+do=""
 
 if   [ "$(uname)" = "Linux" ]; then
     args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
@@ -103,6 +104,7 @@ while true ; do
 	-h| --help)	usage; exit;;		# Show this help, then exit
 	-m| --meeting)	num=$2; shift;;		# Specify the IETF meeting number
 	-M| --message)	msg=$2; shift;;		# Specify extra message text
+	-n| --dry-run)  do="echo -- ==>";;	# Only show what would be done
 	-v| --verbose)	VERBOSE=1;;		# Be more talkative
 	-V| --version)	version; exit;;		# Show program version, then exit
 	--)		shift; break;;
@@ -128,8 +130,8 @@ function mksvndir() {
     who=$1
     if [ "$2" ]; then dir=$2; else dir=$who; fi
     if ! svn info https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$dir >/dev/null 2>&1 ; then
-	echo "Creating personal directory area for IETF datatracker coding: /personal/$dir"
-	svn mkdir https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$dir -m "Personal SVN dir for $who, for IETF datatracker code"
+	$do echo "Creating personal directory area for IETF datatracker coding: /personal/$dir"
+	$do svn mkdir https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$dir -m "Personal SVN dir for $who, for IETF datatracker code"
     else
 	echo "Repository area personal/$dir is already in place."
     fi
@@ -143,7 +145,7 @@ cd $progdir
 
 if [ "$who" ]; then
     mksvndir $who
-    svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$who/$target/ -m "New branch for $target"
+    $do svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$who/$target/ -m "New branch for $target"
     echo "New branch: ^/personal/$who/$target"
 else
     [ "$msg" ] && msg="
@@ -154,11 +156,21 @@ $msg
 	trac-admin /www/tools.ietf.org/tools/ietfdb wiki export IETF${n}SprintSignUp	\
 	    | egrep "^\|\|" | tail -n +2 | python -c '
 import sys, re
-afile = open("aliases")
-aliases = dict([ line.strip().split(None,1) for line in afile.read().splitlines() ])
+with open("aliases") as afile:
+    try:
+	aliases = dict([ line.strip().split(None,1) for line in afile.read().splitlines() if line.strip() ])
+    except ValueError:
+	sys.stderr.write([ line.strip().split(None,1) for line in afile.read().splitlines() if line.strip() ])
+	raise
 
 for line in sys.stdin:
-    blank, name, email, rest = line.strip().split("||", 3)
+    try:
+	blank, name, email, rest = line.strip().split("||", 3)
+	email = email.strip()
+    except ValueError:
+	sys.stderr.write(line+"\n")
+	raise
+	
     login, dummy  = re.split("[@.]", email, 1)
     if email in aliases:
 	login = aliases[email]
@@ -171,9 +183,9 @@ for line in sys.stdin:
 	echo "$login ($name <$email>):"
 	mksvndir $login
 	if ! svn info https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target >/dev/null 2>&1 ; then
-	    echo "  creating $target branch for $login ($name)."
-	    svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target/ -m "New IETF datatracker coding branch for $name"	\
-	    && mail "$name <$email>" -s "A new SVN branch for you for IETF datatracker coding${rev:+, based on $rev}." -b henrik@levkowetz.com <<-EOF
+	    $do echo "  creating $target branch for $login ($name)."
+	    $do svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target/ -m "New IETF datatracker coding branch for $name"	\
+	    && $do mail "$name <$email>" -s "A new SVN branch for you for IETF datatracker coding${rev:+, based on $rev}." -b henrik@levkowetz.com <<-EOF
 		Hi,
                 $msg
 		This mail has been automatically generated by the $program script.
@@ -199,7 +211,7 @@ for line in sys.stdin:
 
 	EOF
 	else
-	    echo "  branch personal/$login/$target already exists."
+	    $do echo "  branch personal/$login/$target already exists."
 	fi
     done
 fi

bin/mkpatch

@@ -71,8 +71,8 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)";
 # Option parsing
 
 # Options
-shortopts=c:r:hvV
-longopts=change=,revision=,help,verbose,version
+shortopts=c:or:hvV
+longopts=change=,overwrite,revision=,help,verbose,version
 
 # Default values
 
@@ -92,7 +92,7 @@ fi
 while true ; do
     case "$1" in
 	-c| --change)	CHG="$2"; shift;;	# the change made by revision ARG
-	-r| --revision)	REV="$2"; shift;;	# the change made between revisions REV
+        -o| --overwrite) OVER=1;;                # overwrite any existing patch file
 	-h| --help)	usage; exit;;		# Show this help, then exit
 	-v| --verbose)	VERBOSE=1;;	# Be more talkative
 	-V| --version)	version; exit;;		# Show program version, then exit
@@ -105,16 +105,19 @@ done
 # ----------------------------------------------------------------------
 # The program itself
 
-if [ $# -lt 2 ]; then die "Expected patch name and file list on the command line."; fi
-if [[ $1 =~ / ]]; then die "Expected a patch name, but the first argument to $program seems to be a file path: '$1'"; fi
-
-name=$1; shift;
-if [ "$CHG" ]; then name="$name-c$CHG"; fi
+if   [ "$CHG" ]; then
+  name=$(echo $(svn log -c $CHG | sed -r -e '/^---/d' -e '/^r[0-9]+/d' -e '/^$/d' -e 's/Merged in \[[0-9]+\] from [^:]+..//' ) | sed -r -e 's/(.*)/\L\1/' -e 's/[^[:alnum:]]/-/g' -e 's/-+/-/g' | cut -c 1-40)
+  name="$name-c$CHG"
+else
+  if [ $# -lt 2 ]; then die "Expected patch name and file list on the command line."; fi
+  if [[ $1 =~ / ]]; then die "Expected a patch name, but the first argument to $program seems to be a file path: '$1'"; fi
+  name=$1; shift;
+fi
 
 patchfile=$progdir/../../patches/$(date +%Y-%m-%d)-$name.patch
-if [ -e $patchfile ]; then die "Patchfile $patchfile already exists"; fi
+if [ -e $patchfile -a ! -n "$OVER" ]; then die "Patchfile $patchfile already exists"; fi
 svn diff ${CHG:+ -c $CHG} ${REV:+ -r $REV} "$@" > $patchfile
 less $patchfile
 echo ""
 echo ""
-echo "Patch is in $patchfile."
+echo "Patch is in $patchfile"

bin/mkrelease

@@ -66,6 +66,10 @@ function note() {
 	if [ -n "$VERBOSE" ]; then echo -e "\n$*"; fi
 }
 
+function check() {
+  [ "$(which $1)" ] || die "could not find the '$1' command.  $2"
+}
+
 # ----------------------------------------------------------------------
 function version() {
 	echo -e "$program $version"
@@ -119,6 +123,11 @@ while true ; do
     shift
 done
 
+# ----------------------------------------------------------------------
+# Check some requirements
+
+check bower "It is required to update web resources.  Install with npm."
+
 # ----------------------------------------------------------------------
 # The program itself
 
@@ -212,6 +221,7 @@ if [ -z "$IGNORE_RESOURCES" ]; then
     $do svn commit ietf/externals/static -m "Updated bower-managed static web assets"
     # Get rid of bower-installed files which we don't use:
     $do rm -rf ietf/externals/static/datatracker/
+    $do rm -rf ietf/externals/static/jquery.cookie/
     $do rm -f $(svn st ietf/externals/ | grep '^\?' | awk '{print $2}')
 fi
 
@@ -244,7 +254,9 @@ if [ -d ../coverage ]; then
     rsync -a static/coverage/ ../coverage/$VER/
 fi
 
-contributors=$(echo "$changes" | sed 's/\.[ \t\n]/ /'| tr -c "a-z0-9.@-" "\n" | sort | uniq | grep '@' | sed -r -e 's/^\.+//' -e 's/\.+$//' -e 's/^/-c /' || true)
+contributors=$(echo "$changes" | gawk '/^  \* Merged in \[[0-9]+\] from [^: ]+/ {sub(":",""); print $6;}' | sort | uniq)
+note "Contributors:
+$contributors"
 
 note "Setting the current time on the release notes in the changelog file ..."
 $do sed -r -i -e "1,/^ -- /s/([A-Za-z-]+ <[a-z0-9.-]+@[a-z0-9.-]+>  ).*$/\1$(TZ=UTC date +'%d %b %Y %H:%M:%S %z')/" changelog