Merged trunk@9354 to facelift copy

 - Legacy-Id: 9357

Author: levkowetz

.gitignore

@@ -18,6 +18,7 @@
 /.project
 /.pydevproject
 /.settings
+/.coverage
 /unix.tag
 /testresult
 /mergelog
@@ -28,3 +29,4 @@
 /local
 /lib
 /include
+/latest-coverage.json

bin/.gitignore

@@ -11,3 +11,6 @@
 /activate_this.py
 /pyflakes
 /pip2.7
+/coverage
+/coverage2
+/coverage-2.7

bin/mergeready

@@ -247,8 +247,10 @@ ready += get_ready_commits(repo, 'personal')
 ready += get_ready_commits(repo, 'branch/amsl')
 
 ready_commits = {}
+not_passed = {}
 for entry in ready:
     rev, repo, branch = entry
+    # Get the time, committer, and commit message
     cmd = 'svn log -v -r %s %s/%s/' % (rev, repo, branch)
     if opt_verbose > 1:
         note("Running '%s' ..." % cmd)
@@ -261,9 +263,25 @@ for entry in ready:
         type, path = line[:4], line[5:]
         if 'M' in type or 'A' in type or 'D' in type:
             break
+    # Get the test status
+    cmd = 'svn propget --revprop -r %s "test:unittest"' % rev
+    unittest = pipe(cmd).strip()
+    # 
     merge_path = os.path.join(*path.split(os.path.sep)[:4])
     if not (rev, repo, merge_path) in hold:
-        ready_commits[when] = "%s  %-24s %s@%s" % (when, who+":", merge_path, rev)
+        output_line = "%s  %-24s %s@%s" % (when, who+":", merge_path, rev)
+        if unittest == 'passed':
+            ready_commits[when] = output_line
+        else:
+            not_passed[when] = output_line
+
+keys = not_passed.keys()
+keys.sort()
+if len(keys) > 0:
+    sys.stderr.write("Commits marked ready which haven't passed the test suite:\n")
+    for key in keys:
+        sys.stderr.write(not_passed[key]+'\n')
+    sys.stderr.write('\n')
 
 keys = ready_commits.keys()
 keys.sort()

bin/mkrelease

@@ -79,6 +79,7 @@ longopts=help,dry-run,verbose,version
 
 # Default values
 MSG=""
+PROJ=ietfdb
 VERFILE=ietf/__init__.py
 SETTINGS=ietf/settings.py
 do=""
@@ -158,25 +159,33 @@ SRC=${SRC#/}				# remove leading slash
 MAJOR=${VER%%.*}
 REST=${VER#*.}
 MINOR=${REST%%.*}
-CHANGE=${REST#*.}
-VER="$(printf %d.%d.%d $MAJOR $MINOR $CHANGE)"
-NEXT=$(( $CHANGE + 1 ))
-DEV="$(printf %d.%d.%d-dev0 $MAJOR $MINOR $NEXT)"
+MAINT=${REST#*.}
+VER="$(printf %d.%d.%d $MAJOR $MINOR $MAINT)"
+NEXT=$(( $MAINT + 1 ))
+DEV="$(printf %d.%d.%d.dev0 $MAJOR $MINOR $NEXT)"
 
-note "Checking that there's a recent test-crawler log"
-touch -d $RDATE .svn/.latest-commit
-TCLOG=$(ls -t ../test-crawl-*.log | head -n 1)
-[ $TCLOG -nt .svn/.latest-commit ] || die "Looked for ../test-crawl-*.log, but didn't find one newer than the latest repository commit ($RDATE)"
+#cd $DIR ??
+
+note "Running the tests suite and writing release coverage data ..."
+$do ietf/manage.py test --settings=ietf.settings_releasetest --save-version-coverage=$VER
+
+note "Committing the release coverage data ..."
+$do svn commit release-coverage.json -m "Code coverage data for release $VER"
+
+# note "Checking that there's a recent test-crawler log"
+# touch -d $RDATE .svn/.latest-commit
+# TCLOG=$(ls -t ../test-crawl-*.log | head -n 1)
+# [ $TCLOG -nt .svn/.latest-commit ] || die "Looked for ../test-crawl-*.log, but didn't find one newer than the latest repository commit ($RDATE)"
 
 note "Checking that changelog information is available"
-changes=$( sed -n "/^ietfdb ($VER.*)/,/^ -- /p" changelog )
+changes=$( sed -n "/^$PROJ ($VER.*)/,/^ -- /p" changelog )
 [ "$changes" ] || die "No changelog information for $VER found"
 note "$changes"
 
 contributors=$(echo "$changes" | sed 's/\.[ \t\n]/ /'| tr -c "a-z0-9.@-" "\n" | sort | uniq | grep '@' | sed -r -e 's/^\.+//' -e 's/\.+$//' -e 's/^/-c /' || true)
 
 note "Set the current time on the release notes in the changelog file"
-sed -r -i -e "1,/^ -- /s/([A-Za-z-]+ <[a-z0-9.-]+@[a-z0-9.-]+>  ).*$/\1$(TZ=UTC date +'%d %b %Y %H:%M:%S %z')/" changelog
+$do sed -r -i -e "1,/^ -- /s/([A-Za-z-]+ <[a-z0-9.-]+@[a-z0-9.-]+>  ).*$/\1$(TZ=UTC date +'%d %b %Y %H:%M:%S %z')/" changelog
 note " $(grep -m1 "^ -- " changelog)"
 
 note "Verifying that version $VER doesn't already exist..."
@@ -190,7 +199,7 @@ note "Verifying there's no uncommitted changes..."
 $do svn st | grep "^[AMGRD] " && die "There seems to be uncommitted changes in this working copy"
 
 note "\nUpdating the version info in $VERFILE and making sure'\$Rev\$' is Ok"
-$do sed -i -r -e "/^__version__/s/\"[.0-9]+(-dev[0-9]*)?\"/\"$VER\"/" \
+$do sed -i -r -e "/^__version__/s/\"[.0-9]+(dev[0-9]+)?\"/\"$VER\"/" \
 	  -e "/^__rev__/s/\".*\"/\"\$Rev:\$\"/" \
 	$VERFILE
 
@@ -234,10 +243,10 @@ Release notes:
 $changes
 
 The new version is available for installation through SVN checkout, with
-  'svn checkout http://svn.tools.ietf.org/svn/tools/ietfdb/tags/$VER'
+  'svn checkout http://svn.tools.ietf.org/svn/tools/$PROJ/tags/$VER'
 
 For development, check out the new development version instead:
-  'svn checkout http://svn.tools.ietf.org/svn/tools/ietfdb/tags/dev/$DEV'
+  'svn checkout http://svn.tools.ietf.org/svn/tools/$PROJ/tags/dev/$DEV'
 
 Regards,
 

bin/test-crawl

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os, sys, re, datetime, argparse, traceback, tempfile
+import os, sys, re, datetime, argparse, traceback, tempfile, json
 
 # args
 parser = argparse.ArgumentParser(
@@ -14,6 +14,7 @@ parser.add_argument('--urls', '-u', dest='url_file',
 parser.add_argument('--slow', dest='slow_threshold', type=float, default=1.0,
                     help='responses taking longer than this (in seconds) results in SLOW being printed')
 parser.add_argument('--settings', dest='settings', help='custom settings file')
+parser.add_argument('--logfile', dest='logfile', help='write to logfile')
 
 args = parser.parse_args()
 
@@ -51,6 +52,7 @@ if args.url_file:
 
 if not initial_urls:
     initial_urls.append("/")
+    initial_urls.append("/api/v1")
 
 visited = set()
 urls = {} # url -> referrer
@@ -78,8 +80,30 @@ def extract_html_urls(content):
 
         yield url
 
+def extract_tastypie_urls(content):
+    VISIT_OBJECTS = False
+    VISIT_NEXT = False
+    data = json.loads(content)
+    for item in data:
+        if type(data[item]) is dict:
+            if "list_endpoint" in data[item]:
+                uri = data[item]["list_endpoint"]
+                yield uri
+    if VISIT_NEXT:
+        if "meta" in data and "next" in data["meta"]:
+            uri = data["meta"]["next"]
+            if uri != None:
+                yield uri
+    if VISIT_OBJECTS:
+        if "objects" in data:
+            object_list = data["objects"]
+            for i in range(len(object_list)):
+                if "resource_uri" in object_list[i]:
+                    uri = object_list[i]["resource_uri"]
+                    yield uri
+
 django.setup()
-client = django.test.Client()
+client = django.test.Client(Accept='text/html,text/plain,application/json')
 
 for url in initial_urls:
     urls[url] = "[initial]"
@@ -88,14 +112,16 @@ errors = 0
 count = 0
 
 start_time = datetime.datetime.now()
-fh, fn = tempfile.mkstemp(prefix="test-crawl-", suffix=".log", dir="../")
-logfile = open(fn, "w")
-os.close(fh)
+
+logfile = None
+if args.logfile:
+    logfile = open(args.logfile, "w")
 
 def log(s):
     print(s)
-    logfile.write(s)
-    logfile.write('\n')
+    if logfile:
+        logfile.write(s)
+        logfile.write('\n')
 
 def get_referrers(url):
     ref_list = []
@@ -150,8 +176,19 @@ while urls:
                     log("=============")
                     log(traceback.format_exc())
                     log("=============")
+            elif ctype == "application/json":
+                try:
+                    for u in extract_tastypie_urls(r.content):
+                        if u not in visited and u not in urls:
+                            urls[u] = url
+                            referrers[u] = url
+                except:
+                    log("error extracting urls from %s" % url)
+                    log("=============")
+                    log(traceback.format_exc())
+                    log("=============")
         else:
-            tags.append(u"FAIL (from %s)" % referrer)
+            tags.append(u"FAIL for %s\n   (from %s)" % (url, referrer))
             errors += 1
 
         if elapsed.total_seconds() > slow_threshold:
@@ -164,12 +201,13 @@ while urls:
         sec = acc_secs % 60
 
         if (len(visited) % 100) == 1:
-            log("\nElapsed  Visited  Queue   Code  Time  Url  ...  Notes")
+            log("\nElapsed  Visited  Queue Code   Time  Url  ...  Notes")
 
-        log("%2d:%02d:%02d %7d %6d  %s %.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags)))
+        log("%2d:%02d:%02d %7d %6d  %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags)))
 
-logfile.close()
-sys.stderr.write("Output written to %s\n\n" % logfile.name)
+if logfile:
+    logfile.close()
+    sys.stderr.write("Output written to %s\n\n" % logfile.name)
 
 if errors > 0:
     sys.stderr.write("Found %s errors, grep output for FAIL for details\n" % errors)