Adding diff capability to the URL tests.

If you list a second url or file on a line in an urltest.list file, the
second url will be downloaded, both urls will be converted to text, and
then compared.  If there are differences, a unified diff will be output.
 - Legacy-Id: 278

Author: levkowetz

ietf/tests.py

@@ -1,6 +1,12 @@
 import os
 import re
+import sys
 import traceback
+import urllib2 as urllib
+
+from ietf.utils import soup2text as html2text
+from difflib import unified_diff
+import textwrap
 
 import django.test.simple
 from django.test import TestCase
@@ -18,6 +24,16 @@ def run_tests(module_list, verbosity=1, extra_tests=[]):
     # during the search for a 'tests' module ...
     return django.test.simple.run_tests(module_list, verbosity, extra_tests)
 
+def reduce(html):
+    html = re.sub(" :", ":", html)
+    text = html2text(html)
+    text = re.sub('\."', '".', text)
+    #text = re.sub("\n\n+", "\n\n", text)
+    #text = "\n\n".join([textwrap.fill(para, 80) for para in text.split("\n\n")])
+    #text = re.sub(" +", " ", text)
+    text = [ line.strip() for line in text.split("\n") ]
+    return text
+
 def get_patterns(module):
     all = []
     try:
@@ -113,6 +129,22 @@ def doUrlsTest(self, lst):
                     res = ("Fail", "Exc")
                     print "Exception for URL '%s'" % url
                     traceback.print_exc()
+                if master:
+                    try:
+                        print "Fetching", master, "...",
+                        mfile = urllib.urlopen(master)
+                        goodhtml = mfile.read()
+                        mfile.close()
+                        print ""
+                        if goodhtml and response.content:
+                            testtext = reduce(response.content)
+                            goodtext = reduce(goodhtml)
+                            if not testtext == goodtext:
+                                for line in unified_diff(goodtext, testtext, url, master, lineterm=False):
+                                    print line
+                    except urllib.URLError, e:
+                        print "Failed retrieving master text for comparison: %s" % e
+
                 if not res in response_count:
                     response_count[res] = 0
                 response_count[res] += 1
@@ -131,7 +163,7 @@ def testUrlsList(self):
         print "\nTesting specified URLs:"
         self.doUrlsTest(self.testtuples)
 
-    def testUrlsFallback(self):
+    def XtestUrlsFallback(self):
         patterns = get_patterns(ietf.urls)
         lst = []
         for pattern in patterns: