Added test cases for rfc-index.xml/queue2.xml parsing; refactored mirroring scripts to support testing; fixed minor bug in queue2.xml parsing (was creating unnecessary references)

 - Legacy-Id: 1826

Author: pasieronen

ietf/idrfc/mirror_rfc_editor_queue.py

@@ -49,7 +49,7 @@
 log_data = ""
 def log(line):
     global log_data
-    if len(sys.argv) > 1:
+    if __name__ == '__main__' and len(sys.argv) > 1:
         print line
     else:
         log_data += line + "\n"
@@ -125,6 +125,14 @@ def recurse(draft_name, ref_set, level):
                     ref_set.add(destination)
                     recurse(destination, ref_set, level+1)
         if level == 0:
+            # Remove self-reference
+            ref_set.remove(draft_name)
+            # Remove direct references
+            for (source, destination, in_queue, direct) in refs:
+                if source == draft_name:
+                    if destination in ref_set:
+                        ref_set.remove(destination)
+            # The rest are indirect references
             for ref in ref_set:
                 if draft_name != ref:
                     result.append([draft_name, ref, ref in draft_names, False])
@@ -151,14 +159,7 @@ def find_document_ids(cursor, drafts, refs):
             refs2.append([draft_ids[ref[0]]]+ref[1:])
     return (drafts2, refs2)
 
-try:
-    log("output from mirror_rfc_editor_queue.py:\n")
-    log("time: "+str(datetime.now()))
-    log("host: "+socket.gethostname())
-    log("url: "+QUEUE_URL)
-
-    log("downloading...")
-    response = urllib2.urlopen(QUEUE_URL)
+def parse_all(response):
     log("parsing...")
     (drafts, refs) = parse(response)
     log("got "+ str(len(drafts)) + " drafts and "+str(len(refs))+" direct refs")
@@ -168,18 +169,17 @@ def find_document_ids(cursor, drafts, refs):
     refs.extend(indirect_refs)
     del(indirect_refs)
 
-    if len(drafts) < 10 or len(refs) < 10:
-        raise Exception('not enough data')
-
     # convert filenames to id_document_tags
     log("connecting to database...")
     cursor = db.connection.cursor()
     log("finding id_document_tags...")
     (drafts, refs) = find_document_ids(cursor, drafts, refs)
+    cursor.close()
+    return (drafts, refs)
 
-    if len(drafts) < 10 or len(refs) < 10:
-        raise Exception('not enough data')
-
+def insert_into_database(drafts, refs):
+    log("connecting to database...")
+    cursor = db.connection.cursor()
     log("removing old data...")
     cursor.execute("DELETE FROM "+TABLE)
     cursor.execute("DELETE FROM "+REF_TABLE)
@@ -190,10 +190,26 @@ def find_document_ids(cursor, drafts, refs):
     cursor.close()
     db.connection._commit()
     db.connection.close()
-
-    log("all done!")
-    if log_data.find("WARNING") < 0:
-        log_data = ""
-finally:
-    if len(log_data) > 0:
-        print log_data
+    
+if __name__ == '__main__':
+    try:
+        log("output from mirror_rfc_editor_queue.py:\n")
+        log("time: "+str(datetime.now()))
+        log("host: "+socket.gethostname())
+        log("url: "+QUEUE_URL)
+        
+        log("downloading...")
+        response = urllib2.urlopen(QUEUE_URL)
+
+        (drafts, refs) = parse_all(response)
+        if len(drafts) < 10 or len(refs) < 10:
+            raise Exception('not enough data')
+    
+        insert_into_database(drafts, refs)
+
+        log("all done!")
+        if log_data.find("WARNING") < 0:
+            log_data = ""
+    finally:
+        if len(log_data) > 0:
+            print log_data

ietf/idrfc/mirror_rfc_index.py

@@ -48,7 +48,7 @@
 log_data = ""
 def log(line):
     global log_data
-    if len(sys.argv) > 1:
+    if __name__ == '__main__' and len(sys.argv) > 1:
         print line
     else:
         log_data += line + "\n"
@@ -127,21 +127,7 @@ def getDocList(parentNode, tagName):
             d[9] = ",".join(also_list[k])
     return data
 
-try:
-    log("output from mirror_rfc_index.py:\n")
-    log("time: "+str(datetime.now()))
-    log("host: "+socket.gethostname())
-    log("url: "+INDEX_URL)
-
-    log("downloading...")
-    response = urllib2.urlopen(INDEX_URL)
-    log("parsing...")
-    data = parse(response)
-
-    log("got " + str(len(data)) + " entries")
-    if len(data) < 5000:
-        raise Exception('not enough data')
-
+def insert_to_database(data):
     log("connecting to database...")
     cursor = db.connection.cursor()
     log("removing old data...")
@@ -152,9 +138,27 @@ def getDocList(parentNode, tagName):
     db.connection._commit()
     db.connection.close()
 
-    log("all done!")
-    log_data = ""
+if __name__ == '__main__':
+    try:
+        log("output from mirror_rfc_index.py:\n")
+        log("time: "+str(datetime.now()))
+        log("host: "+socket.gethostname())
+        log("url: "+INDEX_URL)
+
+        log("downloading...")
+        response = urllib2.urlopen(INDEX_URL)
+        log("parsing...")
+        data = parse(response)
+
+        log("got " + str(len(data)) + " entries")
+        if len(data) < 5000:
+            raise Exception('not enough data')
+
+        insert_to_database(data)
+
+        log("all done!")
+        log_data = ""
 
-finally:
-    if len(log_data) > 0:
-        print log_data
+    finally:
+        if len(log_data) > 0:
+            print log_data