Parse stream/wg/file formats from rfc-index.xml; prepare for parsing auth48 info from queue2.xml

 - Legacy-Id: 2056

Author: pasieronen

ietf/idrfc/mirror_rfc_index.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
 # All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
 #
 # Redistribution and use in source and binary forms, with or without
@@ -109,6 +109,15 @@ def getDocList(parentNode, tagName):
             updated_by = getDocList(node, "updated-by")
             obsoletes = getDocList(node, "obsoletes") 
             obsoleted_by = getDocList(node, "obsoleted-by")
+            stream = getChildText(node, "stream")
+            wg = getChildText(node, "wg_acronym")
+            if wg and ((wg == "NON WORKING GROUP") or len(wg) > 15):
+                wg = None
+           
+            l = []
+            for format in node.getElementsByTagName("format"):
+                l.append(getChildText(format, "file-format"))
+            file_formats = (",".join(l)).lower()
 
             draft = getChildText(node, "draft")
             if draft and re.search("-\d\d$", draft):
@@ -119,7 +128,7 @@ def getDocList(parentNode, tagName):
             else:
                 has_errata = 0
 
-            data.append([rfc_number,title,authors,rfc_published_date,current_status,updates,updated_by,obsoletes,obsoleted_by,None,draft,has_errata])
+            data.append([rfc_number,title,authors,rfc_published_date,current_status,updates,updated_by,obsoletes,obsoleted_by,None,draft,has_errata,stream,wg,file_formats])
 
     for d in data:
         k = "RFC%04d" % d[0]
@@ -133,7 +142,7 @@ def insert_to_database(data):
     log("removing old data...")
     cursor.execute("DELETE FROM "+TABLE)
     log("inserting new data...")
-    cursor.executemany("INSERT INTO "+TABLE+" (rfc_number, title, authors, rfc_published_date, current_status,updates,updated_by,obsoletes,obsoleted_by,also,draft,has_errata) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", data)
+    cursor.executemany("INSERT INTO "+TABLE+" (rfc_number, title, authors, rfc_published_date, current_status,updates,updated_by,obsoletes,obsoleted_by,also,draft,has_errata,stream,wg,file_formats) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", data)
     cursor.close()
     db.connection._commit()
     db.connection.close()

ietf/idrfc/models.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+# Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
 # All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
 #
 # Redistribution and use in source and binary forms, with or without
@@ -48,6 +48,8 @@ class RfcEditorQueue(models.Model):
     # this field will be NULL (but we could get it from other sources)
     state_date = models.DateField(blank=True,null=True)
     stream = models.IntegerField(choices=STREAM_CHOICES)
+    auth48_url =  models.CharField(max_length=200, blank=True, null=True)
+    rfc_number = models.IntegerField(null=True)
     def __str__(self):
         return "RfcEditorQueue"+str([self.draft, self.date_received, self.state, self.state_date, self.stream])
     class Meta:
@@ -74,6 +76,9 @@ class RfcIndex(models.Model):
     also = models.CharField(max_length=50,blank=True,null=True)
     draft = models.CharField(max_length=200,null=True)
     has_errata = models.BooleanField()
+    stream = models.CharField(max_length=15,blank=True,null=True)
+    wg = models.CharField(max_length=15,blank=True,null=True)
+    file_formats = models.CharField(max_length=20,blank=True,null=True)
     def __str__(self):
         return "RfcIndex"+str(self.rfc_number)
     class Meta:

test/sql_fixup.sql

@@ -1,3 +1,10 @@
 -- This file holds needed corrections to the database until they have been applied to
 -- the live database.  This file is applied after importing a new dump of the live DB.
 DROP TABLE idtracker_areaurl;
+
+ALTER TABLE rfc_editor_queue_mirror ADD COLUMN auth48_url VARCHAR(200);
+ALTER TABLE rfc_editor_queue_mirror ADD COLUMN rfc_number INTEGER;
+ALTER TABLE rfc_index_mirror ADD COLUMN stream VARCHAR(15);
+ALTER TABLE rfc_index_mirror ADD COLUMN wg VARCHAR(15);
+ALTER TABLE rfc_index_mirror ADD COLUMN file_formats VARCHAR(20);
+