Import document replaces, revamp document relationships a little bit to remove a model, fix bugs

 - Legacy-Id: 2730

Author: OleLaursen

redesign/doc/admin.py

@@ -2,12 +2,6 @@
 from models import *
 from person.models import *
 
-class RelatedDocAdmin(admin.ModelAdmin):
-    list_display = ["source", "relationship", "target"]
-    search_fields = ["doc_alias__name", "related_document_set__name", ]
-    list_display_links = ["relationship", ]
-admin.site.register(RelatedDoc, RelatedDocAdmin)    
-
 class DocumentAdmin(admin.ModelAdmin):
     list_display = ['name', 'rev', 'state', 'group', 'pages', 'intended_std_level', 'author_list', 'time']
     search_fields = ['name']

redesign/doc/models.py

@@ -8,14 +8,6 @@
 
 import datetime
 
-class RelatedDoc(models.Model):
-    relationship = models.ForeignKey(DocRelationshipName)
-    doc_alias = models.ForeignKey('DocAlias')
-    def __unicode__(self):
-        return "%s %s" % (self.relationship.name, self.doc_alias.name)
-    source = admin_link("related_document_set")
-    target = admin_link("doc_alias__document")
-
 class DocumentInfo(models.Model):
     """Any kind of document.  Draft, RFC, Charter, IPR Statement, Liaison Statement"""
     time = models.DateTimeField() # should probably have auto_now=True
@@ -38,7 +30,6 @@ class DocumentInfo(models.Model):
     intended_std_level = models.ForeignKey(IntendedStatusName, blank=True, null=True)
     std_level = models.ForeignKey(StdStatusName, blank=True, null=True)
     authors = models.ManyToManyField(Email, blank=True, null=True)
-    related = models.ManyToManyField(RelatedDoc, related_name='related_%(class)s_set', blank=True)
     ad = models.ForeignKey(Email, related_name='ad_%(class)s_set', blank=True, null=True)
     shepherd = models.ForeignKey(Email, related_name='shepherd_%(class)s_set', blank=True, null=True)
     notify = models.CharField(max_length=255, blank=True)
@@ -58,8 +49,16 @@ def latest_event(self, *args, **filter_args):
         e = model.objects.filter(doc=self).filter(**filter_args).order_by('-time')[:1]
         return e[0] if e else None
 
+class RelatedDocument(models.Model):
+    document = models.ForeignKey('Document')  # source
+    doc_alias = models.ForeignKey('DocAlias') # target
+    relationship = models.ForeignKey(DocRelationshipName)
+    def __unicode__(self):
+        return u"%s %s %s" % (self.document.name, self.relationship.name.lower(), self.doc_alias.name)
+
 class Document(DocumentInfo):
     name = models.CharField(max_length=255, primary_key=True)           # immutable
+    related = models.ManyToManyField('DocAlias', through=RelatedDocument, blank=True, related_name="reversely_related_document_set")
     def __unicode__(self):
         return self.name
     def values(self):
@@ -90,6 +89,7 @@ def save_with_history(self, force_insert=False, force_update=False):
             snap = DocHistory(**fields)
             snap.save()
             for m in many2many:
+                # FIXME: check that this works with related
                 #print "m2m:", m, many2many[m]
                 rel = getattr(snap, m)
                 for item in many2many[m]:
@@ -102,8 +102,16 @@ def save_with_history(self, force_insert=False, force_update=False):
             print "Deleted list:", snap
         super(Document, self).save(force_insert, force_update)
 
+class RelatedDocHistory(models.Model):
+    document = models.ForeignKey('DocHistory')
+    doc_alias = models.ForeignKey('DocAlias', related_name="reversely_related_document_history_set")
+    relationship = models.ForeignKey(DocRelationshipName)
+    def __unicode__(self):
+        return u"%s %s %s" % (self.document.name, self.relationship.name.lower(), self.doc_alias.name)
+
 class DocHistory(DocumentInfo):
     doc = models.ForeignKey(Document)   # ID of the Document this relates to
+    related = models.ManyToManyField('DocAlias', through=RelatedDocHistory, blank=True)
     def __unicode__(self):
         return unicode(self.doc.name)
 

redesign/doc/proxy.py

@@ -133,20 +133,18 @@ def last_modified_date(self):
     #replaced_by = models.ForeignKey('self', db_column='replaced_by', blank=True, null=True, related_name='replaces_set')
     @property
     def replaced_by(self):
-        r = InternetDraft.objects.filter(docalias__relateddoc__relationship="replaces", docalias__relateddoc__related_document_set=self)
+        r = InternetDraft.objects.filter(related__document=self, related__relateddocument__relationship="replaces")
         return r[0] if r else None
 
     #replaces = FKAsOneToOne('replaces', reverse=True)
     @property
     def replaces(self):
-        r = InternetDraft.objects.filter(related__doc_alias__document=self, related__relationship="replaces")
-        return r[0] if r else Non
+        r = self.replaces_set()
+        return r[0] if r else None
 
-    
     @property
     def replaces_set(self):
-        # this is replaced_by
-        return InternetDraft.objects.filter(docalias__relateddoc__relationship="replaces", docalias__relateddoc__related_document_set=self)
+        return InternetDraft.objects.filter(docalias__relateddocument__relationship="replaces", docalias__relateddocument__document=self)
 
     #review_by_rfc_editor = models.BooleanField()
     @property

redesign/import-document-state.py

@@ -46,6 +46,8 @@ def name(name_class, slug, name, desc=""):
 type_draft = name(DocTypeName, "draft", "Draft")
 stream_ietf = name(DocStreamName, "ietf", "IETF")
 
+relationship_replaces = name(DocRelationshipName, "replaces", "Replaces")
+
 intended_status_mapping = {
     "BCP": name(IntendedStatusName, "bcp", "Best Current Practice"),
     "Draft Standard": name(IntendedStatusName, "ds", name="Draft Standard"),
@@ -145,7 +147,7 @@ def iesg_login_to_email(l):
 def date_in_match(match):
     return datetime.date(int(match.group('year')), int(match.group('month')), int(match.group('day')))
 
-re_telechat_agenda = re.compile(r"(Placed on|Removed from) agenda for telechat - %s by" % date_re_str)
+re_telechat_agenda = re.compile(r"(Placed on|Removed from) agenda for telechat(| - %s) by" % date_re_str)
 re_ballot_position = re.compile(r"\[Ballot Position Update\] (New position, (?P<position>.*), has been recorded (|for (?P<for>.*) )|Position (|for (?P<for2>.*) )has been changed to (?P<position2>.*) from .*)by (?P<by>.*)")
 re_ballot_issued = re.compile(r"Ballot has been issued(| by)")
 re_state_changed = re.compile(r"(State (has been changed|changed|Changes) to <b>(?P<to>.*)</b> from <b>(?P<from>.*)</b> by|Sub state has been changed to (?P<tosub>.*) from (?P<fromsub>.*))")
@@ -191,14 +193,18 @@ def date_in_match(match):
 #    d.std_level =
 #    d.authors =
 #    d.related =
-    d.ad = iesg_login_to_email(o.idinternal.job_owner)
+    d.ad = iesg_login_to_email(o.idinternal.job_owner) if o.idinternal else None
     d.shepherd = None
     d.notify = o.idinternal.state_change_notice_to or "" if o.idinternal else ""
     d.external_url = ""
     d.note = o.idinternal.note or "" if o.idinternal else ""
     d.internal_comments = o.comments or "" # FIXME: maybe put these somewhere else
     d.save()
 
+    # make sure our alias is updated
+    DocAlias.objects.filter(name=d.name).exclude(document=d).delete()
+    d_alias, _ = DocAlias.objects.get_or_create(name=d.name, document=d)
+    
     # clear already imported events
     d.event_set.all().delete()
 
@@ -382,21 +388,60 @@ def date_in_match(match):
                 save_event(d, e, c)
 
                 # stop typical comments from being output
-                typical_comments = ["Who is the Document Shepherd for this document",
-                                    "We understand that this document doesn't require any IANA actions",
+                typical_comments = [
+                    "Document Shepherd Write-up for %s" % d.name,
+                    "Who is the Document Shepherd for this document",
+                    "We understand that this document doesn't require any IANA actions",
                                     ]
                 for t in typical_comments:
                     if t in c.comment_text:
                         handled = True
                         break
 
             if not handled:
-                print "couldn't handle %s '%s'" % (c.id, c.comment_text.replace("\n", "").replace("\r", ""))
+                print "couldn't handle %s '%s'" % (c.id, c.comment_text.replace("\n", "").replace("\r", "")[0:80])
 
 
-        # import events that might be missing, we can't be sure where
-        # to place them but if we don't generate them, we'll be
-        # missing the information completely
+    # import missing revision changes from DraftVersions
+    known_revisions = set(e.newrevision.rev for e in d.event_set.filter(type="new_revision").select_related('newrevision'))
+    for v in DraftVersions.objects.filter(filename=d.name).order_by("revision"):
+        if v.revision not in known_revisions:
+            e = NewRevision(type="new_revision")
+            e.rev = v.revision
+            # we don't have time information in this source, so
+            # hack the seconds to include the revision to ensure
+            # they're ordered correctly
+            e.time = datetime.datetime.combine(v.revision_date, datetime.time(0, 0, int(v.revision)))
+            e.by = system_email
+            e.doc = d
+            e.desc = "New version available"
+            e.save()
+            known_revisions.add(v.revision)
+    
+    # import events that might be missing, we can't be sure where
+    # to place them but if we don't generate them, we'll be
+    # missing the information completely
+    e = d.latest_event(Event, type="iesg_approved")
+    approved_date = e.time.date() if e else None
+    if o.b_approve_date != approved_date:
+        e = Event(type="iesg_approved")
+        e.time = o.idinternal.b_approve_date
+        e.by = system_email
+        e.doc = d
+        e.desc = "IESG has approved"
+        e.save()
+
+    if o.lc_expiration_date:
+        e = Expiration(type="sent_last_call", expires=o.lc_expiration_date)
+        e.time = o.lc_sent_date
+        # let's try to figure out who did it
+        events = d.event_set.filter(type="changed_document", desc__contains=" to <b>In Last Call</b>").order_by('-time')[:1]
+        e.by = events[0].by if events else system_email
+        e.doc = d
+        e.desc = "Last call sent"
+        e.save()
+
+    if o.idinternal:
         e = d.latest_event(Status, type="changed_status_date")
         status_date = e.date if e else None
         if o.idinternal.status_date != status_date:
@@ -407,65 +452,36 @@ def date_in_match(match):
             e.desc = "Status date has been changed to <b>%s</b> from <b>%s</b>" % (o.idinternal.status_date, status_date)
             e.save()
 
-        e = d.latest_event(Event, type="iesg_approved")
-        approved_date = e.time.date() if e else None
-        if o.b_approve_date != approved_date:
-            e = Event(type="iesg_approved")
-            e.time = o.idinternal.b_approve_date
-            e.by = system_email
-            e.doc = d
-            e.desc = "IESG has approved"
-            e.save()
+        e = d.latest_event(Telechat, type="scheduled_for_telechat")
+        telechat_date = e.telechat_date if e else None
+        if not o.idinternal.agenda:
+            o.idinternal.telechat_date = None # normalize
 
-        if o.lc_expiration_date:
-            e = Expiration(type="sent_last_call", expires=o.lc_expiration_date)
-            e.time = o.lc_sent_date
-            # let's try to figure out who did it
-            events = d.event_set.filter(type="changed_document", desc__contains=" to <b>In Last Call</b>").order_by('-time')[:1]
-            e.by = events[0].by if events else system_email
+        if telechat_date != o.idinternal.telechat_date:
+            e = Telechat(type="scheduled_for_telechat",
+                         telechat_date=o.idinternal.telechat_date,
+                         returning_item=bool(o.idinternal.returning_item))
+            e.time = made_up_date # FIXME: this isn't good, will override new values, estimate one instead 
+            e.by = system_email
+            args = ("Placed on", o.idinternal.telechat_date) if o.idinternal.telechat_date else ("Removed from", telechat_date)
             e.doc = d
-            e.desc = "Last call sent"
+            e.desc = "%s agenda for telechat - %s by system" % args
             e.save()
-
-        if o.idinternal:
-            e = d.latest_event(Telechat, type="scheduled_for_telechat")
-            telechat_date = e.telechat_date if e else None
-            if not o.idinternal.agenda:
-                o.idinternal.telechat_date = None # normalize
-                
-            if telechat_date != o.idinternal.telechat_date:
-                e = Telechat(type="scheduled_for_telechat",
-                             telechat_date=o.idinternal.telechat_date,
-                             returning_item=bool(o.idinternal.returning_item))
-                e.time = made_up_date
-                e.by = system_email
-                args = ("Placed on", o.idinternal.telechat_date) if o.idinternal.telechat_date else ("Removed from", telechat_date)
-                e.doc = d
-                e.desc = "%s agenda for telechat - %s by system" % args
-                e.save()
 
          # FIXME: import writeups
 
+    # import other attributes
+
     # RFC alias
     if o.rfc_number:
         rfc_name = "rfc%s" % o.rfc_number
         DocAlias.objects.get_or_create(document=d, name=rfc_name)
-            
-    # import missing revision changes from DraftVersions
-    known_revisions = set(e.newrevision.rev for e in d.event_set.filter(type="new_revision").select_related('newrevision'))
-    for v in DraftVersions.objects.filter(filename=d.name).order_by("revision"):
-        if v.revision not in known_revisions:
-            e = NewRevision(type="new_revision")
-            e.rev = v.revision
-            # we don't have time information in this source, so
-            # hack the seconds to include the revision to ensure
-            # they're ordered correctly
-            e.time = datetime.datetime.combine(v.revision_date, datetime.time(0, 0, int(v.revision)))
-            e.by = system_email
-            e.doc = d
-            e.desc = "New version available"
-            e.save()
-            known_revisions.add(v.revision)
+        # FIXME: some RFCs seem to be called rfc1234bis?
+
+    if o.replaced_by:
+        replacement, _ = Document.objects.get_or_create(name=o.replaced_by.filename)
+        RelatedDocument.objects.get_or_create(document=replacement, doc_alias=d_alias, relationship=relationship_replaces)
+        
 
     print "imported", d.name, " - ", d.iesg_state
 
@@ -503,8 +519,8 @@ class CheckListInternetDraft(models.Model):
 #    rfc_number = models.IntegerField(null=True, blank=True, db_index=True)
 #    comments = models.TextField(blank=True,null=True)
 #    last_modified_date = models.DateField()
-    replaced_by = BrokenForeignKey('self', db_column='replaced_by', blank=True, null=True, related_name='replaces_set')
-    replaces = FKAsOneToOne('replaces', reverse=True)
+#    replaced_by = BrokenForeignKey('self', db_column='replaced_by', blank=True, null=True, related_name='replaces_set')
+#    replaces = FKAsOneToOne('replaces', reverse=True)
     review_by_rfc_editor = models.BooleanField()
     expired_tombstone = models.BooleanField()
 #    idinternal = FKAsOneToOne('idinternal', reverse=True, query=models.Q(rfc_flag = 0))