Merged in more shim-layer removals from olau@iola.dk

(-r5465:HEAD from branch/iola/shimfree).  Copying relevant
commit messages here:

 - Deleted dead code in many places.

 - Renamed id-something to draft-something, make the "is
   this eligible for expiration" logic clearer

 - Added a name for IPR search URL

 - Revamped the ballot popup view

 - URL reversed the IPR search link instead of hardcoding it

 - Cleaned up search views and remove dead code, port them to the new DB
   schema, hack related views in iesg/ and wginfo/ to use the new search
   interfaces, avoid camelCase in search GET parameters (with
   backwards-compat fallback), add some simple search unit tests, remove
   caching from views_search.py index pages as they're now pretty fast to
   generate, rewrite ballot popup JS, regularize some CSS classes to use
   hyphen-notation rather than camelCase, move some of the search
   templates to doc/.

   idrfc/ now mostly contains some wrapper code still in use by other
   subdirs, some ported code not yet moved, and dead code.

 - Fixed output bug in test crawler and print referrer upon errors so it's
   easier to figure out where a link came from

 - Added /doc/in-last-call/ to crawler, report original page as referrer in
   a redirect chain rather than intermediate URL

 - Ported idindex to new schema, speed them up, add tests, refactor index
   page in views_search to share code with the text index file, get rid
   of some special-case idindex filters from ietf_filters, move
   "/drafts/" redirects to a file in /doc/

 - Ported /idtracker/status/ and /idtracker/status/last-call/ overview of
   drafts in IESG process to new schema in /doc/iesg/ and
   /doc/iesg/last-call/

 - Added redirects for all of /idtracker/*, removed all view code and other
   dead code from idtracker/

 - Removed the idtracker sitemap indexing drafts - in its current form,
   it adds nothing of value to the HTML-based /doc/all/ view, and it's
   pretty slow

 - Ported idtracker feeds to new schema, move them to doc/, cleaned up
   idtracker/ - only templatetags/ietf_filters and proxy code is left
 - Legacy-Id: 5836

Author: levkowetz

bin/mkrelease

@@ -231,7 +231,7 @@ Regards,
 	Henrik
 	(via the mkrelease script)
 " > ~/src/db/release-mail-v$VER.txt
-cat ~/src/db/release-mail-v$VER.txt | $do mail -s "New datatracker release: v$VER" -c henrik@levkowetz.com  -c glen@amsl.com -c fenner@fenron.net -c rjs@nostrum.com -c housley@vigilsec.com -c cmorgan@amsl.com -c avezza@amsl.com -c amorris@amsl.com -c smccammon@amsl.com -c kmoreland@amsl.com -c stevey@amsl.com -c wchen@amsl.com -c olau@iola.dk $contributors codesprints@ietf.org
+cat ~/src/db/release-mail-v$VER.txt | $do mail -s "New datatracker release: v$VER" -c henrik@levkowetz.com  -c glen@amsl.com -c fenner@fenron.net -c rjs@nostrum.com -c housley@vigilsec.com -c cmorgan@amsl.com -c avezza@amsl.com -c amorris@amsl.com -c smccammon@amsl.com -c kmoreland@amsl.com -c stevey@amsl.com -c olau@iola.dk $contributors codesprints@ietf.org
 
 $do toolsfeed control changelog /www/tools.ietf.org/tools/atomfeed.xml
 $do toolpush /www/tools.ietf.org/tools/atomfeed.xml

ietf/bin/expire-ids

@@ -11,10 +11,11 @@ syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER)
 
 from ietf.idrfc.expire import *
 
-if not in_id_expire_freeze():
-    for doc in get_expired_ids():
-        send_expire_notice_for_id(doc)
-        expire_id(doc)
-        syslog.syslog("Expired %s (id=%s)%s" % (doc.file_tag(), doc.pk, " in the ID Tracker" if doc.latest_event(type="started_iesg_process") else ""))
+if not in_draft_expire_freeze():
+    for doc in get_expired_drafts():
+        send_expire_notice_for_draft(doc)
+        expire_draft(doc)
+        syslog.syslog("Expired draft %s-%s" % (doc.name, doc.rev))
 
-clean_up_id_files()
+syslog.syslog("Cleaning up draft files")
+clean_up_draft_files()

ietf/bin/notify-expirations

@@ -6,11 +6,11 @@ from ietf import settings
 from django.core import management
 management.setup_environ(settings)
 
-from ietf.idrfc.expire import get_soon_to_expire_ids, send_expire_warning_for_id
+from ietf.idrfc.expire import get_soon_to_expire_drafts, send_expire_warning_for_draft
 
 
 # notify about documents that expire within the next 2 weeks
-notify_days = 14       
+notify_days = 14
 
-for doc in get_soon_to_expire_ids(notify_days):
-    send_expire_warning_for_id(doc)
+for doc in get_soon_to_expire_drafts(notify_days):
+    send_expire_warning_for_draft(doc)

ietf/bin/test-crawl

@@ -24,6 +24,12 @@ connection.queries = DontSaveQueries()
 MAX_URL_LENGTH = 500
 SLOW_THRESHOLD = 1.0
 
+initial = ["/doc/all/", "/doc/in-last-call/"]
+
+visited = set()
+urls = {} # url -> referrer
+
+
 def strip_url(url):
     if url.startswith("http://testserver"):
         url = url[len("http://testserver"):]
@@ -40,15 +46,13 @@ def extract_html_urls(content):
 
         yield url
 
-
-visited = set()
-blacklist = set()
-urls = set(["/doc/all/"])
-
 client = django.test.Client()
 
+for url in initial:
+    urls[url] = "[initial]"
+
 while urls:
-    url = urls.pop()
+    url, referrer = urls.popitem()
 
     visited.add(url)
 
@@ -62,15 +66,15 @@ while urls:
     except:
         print "FAIL", url
         print "============="
-        traceback.print_exc()
+        print traceback.format_exc()
         print "============="
     else:
         tags = []
 
         if r.status_code in (301, 302):
             u = strip_url(r["Location"])
             if u not in visited and u not in urls:
-                urls.add(u)
+                urls[u] = referrer # referrer is original referrer, not redirected url
 
         elif r.status_code == 200:
             ctype = r["Content-Type"]
@@ -80,9 +84,9 @@ while urls:
             if ctype == "text/html":
                 for u in extract_html_urls(r.content):
                     if u not in visited and u not in urls:
-                        urls.add(u)
+                        urls[u] = url
         else:
-            tags.append("FAIL")
+            tags.append(u"FAIL (from %s)" % referrer)
 
         if elapsed.total_seconds() > SLOW_THRESHOLD:
             tags.append("SLOW")

ietf/doc/feeds.py

@@ -0,0 +1,86 @@
+# Copyright The IETF Trust 2007, All Rights Reserved
+
+import datetime, re
+
+from django.conf import settings
+from django.contrib.syndication.feeds import Feed, FeedDoesNotExist
+from django.utils.feedgenerator import Atom1Feed
+from django.core.urlresolvers import reverse as urlreverse
+from django.template.defaultfilters import truncatewords_html, date as datefilter, linebreaks
+from django.utils.html import strip_tags
+from django.utils.text import truncate_words
+
+from ietf.doc.models import *
+from ietf.doc.utils import augment_events_with_revision
+from ietf.idtracker.templatetags.ietf_filters import format_textarea
+
+class DocumentChanges(Feed):
+    feed_type = Atom1Feed
+
+    def get_object(self, bits):
+	if len(bits) != 1:
+	    raise Document.DoesNotExist
+
+        return Document.objects.get(docalias__name=bits[0])
+
+    def title(self, obj):
+        return "Changes for %s" % obj.display_name()
+
+    def link(self, obj):
+	if obj is None:
+	    raise FeedDoesNotExist
+        if not hasattr(self, "cached_link"):
+            self.cached_link = urlreverse("doc_history", kwargs=dict(name=obj.canonical_name()))
+	return self.cached_link
+
+    def subtitle(self, obj):
+        return "History of change entries for %s." % obj.display_name()
+
+    def items(self, obj):
+        events = obj.docevent_set.all().order_by("-time","-id")
+        augment_events_with_revision(obj, events)
+	return events
+
+    def item_title(self, item):
+        return u"[%s] %s [rev. %s]" % (item.by, truncate_words(strip_tags(item.desc), 15), item.rev)
+
+    def item_description(self, item):
+        return truncatewords_html(format_textarea(item.desc), 20)
+
+    def item_pubdate(self, item):
+	return item.time
+
+    def item_author_name(self, item):
+	return unicode(item.by)
+
+    def item_link(self, item):
+        return self.cached_link + "#history-%s" % item.pk
+
+class InLastCall(Feed):
+    title = "Documents in Last Call"
+    subtitle = "Announcements for documents in last call."
+    feed_type = Atom1Feed
+    author_name = 'IESG Secretary'
+    link = "/doc/iesg/last-call/"
+
+    def items(self):
+        docs = list(Document.objects.filter(type="draft", states=State.objects.get(type="draft-iesg", slug="lc")))
+        for d in docs:
+            d.lc_event = d.latest_event(LastCallDocEvent, type="sent_last_call")
+
+        docs = [d for d in docs if d.lc_event]
+	docs.sort(key=lambda d: d.lc_event.expires)
+
+	return docs
+
+    def item_title(self, item):
+        return u"%s (%s - %s)" % (item.name,
+                                datefilter(item.lc_event.time, "F j"),
+                                datefilter(item.lc_event.expires, "F j, Y"))
+
+    def item_description(self, item):
+        return linebreaks(item.lc_event.desc)
+
+    def item_pubdate(self, item):
+        return item.lc_event.time
+

ietf/doc/models.py

@@ -37,6 +37,9 @@ def __unicode__(self):
     class Meta:
         ordering = ["type", "order"]
 
+IESG_BALLOT_ACTIVE_STATES = ("lc", "writeupw", "goaheadw", "iesg-eva", "defer")
+IESG_SUBSTATE_TAGS = ('point', 'ad-f-up', 'need-rev', 'extpty')
+
 class DocumentInfo(models.Model):
     """Any kind of document.  Draft, RFC, Charter, IPR Statement, Liaison Statement"""
     time = models.DateTimeField(default=datetime.datetime.now) # should probably have auto_now=True
@@ -186,9 +189,7 @@ def __unicode__(self):
     def get_absolute_url(self):
         name = self.name
         if self.type_id == "draft" and self.get_state_slug() == "rfc":
-            aliases = self.docalias_set.filter(name__startswith="rfc")
-            if aliases:
-                name = aliases[0].name
+            name = self.canonical_name()
         elif self.type_id in ('slides','agenda','minutes'):
             session = self.session_set.all()[0]
             meeting = session.meeting
@@ -260,11 +261,10 @@ def related_that_doc(self, relationship):
             raise TypeError("Expected a string, tuple or list, received %s" % type(relationship))
         return DocAlias.objects.filter(relateddocument__source=self, relateddocument__relationship__in=relationship)
 
-    #TODO can/should this be a function instead of a property? Currently a view uses it as a property
-    @property
-    def telechat_date(self):
-        e = self.latest_event(TelechatDocEvent, type="scheduled_for_telechat")
-        return e.telechat_date if e else None
+    def telechat_date(self, e=None):
+        if not e:
+            e = self.latest_event(TelechatDocEvent, type="scheduled_for_telechat")
+        return e.telechat_date if e and e.telechat_date and e.telechat_date >= datetime.date.today() else None
 
     def area_acronym(self):
         g = self.group
@@ -283,10 +283,6 @@ def group_acronym(self):
         else:
             return "none"
 
-    def on_upcoming_agenda(self):
-        e = self.latest_event(TelechatDocEvent, type="scheduled_for_telechat")
-        return bool(e and e.telechat_date and e.telechat_date >= datetime.date.today())
-
     def returning_item(self):
         e = self.latest_event(TelechatDocEvent, type="scheduled_for_telechat")
         return e.returning_item if e else None
@@ -311,56 +307,56 @@ def displayname_with_link(self):
         return '<a href="%s">%s-%s</a>' % (self.get_absolute_url(), self.name , self.rev)
 
     def rfc_number(self):
-        qs = self.docalias_set.filter(name__startswith='rfc')
-        return qs[0].name[3:] if qs else None
+        n = self.canonical_name()
+        return n[3:] if n.startswith("rfc") else None
 
     def friendly_state(self):
-        """ Return a concise text description of the document's current state """
-        if self.type_id=='draft':
-            # started_iesg_process is is how the redesigned database schema (as of May2012) captured what 
-            # used to be "has an IDInternal", aka *Wrapper.in_ietf_process()=True
-            in_iesg_process = self.latest_event(type='started_iesg_process')
-            iesg_state_summary=None
-            if in_iesg_process:
-                iesg_state = self.states.get(type='draft-iesg')
+        """ Return a concise text description of the document's current state."""
+        state = self.get_state()
+        if not state:
+            return "Unknown state"
+
+        if self.type_id == 'draft':
+            iesg_state = self.get_state("draft-iesg")
+            iesg_state_summary = None
+            if iesg_state:
                 # This knowledge about which tags are reportable IESG substate tags is duplicated in idrfc
-                IESG_SUBSTATE_TAGS = ('point', 'ad-f-up', 'need-rev', 'extpty')
                 iesg_substate = self.tags.filter(slug__in=IESG_SUBSTATE_TAGS)
                 # There really shouldn't be more than one tag in iesg_substate, but this will do something sort-of-sensible if there is
                 iesg_state_summary = iesg_state.name
                 if iesg_substate:
                      iesg_state_summary = iesg_state_summary + "::"+"::".join(tag.name for tag in iesg_substate)
 
-            if self.get_state_slug() == "rfc":
-                n = self.rfc_number()
-                return "<a href=\"%s\">RFC %s</a>" % (urlreverse('doc_view', kwargs=dict(name='rfc%s' % n)), n)
-            elif self.get_state_slug() == "repl":
+            if state.slug == "rfc":
+                return "RFC %s (%s)" % (self.rfc_number(), self.std_level)
+            elif state.slug == "repl":
                 rs = self.related_that("replaces")
                 if rs:
-                    return mark_safe("Replaced by " + ", ".join("<a href=\"%s\">%s</a>" % (urlreverse('doc_view', args=[name]), name) for name in rs))
+                    return mark_safe("Replaced by " + ", ".join("<a href=\"%s\">%s</a>" % (urlreverse('doc_view', kwargs=dict(name=name)), name) for name in rs))
                 else:
                     return "Replaced"
-            elif self.get_state_slug() == "active":
-                if in_iesg_process:
+            elif state.slug == "active":
+                if iesg_state:
                     if iesg_state.slug == "dead":
                         # Many drafts in the draft-iesg "Dead" state are not dead
                         # in other state machines; they're just not currently under 
                         # IESG processing. Show them as "I-D Exists (IESG: Dead)" instead...
-                        return "I-D Exists (IESG: "+iesg_state_summary+")"
+                        return "I-D Exists (IESG: %s)" % iesg_state_summary
                     elif iesg_state.slug == "lc":
-                        expiration_date = str(self.latest_event(LastCallDocEvent,type="sent_last_call").expires.date())
-                        return iesg_state_summary + " (ends "+expiration_date+")"
-                    else:
-                        return iesg_state_summary
+                        e = self.latest_event(LastCallDocEvent, type="sent_last_call")
+                        if e:
+                            return iesg_state_summary + " (ends %s)" % e.expires.date().isoformat()
+
+                    return iesg_state_summary
                 else:
                     return "I-D Exists"
             else:
-                if in_iesg_process  and iesg_state.slug == "dead":
-                    return self.get_state().name +" (IESG: "+iesg_state_summary+")"
+                if iesg_state and iesg_state.slug == "dead":
+                    return state.name + " (IESG: %s)" % iesg_state_summary
                 # Expired/Withdrawn by Submitter/IETF
-                return self.get_state().name
+                return state.name
         else:
-           return self.get_state().name
+           return state.name
 
     def ipr(self):
         """Returns the IPR disclosures against this document (as a queryset over IprDocAlias)."""

ietf/doc/redirect_drafts_urls.py

@@ -0,0 +1,23 @@
+# Copyright The IETF Trust 2007, All Rights Reserved
+
+from django.conf import settings
+from django.conf.urls.defaults import patterns
+
+
+from django.http import HttpResponsePermanentRedirect
+from django.shortcuts import get_object_or_404
+
+from ietf.group.models import Group
+
+urlpatterns = patterns('',
+     (r'^$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/'}),
+     (r'^all/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/all/'}),
+     (r'^rfc/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/all/#rfc'}),
+     (r'^dead/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/all/#expired'}),
+     (r'^current/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/active/'}),
+     (r'^(?P<object_id>\d+)/(related/)?$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/' }),
+     (r'^(?P<name>[^/]+)/(related/)?$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/%(name)s/' }),
+     (r'^wgid/(?P<id>\d+)/$', lambda request, id: HttpResponsePermanentRedirect("/wg/%s/" % get_object_or_404(Group, id=id).acronym)),
+     (r'^wg/(?P<acronym>[^/]+)/$', 'django.views.generic.simple.redirect_to', { 'url': '/wg/%(acronym)s/' }),
+     (r'^all_id(?:_txt)?.html$', 'django.views.generic.simple.redirect_to', { 'url': 'http://www.ietf.org/id/all_id.txt' }),
+)

ietf/doc/redirect_idtracker_urls.py

@@ -0,0 +1,13 @@
+from django.conf.urls.defaults import patterns, url
+from django.views.generic.simple import redirect_to
+
+urlpatterns = patterns('',
+     (r'^help/(?:sub)?state/(?:\d+/)?$', redirect_to, {'url': '/doc/help/state/draft-iesg/' }),
+     (r'^help/evaluation/$', redirect_to, {'url':'http://www.ietf.org/iesg/voting-procedures.html' }),
+     (r'^status/$', redirect_to, {'url':'/doc/iesg/' }),
+     (r'^status/last-call/$', redirect_to, {'url':'/doc/iesg/last-call/' }),
+     (r'^rfc0*(?P<rfc_number>\d+)/$', redirect_to, {'url':'/doc/rfc%(rfc_number)s/' }),
+     (r'^(?P<name>[^/]+)/$', redirect_to, {'url':'/doc/%(name)s/' }),
+     (r'^(?P<name>[^/]+)/comment/\d+/$', redirect_to, {'url':'/doc/%(name)s/history/' }),
+     (r'^$', redirect_to, { 'url': '/doc/'}),
+)