Basic charting of new revision activity for document search results.

 - Legacy-Id: 11922

Author: levkowetz

debug.py

@@ -36,7 +36,7 @@
 # Number of times to indent output
 # A list is used to force access by reference
 _report_indent = [4]
-_mark = [ timeutils.clock() ]
+_mark = [ timeutils.time() ]
 
 def set_indent(i):
     _report_indent[0] = i
@@ -82,27 +82,29 @@ def wrap(fn, *params,**kwargs):
         return fn
 
 def mark():
-    _mark[0] = timeutils.clock()
+    _mark[0] = timeutils.time()
 
 def lap(s):
-    tau = timeutils.clock() - _mark[0]
-    say(">  %s: %.3fs since mark" % (s, tau))
+    clk = timeutils.time()
+    tau = clk - _mark[0]
+    ts = timeutils.strftime("%H:%M:%S", timeutils.localtime(clk))
+    say("%s: %.3fs since mark: %s" % (ts, tau, s))
 
 def clock(s):
     lap(s)
-    _mark[0] = timeutils.clock()
+    _mark[0] = timeutils.time()
 
 def time(fn):
     """Decorator to print timing information about a function call.
     """
     def wrap(fn, *params,**kwargs):
-        mark = timeutils.clock()
+        mark = timeutils.time()
 
         indent = ' ' * _report_indent[0]
         fc = "%s.%s()" % (fn.__module__, fn.__name__,)
 
         ret = fn(*params,**kwargs)
-        tau = timeutils.clock() - mark
+        tau = timeutils.time() - mark
         sys.stderr.write("%s| %s | %.3fs\n" % (indent, fc, tau))
 
         return ret
@@ -160,6 +162,7 @@ def say(s):
     if debug:
         indent = ' ' * (_report_indent[0])
         sys.stderr.write("%s%s\n" % (indent, s))
+        sys.stderr.flush()
 
 def profile(fn):
     def wrapper(*args, **kwargs):

ietf/bower.json

@@ -8,6 +8,7 @@
     "d3": "~3",
     "font-awesome": "~4",
     "html5shiv": "~3",
+    "highcharts": "~4.2", 
     "jquery": "~1",
     "jquery.cookie": "~1",
     "jquery.tablesorter": "~2",
@@ -31,6 +32,16 @@
     },
     "respond": {
       "main": "dest/respond.min.js"
+    },
+    "highcharts": {
+	"main": [
+	  "highcharts.js",
+	  "highcharts-more.js",
+	  "highstock.js",
+	  "highmaps.js",
+	  "modules/exporting.js",
+	  "modules/map.js"
+	]
     }
   }
 }

ietf/doc/models.py

@@ -420,6 +420,8 @@ def latest_event(self, *args, **filter_args):
         return e[0] if e else None
 
     def canonical_name(self):
+        if hasattr(self, '_canonical_name'):
+            return self._canonical_name
         name = self.name
         if self.type_id == "draft" and self.get_state_slug() == "rfc":
             a = self.docalias_set.filter(name__startswith="rfc")
@@ -441,6 +443,8 @@ def display_name(self):
 
 
     def telechat_date(self, e=None):
+        if hasattr(self, '_telechat_date'):
+            return self._telechat_date
         if not e:
             e = self.latest_event(TelechatDocEvent, type="scheduled_for_telechat")
         return e.telechat_date if e and e.telechat_date and e.telechat_date >= datetime.date.today() else None
@@ -551,6 +555,8 @@ def __unicode__(self):
         return unicode(self.doc.name)
 
     def canonical_name(self):
+        if hasattr(self, '_canonical_name'):
+            return self._canonical_name
         return self.name
 
     def latest_event(self, *args, **kwargs):

ietf/doc/templatetags/ietf_filters.py

@@ -590,3 +590,11 @@ def urlize_html(html, autoescape=False):
 def emailwrap(email):
     email = str(email)
     return mark_safe(email.replace('@', '<wbr>@'))
+
+@register.filter()
+def comma_separated_list(seq, end_word="and"):
+    if len(seq) < 2:
+        return u"".join(seq)
+    else:
+        return u", ".join(seq[:-1]) + u" %s %s"%(end_word, seq[-1])
+        

ietf/doc/urls.py

@@ -33,9 +33,8 @@
 from django.conf.urls import patterns, url, include
 from django.views.generic import RedirectView
 
-from ietf.doc import views_search, views_draft, views_ballot
-from ietf.doc import views_status_change
-from ietf.doc import views_doc
+from ietf.doc import views_search, views_draft, views_ballot, views_status_change, views_doc
+from ietf.doc import views_stats
 
 session_patterns = [
     url(r'^add$', views_doc.add_sessionpresentation),
@@ -54,6 +53,8 @@
     url(r'^start-rfc-status-change/(?P<name>[A-Za-z0-9._+-]*)$', views_status_change.start_rfc_status_change, name='start_rfc_status_change'), 
     url(r'^iesg/(?P<last_call_only>[A-Za-z0-9.-]+/)?$', views_search.drafts_in_iesg_process, name="drafts_in_iesg_process"),
     url(r'^email-aliases/$', views_doc.email_aliases),
+    url(r'^stats/newrevisiondocevent/?$', views_stats.doc_stats_newrevisiondocevent),
+    url(r'^stats/data/newrevisiondocevent/?$', views_stats.doc_stats_data_newrevisiondocevent),
 
     url(r'^all/$', views_search.index_all_drafts, name="index_all_drafts"),
     url(r'^active/$', views_search.index_active_drafts, name="index_active_drafts"),

ietf/doc/utils.py

@@ -3,6 +3,8 @@
 import urllib
 import math
 import datetime
+import hashlib
+import json
 
 from django.conf import settings
 from django.db.models.query import EmptyQuerySet
@@ -583,3 +585,12 @@ def get_ancestors(doc):
         }
     history = history.values()
     return sorted(history, key=lambda x: x['published'])
+
+
+def get_search_cache_key(params):
+    from ietf.doc.views_search import SearchForm
+    fields = set(SearchForm.base_fields) - set(['sort',])
+    kwargs = dict([ (k,v) for (k,v) in params.items() if k in fields ])
+    key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True)).hexdigest()
+    return key
+    

ietf/doc/views_search.py

@@ -30,15 +30,16 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import datetime, re
+import re
+import datetime
 
 from django import forms
 from django.conf import settings
 from django.core.cache import cache
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse as urlreverse
 from django.db.models import Q
-from django.http import Http404, HttpResponseBadRequest, HttpResponse, HttpResponseRedirect
+from django.http import Http404, HttpResponseBadRequest, HttpResponse, HttpResponseRedirect, QueryDict
 from django.shortcuts import render
 from django.utils.cache import _generate_cache_key
 
@@ -49,6 +50,7 @@
     DocEvent, LastCallDocEvent, TelechatDocEvent, IESG_SUBSTATE_TAGS )
 from ietf.doc.expire import expirable_draft
 from ietf.doc.fields import select2_id_doc_name_json
+from ietf.doc.utils import get_search_cache_key
 from ietf.group.models import Group
 from ietf.idindex.index import active_drafts_index_by_group
 from ietf.name.models import DocTagName, DocTypeName, StreamName
@@ -123,9 +125,6 @@ def clean(self):
             q['state'] = q['substate'] = None
         return q
 
-def wrap_value(v):
-    return lambda: v
-
 def fill_in_search_attributes(docs):
     # fill in some attributes for the search results to save some
     # hairy template code and avoid repeated SQL queries
@@ -153,13 +152,13 @@ def fill_in_search_attributes(docs):
     for e in TelechatDocEvent.objects.filter(doc__in=doc_ids, type="scheduled_for_telechat").order_by('-time'):
         if e.doc_id not in seen:
             d = docs_dict[e.doc_id]
-            d.telechat_date = wrap_value(d.telechat_date(e))
+            d._telechat_date = d.telechat_date(e)
             seen.add(e.doc_id)
 
     # misc
     for d in docs:
-        # emulate canonical name which is used by a lot of the utils
-        d.canonical_name = wrap_value(rfc_aliases[d.pk] if d.pk in rfc_aliases else d.name)
+        # set up information for canonical_name() which is used by a lot of the utils
+        d._canonical_name = rfc_aliases[d.pk] if d.pk in rfc_aliases else d.name
 
         if d.rfc_number() != None and d.latest_event_cache["published_rfc"]:
             d.latest_revision_date = d.latest_event_cache["published_rfc"].time
@@ -265,7 +264,7 @@ def retrieve_search_results(form, all_types=False):
     # radio choices
     by = query["by"]
     if by == "author":
-        docs = docs.filter(authors__person__name__icontains=query["author"])
+        docs = docs.filter(authors__person__alias__name__icontains=query["author"])
     elif by == "group":
         docs = docs.filter(group__acronym=query["group"])
     elif by == "area":
@@ -362,7 +361,7 @@ def sort_key(d):
                 d["sort"] = h["key"]
             h["sort_url"] = "?" + d.urlencode()
 
-    return (results, meta)
+    return (results, meta, docs)
 
 
 def get_doc_is_tracked(request, results):
@@ -392,20 +391,27 @@ def search(request):
             get_params['substate'] = request.GET['subState']
 
         form = SearchForm(get_params)
-        if not form.is_valid():
-            return HttpResponseBadRequest("form not valid: %s" % form.errors)
+        key = get_search_cache_key(get_params)
+        info = cache.get(key)
+        if not info:
+            if not form.is_valid():
+                return HttpResponseBadRequest("form not valid: %s" % form.errors)
+
+            info = retrieve_search_results(form)
+            cache.set(key, info)
 
-        results, meta = retrieve_search_results(form)
+        results, meta, qs = info            
         meta['searching'] = True
     else:
         form = SearchForm()
         results = []
         meta = { 'by': None, 'advanced': False, 'searching': False }
+        get_params = QueryDict('')
 
     doc_is_tracked = get_doc_is_tracked(request, results)
 
     return render(request, 'doc/search/search.html', {
-        'form':form, 'docs':results, 'doc_is_tracked':doc_is_tracked, 'meta':meta, },
+        'form':form, 'docs':results, 'doc_is_tracked':doc_is_tracked, 'meta':meta, 'queryargs':get_params.urlencode() },
     )
 
 def frontpage(request):
@@ -580,7 +586,7 @@ def docs_for_ad(request, name):
     form = SearchForm({'by':'ad','ad': ad.id,
                        'rfcs':'on', 'activedrafts':'on', 'olddrafts':'on',
                        'sort': 'status'})
-    results, meta = retrieve_search_results(form, all_types=True)
+    results, meta, _ = retrieve_search_results(form, all_types=True)
     results.sort(key=ad_dashboard_sort_key)
     del meta["headers"][-1]
     #
@@ -594,7 +600,7 @@ def docs_for_ad(request, name):
 def drafts_in_last_call(request):
     lc_state = State.objects.get(type="draft-iesg", slug="lc").pk
     form = SearchForm({'by':'state','state': lc_state, 'rfcs':'on', 'activedrafts':'on'})
-    results, meta = retrieve_search_results(form)
+    results, meta, _ = retrieve_search_results(form)
 
     return render(request, 'doc/drafts_in_last_call.html', {
         'form':form, 'docs':results, 'meta':meta