Add branch from trunk @r12628 for the author statistics project, add document statistics page with the first statistics with the number of authors per document

 - Legacy-Id: 12629

Author: OleLaursen

ietf/static/ietf/css/ietf.css

@@ -568,6 +568,11 @@ table.simple-table td:last-child {
   width: 7em;
 }
 
+.popover .docname {
+  padding-left: 1em;
+  text-indent: -1em;
+}
+
 .stats-time-graph {
   height: 15em;
 }

ietf/static/ietf/js/document-stats.js

@@ -0,0 +1,34 @@
+$(document).ready(function () {
+    if (window.chartConf) {
+        var chart = Highcharts.chart('chart', window.chartConf);
+    }
+
+    $(".popover-docnames").each(function () {
+        var stdNameRegExp = new RegExp("^(rfc|bcp|fyi|std)[0-9]+$", 'i');
+
+        var html = [];
+        $.each(($(this).data("docnames") || "").split(" "), function (i, docname) {
+            if (!$.trim(docname))
+                return;
+
+            var displayName = docname;
+
+            if (stdNameRegExp.test(docname))
+                displayName = docname.slice(0, 3).toUpperCase() + " " + docname.slice(3);
+
+            html.push('<div class="docname"><a href="/doc/' + docname + '/">' + displayName + '</a></div>');
+        });
+
+        if ($(this).data("sliced"))
+            html.push('<div class="text-center">&hellip;</div>');
+
+        $(this).popover({
+            trigger: "focus",
+            template: '<div class="popover" role="tooltip"><div class="arrow"></div><h3 class="popover-title"></h3><div class="popover-content"></div></div>',
+            content: html.join(""),
+            html: true
+        }).on("click", function (e) {
+            e.preventDefault();
+        });
+    });
+});

ietf/stats/urls.py

@@ -5,5 +5,6 @@
 
 urlpatterns = patterns('',
     url("^$", ietf.stats.views.stats_index),
+    url("^document/(?:(?P<stats_type>authors|pages|format|spectech)/)?(?:(?P<document_state>all|rfc|draft)/)?$", ietf.stats.views.document_stats),
     url("^review/(?:(?P<stats_type>completion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, ietf.stats.views.review_stats),
 )

ietf/stats/views.py

@@ -1,9 +1,12 @@
 import datetime, itertools, json, calendar
+from collections import defaultdict
 
 from django.shortcuts import render
 from django.contrib.auth.decorators import login_required
 from django.core.urlresolvers import reverse as urlreverse
 from django.http import HttpResponseRedirect, HttpResponseForbidden
+from django.db.models import Count
+from django.utils.safestring import mark_safe
 
 import dateutil.relativedelta
 
@@ -15,11 +18,116 @@
 from ietf.group.models import Role, Group
 from ietf.person.models import Person
 from ietf.name.models import ReviewRequestStateName, ReviewResultName
+from ietf.doc.models import Document
 from ietf.ietfauth.utils import has_role
 
 def stats_index(request):
     return render(request, "stats/index.html")
 
+def generate_query_string(query_dict, overrides):
+    query_part = u""
+
+    if query_dict or overrides:
+        d = query_dict.copy()
+        for k, v in overrides.iteritems():
+            if type(v) in (list, tuple):
+                if not v:
+                    if k in d:
+                        del d[k]
+                else:
+                    d.setlist(k, v)
+            else:
+                if v is None or v == u"":
+                    if k in d:
+                        del d[k]
+                else:
+                    d[k] = v
+
+        if d:
+            query_part = u"?" + d.urlencode()
+
+    return query_part
+
+
+def document_stats(request, stats_type=None, document_state=None):
+    def build_document_stats_url(stats_type_override=Ellipsis, document_state_override=Ellipsis, get_overrides={}):
+        kwargs = {
+            "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override,
+            "document_state": document_state if document_state_override is Ellipsis else document_state_override,
+        }
+
+        return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides)
+
+    # statistics type - one of the tables or the chart
+    possible_stats_types = [
+        ("authors", "Number of authors"),
+#        ("pages", "Pages"),
+#        ("format", "Format"),
+#        ("spectech", "Specification techniques"),
+    ]
+
+    possible_stats_types = [ (slug, label, build_document_stats_url(stats_type_override=slug))
+                             for slug, label in possible_stats_types ]
+
+    if not stats_type:
+        return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_stats_types[0][0]))
+
+    possible_document_states = [
+        ("all", "All"),
+        ("rfc", "RFCs"),
+        ("draft", "Drafts (not published as RFC)"),
+    ]
+
+    possible_document_states = [ (slug, label, build_document_stats_url(document_state_override=slug))
+                                for slug, label in possible_document_states ]
+
+    if not document_state:
+        return HttpResponseRedirect(build_document_stats_url(document_state_override=possible_document_states[0][0]))
+    
+
+    # filter documents
+    doc_qs = Document.objects.filter(type="draft")
+
+    if document_state == "rfc":
+        doc_qs = doc_qs.filter(states__type="draft", states__slug="rfc")
+    elif document_state == "draft":
+        doc_qs = doc_qs.exclude(states__type="draft", states__slug="rfc")
+
+    chart_data = []
+    table_data = []
+    stats_title = ""
+
+    if stats_type == "authors":
+        stats_title = "Number of authors for each document"
+
+        groups = defaultdict(list)
+
+        for name, author_count in doc_qs.values_list("name").annotate(Count("authors")).iterator():
+            groups[author_count].append(name)
+
+        total_docs = sum(len(names) for author_count, names in groups.iteritems())
+
+        series_data = []
+        for author_count, names in sorted(groups.iteritems(), key=lambda t: t[0]):
+            series_data.append((author_count, len(names) * 100.0 / total_docs))
+            table_data.append((author_count, names))
+
+        chart_data.append({
+            "data": series_data,
+            "name": "Percentage of documents",
+        })
+
+
+    return render(request, "stats/document_stats.html", {
+        "chart_data": mark_safe(json.dumps(chart_data)),
+        "table_data": table_data,
+        "stats_title": stats_title,
+        "possible_stats_types": possible_stats_types,
+        "stats_type": stats_type,
+        "possible_document_states": possible_document_states,
+        "document_state": document_state,
+    })
+
 @login_required
 def review_stats(request, stats_type=None, acronym=None):
     # This view is a bit complex because we want to show a bunch of
@@ -39,29 +147,7 @@ def build_review_stats_url(stats_type_override=Ellipsis, acronym_override=Ellips
         if acr:
             kwargs["acronym"] = acr
 
-        base_url = urlreverse(review_stats, kwargs=kwargs)
-        query_part = u""
-
-        if request.GET or get_overrides:
-            d = request.GET.copy()
-            for k, v in get_overrides.iteritems():
-                if type(v) in (list, tuple):
-                    if not v:
-                        if k in d:
-                            del d[k]
-                    else:
-                        d.setlist(k, v)
-                else:
-                    if v is None or v == u"":
-                        if k in d:
-                            del d[k]
-                    else:
-                        d[k] = v
-
-            if d:
-                query_part = u"?" + d.urlencode()
-            
-        return base_url + query_part
+        return urlreverse(review_stats, kwargs=kwargs) + generate_query_string(request.GET, get_overrides)
 
     def get_choice(get_parameter, possible_choices, multiple=False):
         values = request.GET.getlist(get_parameter)

ietf/templates/stats/document_stats.html

@@ -0,0 +1,48 @@
+{% extends "base.html" %}
+
+{% load origin %}
+
+{% load ietf_filters staticfiles bootstrap3 %}
+
+{% block title %}{{ stats_title }}{% endblock %}
+
+{% block pagehead %}
+  <link rel="stylesheet" href="{% static 'bootstrap-datepicker/css/bootstrap-datepicker3.min.css' %}">
+{% endblock %}
+
+{% block content %}
+  {% origin %}
+
+  <h1>Document statistics</h1>
+
+  <div class="stats-options well">
+    <div>
+      Show:
+      <div class="btn-group">
+        {% for slug, label, url in possible_stats_types %}
+          <a class="btn btn-default {% if slug == stats_type %}active{% endif %}" href="{{ url }}">{{ label }}</a>
+        {% endfor %}
+      </div>
+    </div>
+
+    <div>
+      Document types:
+      <div class="btn-group">
+        {% for slug, label, url in possible_document_states %}
+          <a class="btn btn-default {% if slug == document_state %}active{% endif %}" href="{{ url }}">{{ label }}</a>
+        {% endfor %}
+      </div>
+    </div>
+  </div>
+
+  {% if stats_type == "authors" %}
+    {% include "stats/document_stats_authors.html" %}
+  {% endif %}
+{% endblock %}
+
+{% block js %}
+  <script src="{% static 'highcharts/highcharts.js' %}"></script>
+  <script src="{% static 'highcharts/modules/exporting.js' %}"></script>
+  <script src="{% static 'highcharts/modules/offline-exporting.js' %}"></script>
+  <script src="{% static 'ietf/js/document-stats.js' %}"></script>
+{% endblock %}

ietf/templates/stats/document_stats_authors.html

@@ -0,0 +1,71 @@
+<h3>{{ stats_title }}</h3>
+
+<div id="chart"></div>
+
+<script>
+ var chartConf = {
+     chart: {
+         type: 'column'
+     },
+     title: {
+         text: '{{ stats_title|escapejs }}'
+     },
+     xAxis: {
+         tickInterval: 1,
+         title: {
+             text: 'Number of authors'
+         }
+     },
+     yAxis: {
+         title: {
+             text: 'Percentage of documents'
+         },
+         labels: {
+             formatter: function () {
+                 return this.value + '%';
+             }
+         }
+     },
+     legend: {
+         enabled: false,
+     },
+     tooltip: {
+         formatter: function () {
+             var s = '<b>' + this.x + ' ' + (this.x == 1 ? "author" : 'authors') + '</b>';
+             console.log(this.points)
+
+             $.each(this.points, function () {
+                 s += '<br/>' + this.series.name + ': ' +
+                      this.y.toFixed(1) + '%';
+             });
+
+             return s;
+         },
+         shared: true
+     },
+     series: {{ chart_data }}
+ };
+</script>
+
+<h3>Data</h3>
+
+<table class="table table-condensed">
+  <thead>
+    <tr>
+      <th>Authors</th>
+      <th>Documents</th>
+    </tr>
+  </thead>
+  <tbody>
+    {% for author_count, names in table_data %}
+      <tr>
+        <td>{{ author_count }}</td>
+        <td><a class="popover-docnames"
+               href=""
+               data-docnames="{% for n in names|slice:":20" %}{{ n }}{% if not forloop.last %} {% endif %}{% endfor %}"
+               data-sliced="{% if names|length > 20 %}1{% endif %}"
+            >{{ names|length }}</a></td>
+      </tr>
+    {% endfor %}
+  </tbody>
+</table>

ietf/templates/stats/index.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 
-{% load origin %}{% origin %}
+{% load origin %}
 
 {% load ietf_filters staticfiles bootstrap3 %}
 
@@ -9,9 +9,8 @@
 
   <h1>{% block title %}Statistics{% endblock %}</h1>
 
-  <p>Currently, there are statistics for:</p>
-
   <ul>
+    <li><a href="{% url "ietf.stats.views.document_stats" %}">Documents (number of authors, size, formats used)</a></li>
     <li><a rel="nofollow" href="{% url "ietf.stats.views.review_stats" %}">Reviews in review teams</a> (requires login)</li>
   </ul>