Add many-to-many field with formal languages to Document and add formal

language statistics
 - Legacy-Id: 12658

Author: OleLaursen

ietf/doc/migrations/0020_auto_20170112_0753.py

@@ -7,6 +7,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
+        ('name', '0017_formallanguagename'),
         ('doc', '0019_auto_20161207_1036'),
     ]
 
@@ -21,4 +22,14 @@ class Migration(migrations.Migration):
             name='words',
             field=models.IntegerField(null=True, blank=True),
         ),
+        migrations.AddField(
+            model_name='dochistory',
+            name='formal_languages',
+            field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True),
+        ),
+        migrations.AddField(
+            model_name='document',
+            name='formal_languages',
+            field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True),
+        ),
     ]

ietf/doc/models.py

@@ -15,7 +15,7 @@
 
 from ietf.group.models import Group
 from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName,
-    DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName )
+    DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, FormalLanguageName )
 from ietf.person.models import Email, Person
 from ietf.utils.admin import admin_link
 
@@ -76,6 +76,7 @@ class DocumentInfo(models.Model):
     rev = models.CharField(verbose_name="revision", max_length=16, blank=True)
     pages = models.IntegerField(blank=True, null=True)
     words = models.IntegerField(blank=True, null=True)
+    formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document")
     order = models.IntegerField(default=1, blank=True) # This is probably obviated by SessionPresentaion.order
     intended_std_level = models.ForeignKey(IntendedStdLevelName, verbose_name="Intended standardization level", blank=True, null=True)
     std_level = models.ForeignKey(StdLevelName, verbose_name="Standardization level", blank=True, null=True)

ietf/name/admin.py

@@ -3,7 +3,7 @@
 from ietf.name.models import (
     BallotPositionName, ConstraintName, DBTemplateTypeName, DocRelationshipName,
     DocReminderTypeName, DocTagName, DocTypeName, DraftSubmissionStateName,
-    FeedbackTypeName, GroupMilestoneStateName, GroupStateName, GroupTypeName,
+    FeedbackTypeName, FormalLanguageName, GroupMilestoneStateName, GroupStateName, GroupTypeName,
     IntendedStdLevelName, IprDisclosureStateName, IprEventTypeName, IprLicenseTypeName,
     LiaisonStatementEventTypeName, LiaisonStatementPurposeName, LiaisonStatementState,
     LiaisonStatementTagName, MeetingTypeName, NomineePositionStateName,
@@ -32,6 +32,7 @@ class GroupTypeNameAdmin(NameAdmin):
 admin.site.register(DocReminderTypeName, NameAdmin)
 admin.site.register(DocTagName, NameAdmin)
 admin.site.register(DraftSubmissionStateName, NameAdmin)
+admin.site.register(FormalLanguageName, NameAdmin)
 admin.site.register(FeedbackTypeName, NameAdmin)
 admin.site.register(GroupMilestoneStateName, NameAdmin)
 admin.site.register(GroupStateName, NameAdmin)

ietf/name/migrations/0017_formallanguagename.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name', '0016_auto_20161013_1010'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='FormalLanguageName',
+            fields=[
+                ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)),
+                ('name', models.CharField(max_length=255)),
+                ('desc', models.TextField(blank=True)),
+                ('used', models.BooleanField(default=True)),
+                ('order', models.IntegerField(default=0)),
+            ],
+            options={
+                'ordering': ['order', 'name'],
+                'abstract': False,
+            },
+        ),
+    ]

ietf/name/migrations/0018_add_formlang_names.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+def insert_initial_formal_language_names(apps, schema_editor):
+    FormalLanguageName = apps.get_model("name", "FormalLanguageName")
+    FormalLanguageName.objects.get_or_create(slug="abnf", name="ABNF", desc="Augmented Backus-Naur Form", order=1)
+    FormalLanguageName.objects.get_or_create(slug="asn1", name="ASN.1", desc="Abstract Syntax Notation One", order=2)
+    FormalLanguageName.objects.get_or_create(slug="cbor", name="CBOR", desc="Concise Binary Object Representation", order=3)
+    FormalLanguageName.objects.get_or_create(slug="ccode", name="C Code", desc="Code in the C Programming Language", order=4)
+    FormalLanguageName.objects.get_or_create(slug="json", name="JSON", desc="Javascript Object Notation", order=5)
+    FormalLanguageName.objects.get_or_create(slug="xml", name="XML", desc="Extensible Markup Language", order=6)
+
+def noop(apps, schema_editor):
+    pass
+    
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name', '0017_formallanguagename'),
+    ]
+
+    operations = [
+        migrations.RunPython(insert_initial_formal_language_names, noop)
+    ]

ietf/name/models.py

@@ -46,6 +46,8 @@ class StdLevelName(NameModel):
 class IntendedStdLevelName(NameModel):
     """Proposed Standard, (Draft Standard), Internet Standard, Experimental,
     Informational, Best Current Practice, Historic, ..."""
+class FormalLanguageName(NameModel):
+    """ABNF, ASN.1, C code, CBOR, JSON, XML, ..."""
 class DocReminderTypeName(NameModel):
     "Stream state"
 class BallotPositionName(NameModel):

ietf/stats/backfill_data.py

@@ -14,20 +14,24 @@
 from django.conf import settings
 
 from ietf.doc.models import Document
+from ietf.name.models import FormalLanguageName
 from ietf.utils.draft import Draft
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--document", help="specific document name")
 parser.add_argument("--words", action="store_true", help="fill in word count")
+parser.add_argument("--formlang", action="store_true", help="fill in formal languages")
 args = parser.parse_args()
 
+formal_language_dict = { l.pk: l for l in FormalLanguageName.objects.all() }
+
 
 docs_qs = Document.objects.filter(type="draft")
 
 if args.document:
     docs_qs = docs_qs.filter(docalias__name=args.document)
 
-for doc in docs_qs.prefetch_related("docalias_set"):
+for doc in docs_qs.prefetch_related("docalias_set", "formal_languages"):
     canonical_name = doc.name
     for n in doc.docalias_set.all():
         if n.name.startswith("rfc"):
@@ -45,14 +49,33 @@
     with open(path, 'r') as f:
         d = Draft(f.read(), path)
 
+        updated = False
+
         updates = {}
 
         if args.words:
             words = d.get_wordcount()
             if words != doc.words:
                 updates["words"] = words
 
+        if args.formlang:
+            langs = d.get_formal_languages()
+
+            new_formal_languages = set(formal_language_dict[l] for l in langs)
+            old_formal_languages = set(doc.formal_languages.all())
+
+            if new_formal_languages != old_formal_languages:
+                for l in new_formal_languages - old_formal_languages:
+                    doc.formal_languages.add(l)
+                    updated = True
+                for l in old_formal_languages - new_formal_languages:
+                    doc.formal_languages.remove(l)
+                    updated = True
+
         if updates:
             Document.objects.filter(pk=doc.pk).update(**updates)
+            updated = True
+
+        if updated:
             print "updated", canonical_name
 

ietf/stats/views.py

@@ -134,6 +134,8 @@ def generate_canonical_names(docalias_qs):
     stats_title = ""
     bin_size = 1
 
+    total_docs = docalias_qs.count()
+
     if stats_type == "authors":
         stats_title = "Number of authors for each {}".format(doc_label)
 
@@ -142,8 +144,6 @@ def generate_canonical_names(docalias_qs):
         for name, author_count in generate_canonical_names(docalias_qs.values_list("name").annotate(Count("document__authors"))):
             bins[author_count].append(name)
 
-        total_docs = sum(len(names) for author_count, names in bins.iteritems())
-
         series_data = []
         for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]):
             percentage = len(names) * 100.0 / total_docs
@@ -163,8 +163,6 @@ def generate_canonical_names(docalias_qs):
         for name, pages in generate_canonical_names(docalias_qs.values_list("name", "document__pages")):
             bins[pages].append(name)
 
-        total_docs = sum(len(names) for pages, names in bins.iteritems())
-
         series_data = []
         for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]):
             percentage = len(names) * 100.0 / total_docs
@@ -187,8 +185,6 @@ def generate_canonical_names(docalias_qs):
         for name, words in generate_canonical_names(docalias_qs.values_list("name", "document__words")):
             bins[put_into_bin(words, bin_size)].append(name)
 
-        total_docs = sum(len(names) for words, names in bins.iteritems())
-
         series_data = []
         for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]):
             percentage = len(names) * 100.0 / total_docs
@@ -203,7 +199,7 @@ def generate_canonical_names(docalias_qs):
         })
 
     elif stats_type == "format":
-        stats_title = "Formats for each {}".format(doc_label)
+        stats_title = "Submission formats for each {}".format(doc_label)
 
         bins = defaultdict(list)
 
@@ -244,8 +240,6 @@ def generate_canonical_names(docalias_qs):
             if canonical_name:
                 bins[ext.upper()].append(canonical_name)
 
-        total_docs = sum(len(names) for fmt, names in bins.iteritems())
-
         series_data = []
         for fmt, names in sorted(bins.iteritems(), key=lambda t: t[0]):
             percentage = len(names) * 100.0 / total_docs
@@ -258,6 +252,27 @@ def generate_canonical_names(docalias_qs):
             "animation": False,
         })
 
+    elif stats_type == "formlang":
+        stats_title = "Formal languages used for each {}".format(doc_label)
+
+        bins = defaultdict(list)
+
+        for name, formal_language_name in generate_canonical_names(docalias_qs.values_list("name", "document__formal_languages__name")):
+            bins[formal_language_name].append(name)
+
+        series_data = []
+        for formal_language, names in sorted(bins.iteritems(), key=lambda t: t[0]):
+            percentage = len(names) * 100.0 / total_docs
+            if formal_language is not None:
+                series_data.append((formal_language, len(names)))
+                table_data.append((formal_language, percentage, names))
+
+        chart_data.append({
+            "data": series_data,
+            "animation": False,
+        })
+
+        
     return render(request, "stats/document_stats.html", {
         "chart_data": mark_safe(json.dumps(chart_data)),
         "table_data": table_data,

ietf/templates/stats/document_stats_formlang.html

@@ -0,0 +1,60 @@
+<h3>{{ stats_title }}</h3>
+
+<div id="chart"></div>
+
+<script>
+ var chartConf = {
+     chart: {
+         type: 'column'
+     },
+     title: {
+         text: '{{ stats_title|escapejs }}'
+     },
+     xAxis: {
+         type: "category",
+         title: {
+             text: 'Formal language'
+         }
+     },
+     yAxis: {
+         title: {
+             text: 'Number of {{ doc_label }}s'
+         }
+     },
+     tooltip: {
+         formatter: function () {
+             console.log(this);
+             var s = '<b>' + this.points[0].key + '</b>';
+
+             $.each(this.points, function () {
+                 s += '<br/>' + chartConf.yAxis.title.text + ': ' + this.y;
+             });
+
+             return s;
+         },
+         shared: true
+     },
+     series: {{ chart_data }}
+ };
+</script>
+
+<h3>Data</h3>
+
+<table class="table table-condensed stats-data">
+  <thead>
+    <tr>
+      <th>Formal language</th>
+      <th>Percentage of {{ doc_label }}s</th>
+      <th>{{ doc_label|capfirst }}s</th>
+    </tr>
+  </thead>
+  <tbody>
+    {% for formal_language, percentage, names in table_data %}
+      <tr>
+        <td>{{ formal_language }}</td>
+        <td>{{ percentage|floatformat:2 }}%</td>
+        <td>{% include "stats/includes/docnames_cell.html" %}</td>
+      </tr>
+    {% endfor %}
+  </tbody>
+</table>