Refined the sql debug view at the bottom of each page. Added a column showing the WHERE clause, as that is quite helpful in working out where a given query is coming from. Added an sql_debug template variable to make it easier to switch between the sql debug view and using the django-debug-toolbar.

 - Legacy-Id: 12225

Author: levkowetz

ietf/context_processors.py

@@ -18,6 +18,12 @@ def debug_mark_queries_from_view(request):
     if settings.DEBUG and request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS:
         from django.db import connection
         for query in connection.queries:
-            query['where'] = 'V'           # V is for 'view'
+            query['loc'] = 'V'           # V is for 'view'
     return context_extras
+
+def sql_debug(request):
+    if settings.DEBUG and request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS:
+        return {'sql_debug': True }
+    else:
+        return {'sql_debug': False }
 

ietf/templates/debug.html

@@ -1,43 +1,49 @@
 {# Copyright The IETF Trust 2015, All Rights Reserved #}
 {% load origin %}{% origin %}
 {% if debug %}
-  {% load debug_filters %}
-  {% load future %}
+  {% if sql_debug %}
+    {% load debug_filters %}
+    {% load future %}
 
-  <div id="debug">
-  <hr>
-    <p>
-      {{ sql_queries|length }} queries ({{ sql_queries|timesum }}s)
-      {% if sql_queries|length != 0 %}
-        <a class="btn btn-default btn-xs"
-           onclick="$('#debug-query-table').toggleClass('hide');">Show</a>
-      {% endif %}
-    </p>
-    <table class="table table-condensed table-striped tablesorter hide" id="debug-query-table">
-      <thead>
-        <tr>
-	  <th data-header="sequence">#</th>
-	  <th data-header="query">SQL</th>
-	  <th data-header="count">Count</th>
-	  <th data-header="where">View/ Templ.</th>
-	  <th data-header="time">Time</th>
-	  <th data-header="acc">Acc.</th>
-        </tr>
-      </thead>
-      <tbody>
-	{% with sql_queries|annotate_sql_queries as sql_query_info %}
-	    {% for query in sql_query_info %}
-	      <tr>
-		<td>{{ forloop.counter }}</td>
-		<td>{{ query.sql|expand_comma|escape }}</td>
-		<td>{{ query.count }}</td>
-		<td>{{ query.where }}</td>
-		<td>{{ query.time }}</td>
-		<td>{{ query.time_accum }}</td>
-	      </tr>
-	    {% endfor %}
-	{% endwith %}
-      </tbody>
-    </table>
-  </div>
+    <div id="debug">
+    <hr>
+      <p>
+	{{ sql_queries|length }} queries ({{ sql_queries|timesum }}s)
+	{% if sql_queries|length != 0 %}
+	  <a class="btn btn-default btn-xs"
+	     onclick="$('#debug-query-table').toggleClass('hide');">Show</a>
+	{% endif %}
+      </p>
+      <table class="table table-condensed table-striped tablesorter hide" id="debug-query-table">
+	<thead>
+	  <tr>
+	    <th data-header="sequence">#</th>
+	    <th data-header="query">SQL</th>
+	    <th data-header="count">Count</th>
+	    <th data-header="where">WHERE</th>
+	    <th data-header="loc">View/ Templ.</th>
+	    <th data-header="time">Time</th>
+	    <th data-header="acc">Acc.</th>
+	  </tr>
+	</thead>
+	<tbody>
+	  {% with sql_queries|annotate_sql_queries as sql_query_info %}
+	      {% for query in sql_query_info %}
+		<tr>
+		  <td>{{ forloop.counter }}</td>
+		  <td>{{ query.sql|expand_comma|escape }}</td>
+		  <td>{{ query.count }}</td>
+		  <td>{{ query.where }}</td>
+		  <td>{{ query.loc }}</td>
+		  <td>{{ query.time }}</td>
+		  <td>{{ query.time_accum }}</td>
+		</tr>
+	      {% endfor %}
+	  {% endwith %}
+	</tbody>
+      </table>
+    </div>
+  {% else %}
+    <div class='text-center text-muted small'>Add 'ietf.context_processors.sql_debug' to settings.TEMPLATE_CONTECT_PROCESSORS to turn on the SQL statement table</div>
+  {% endif %}
 {% endif %}

ietf/utils/templatetags/debug_filters.py

@@ -1,3 +1,6 @@
+import sys
+import sqlparse
+
 from django import template
 
 register = template.Library()
@@ -20,21 +23,52 @@ def expand_comma(value):
     return value.replace(",", ", ")
 
 
+def get_sql_parts(sql):
+    q = {}
+    s = sqlparse.parse(sql)[0]      # assuming there's only one statement
+    q['where'] = None
+    q['from'] = None
+    # use sqlparse to pick out some interesting parts of the statement
+    state = None
+    for e in s:
+        if e.is_whitespace:
+            continue
+        if state == None:
+            if e.is_keyword:
+                key = e.normalized.lower()
+                state = 'value'
+            elif e.is_group and e[0].is_keyword:
+                key = e[0].normalized.lower()
+                val = str(e)
+                state = 'store'
+            else:
+                pass
+        elif state == 'value':
+            val = str(e)
+            state = 'store'
+        else:
+            sys.stderr.write("Unexpected sqlparse iteration state in annotate_sql_queries(): '%s'" % state )
+        if state == 'store':
+            q[key] = val
+            state = None
+    return q
+
 @register.filter()
 def annotate_sql_queries(queries):
     counts  = {}
     timeacc = {}
     for q in queries:
         sql = q['sql']
+        q.update(get_sql_parts(sql))
         if not sql in counts:
             counts[sql] = 0;
         counts[sql] += 1
         if not sql in timeacc:
             timeacc[sql] = 0.0;
         timeacc[sql] += float(q['time'])
     for q in queries:
-        if q.get('where', None) == None:
-            q['where'] = 'T'            # template
+        if q.get('loc', None) == None:
+            q['loc'] = 'T'            # template
         sql = q['sql']
         q['count'] = str(counts[sql])
         q['time_accum'] = "%4.3f" % timeacc[sql]