Merged in [8970] from olau@iola.dk:

    Summary: Enable support for a custom settings file for the test crawl
- and add a simple settings file that enables caching to speed up the
crawl (shaves ~35% off most pages AFAICT)
 - Legacy-Id: 8984
Note: SVN reference [8970] has been migrated to Git commit 8d4e1a8

Author: levkowetz

bin/test-crawl

@@ -2,13 +2,30 @@
 
 import os, sys, re, datetime, argparse, traceback, tempfile
 
+# args
+parser = argparse.ArgumentParser(
+    description="""Perform a test crawl of the project. For each found URL, the HTTP
+    response status is printed. If it's not OK/redirect, FAIL is
+    printed - in case of errors, a stacktrace is also included.""")
+parser.add_argument('urls', metavar='URL', nargs='*',
+                    help='One or more URLs to start the crawl from')
+parser.add_argument('--urls', '-u', dest='url_file',
+                    help='file with URLs to start the crawl from')
+parser.add_argument('--slow', dest='slow_threshold', type=float, default=1.0,
+                    help='responses taking longer than this (in seconds) results in SLOW being printed')
+parser.add_argument('--settings', dest='settings', help='custom settings file')
+
+args = parser.parse_args()
+
 # boilerplate
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
 sys.path = [ basedir ] + sys.path
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings")
+
+settings_module = args.settings or "ietf.settings"
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_module)
 
 import django.test
-from django.conf import settings
 
 # prevent memory from leaking when settings.DEBUG=True
 from django.db import connection
@@ -19,20 +36,6 @@ connection.queries = DontSaveQueries()
 
 MAX_URL_LENGTH = 500
 
-# args
-parser = argparse.ArgumentParser(
-    description="""Perform a test crawl of the project. For each found URL, the HTTP
-    response status is printed. If it's not OK/redirect, FAIL is
-    printed - in case of errors, a stacktrace is also included.""")
-parser.add_argument('urls', metavar='URL', nargs='*',
-                    help='One or more URLs to start the crawl from')
-parser.add_argument('--urls', '-u', dest='url_file',
-                    help='file with URLs to start the crawl from')
-parser.add_argument('--slow', dest='slow_threshold', type=float, default=1.0,
-                    help='responses taking longer than this (in seconds) results in SLOW being printed')
-
-args = parser.parse_args()
-
 slow_threshold = args.slow_threshold
 
 initial_urls = []

ietf/settings_testcrawl.py

@@ -0,0 +1,22 @@
+# Standard settings except we enable caching like in the production
+# environment, this is useful for speeding up the test crawl, try for
+# instance:
+#
+#   bin/test-crawl --settings=ietf.settings_testcrawl
+#
+
+from settings import *                  # pyflakes:ignore
+
+TEMPLATE_LOADERS = (
+    ('django.template.loaders.cached.Loader', (
+        'django.template.loaders.filesystem.Loader',
+        'django.template.loaders.app_directories.Loader',
+    )),
+    'ietf.dbtemplate.template.Loader',
+)
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+    }
+}