Turned the api.py file into a module. Moved the makeresources management command to the api module. Added some api tests. Added crawling of api files to the test-crawler. Adjusted some resource files discovered by the test suite and test-crawler. Removed a bunch of empty model files.

 - Legacy-Id: 9144

Author: levkowetz

bin/test-crawl

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os, sys, re, datetime, argparse, traceback, tempfile
+import os, sys, re, datetime, argparse, traceback, tempfile, json
 
 # args
 parser = argparse.ArgumentParser(
@@ -51,6 +51,7 @@ if args.url_file:
 
 if not initial_urls:
     initial_urls.append("/")
+    initial_urls.append("/api/v1")
 
 visited = set()
 urls = {} # url -> referrer
@@ -78,8 +79,30 @@ def extract_html_urls(content):
 
         yield url
 
+def extract_tastypie_urls(content):
+    VISIT_OBJECTS = False
+    VISIT_NEXT = False
+    data = json.loads(content)
+    for item in data:
+        if type(data[item]) is dict:
+            if "list_endpoint" in data[item]:
+                uri = data[item]["list_endpoint"]
+                yield uri
+    if VISIT_NEXT:
+        if "meta" in data and "next" in data["meta"]:
+            uri = data["meta"]["next"]
+            if uri != None:
+                yield uri
+    if VISIT_OBJECTS:
+        if "objects" in data:
+            object_list = data["objects"]
+            for i in range(len(object_list)):
+                if "resource_uri" in object_list[i]:
+                    uri = object_list[i]["resource_uri"]
+                    yield uri
+
 django.setup()
-client = django.test.Client()
+client = django.test.Client(Accept='text/html,text/plain,application/json')
 
 for url in initial_urls:
     urls[url] = "[initial]"
@@ -150,8 +173,19 @@ while urls:
                     log("=============")
                     log(traceback.format_exc())
                     log("=============")
+            elif ctype == "application/json":
+                try:
+                    for u in extract_tastypie_urls(r.content):
+                        if u not in visited and u not in urls:
+                            urls[u] = url
+                            referrers[u] = url
+                except:
+                    log("error extracting urls from %s" % url)
+                    log("=============")
+                    log(traceback.format_exc())
+                    log("=============")
         else:
-            tags.append(u"FAIL (from %s)" % referrer)
+            tags.append(u"FAIL for %s\n   (from %s)" % (url, referrer))
             errors += 1
 
         if elapsed.total_seconds() > slow_threshold:
@@ -164,9 +198,9 @@ while urls:
         sec = acc_secs % 60
 
         if (len(visited) % 100) == 1:
-            log("\nElapsed  Visited  Queue   Code  Time  Url  ...  Notes")
+            log("\nElapsed  Visited  Queue Code   Time  Url  ...  Notes")
 
-        log("%2d:%02d:%02d %7d %6d  %s %.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags)))
+        log("%2d:%02d:%02d %7d %6d  %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags)))
 
 logfile.close()
 sys.stderr.write("Output written to %s\n\n" % logfile.name)

ietf/api.py ietf/api/__init__.pyietf/api.py renamed to ietf/api/__init__.py

@@ -0,0 +1,62 @@
+import os
+import sys
+import json
+
+from django.test import Client
+from django.conf import settings
+from django.utils.importlib import import_module
+from django.db import models
+
+from tastypie.test import ResourceTestCase
+
+import debug                            # pyflakes:ignore
+
+OMITTED_APPS = (
+    'ietf.secr.meetings',
+    'ietf.secr.proceedings',
+    'ietf.ipr',
+)
+
+class TastypieApiTestCase(ResourceTestCase):
+    def __init__(self, *args, **kwargs):
+        self.apps = {}
+        for app_name in settings.INSTALLED_APPS:
+            if app_name.startswith('ietf') and not app_name in OMITTED_APPS:
+                app = import_module(app_name)
+                name = app_name.split('.',1)[-1]
+                models_path = os.path.join(os.path.dirname(app.__file__), "models.py")
+                if os.path.exists(models_path):
+                    self.apps[name] = app
+        super(ResourceTestCase, self).__init__(*args, **kwargs)
+
+    def test_api_top_level(self):
+        client = Client(Accept='application/json')
+        r = client.get("/api/v1/")
+        self.assertValidJSONResponse(r)        
+        resource_list = json.loads(r.content)
+
+        for name in self.apps:
+            if not name in self.apps:
+                sys.stderr.write("Expected a REST API resource for %s, but didn't find one\n" % name)
+
+        for name in self.apps:
+            self.assertIn(name, resource_list,
+                        "Expected a REST API resource for %s, but didn't find one" % name)
+
+    def test_all_model_resources_exist(self):
+        client = Client(Accept='application/json')
+        r = client.get("/api/v1")
+        top = json.loads(r.content)
+        for name in self.apps:
+            app = self.apps[name]
+            self.assertEqual("/api/v1/%s/"%name, top[name]["list_endpoint"])
+            r = client.get(top[name]["list_endpoint"])
+            self.assertValidJSONResponse(r)
+            app_resources = json.loads(r.content)
+            model_list = models.get_models(app.models)
+            for model in model_list:
+                if not model._meta.model_name in app_resources.keys():
+                    #print("There doesn't seem to be any resource for model %s.models.%s"%(app.__name__,model.__name__,))
+                    self.assertIn(model._meta.model_name, app_resources.keys(),
+                        "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,))
+

…ils/management/commands/makeresources.py …api/management/commands/makeresources.pyietf/utils/management/commands/makeresources.py renamed to ietf/api/management/commands/makeresources.py ietf/utils/management/commands/makeresources.py renamed to ietf/api/management/commands/makeresources.py

@@ -5,8 +5,8 @@
 
 from ietf import api
 
-from ietf.meeting.models import *       # pyflakes:ignore
-
+from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session,
+                                TimeSlot, ScheduledSession, SessionPresentation )
 
 from ietf.name.resources import MeetingTypeNameResource
 class MeetingResource(ModelResource):
@@ -38,7 +38,7 @@ class ResourceAssociationResource(ModelResource):
     name = ToOneField(RoomResourceNameResource, 'name')
     class Meta:
         queryset = ResourceAssociation.objects.all()
-        #resource_name = 'resourceassociation'
+        resource_name = 'resourceassociation'
         filtering = { 
             "id": ALL,
             "icon": ALL,
@@ -144,7 +144,7 @@ class TimeSlotResource(ModelResource):
     type = ToOneField(TimeSlotTypeNameResource, 'type')
     location = ToOneField(RoomResource, 'location', null=True)
     sessions = ToManyField(SessionResource, 'sessions', null=True)
-    duration = TimedeltaField()
+    duration = api.TimedeltaField()
     class Meta:
         queryset = TimeSlot.objects.all()
         #resource_name = 'timeslot'
@@ -183,3 +183,20 @@ class Meta:
         }
 api.meeting.register(ScheduledSessionResource())
 
+
+
+from ietf.doc.resources import DocumentResource
+class SessionPresentationResource(ModelResource):
+    session          = ToOneField(SessionResource, 'session')
+    document         = ToOneField(DocumentResource, 'document')
+    class Meta:
+        queryset = SessionPresentation.objects.all()
+        #resource_name = 'sessionpresentation'
+        filtering = { 
+            "id": ALL,
+            "rev": ALL,
+            "session": ALL_WITH_RELATIONS,
+            "document": ALL_WITH_RELATIONS,
+        }
+api.meeting.register(SessionPresentationResource())
+

ietf/api/tests.py

@@ -223,7 +223,7 @@ class Meta:
 class RoomResourceNameResource(ModelResource):
     class Meta:
         queryset = RoomResourceName.objects.all()
-        #resource_name = 'roomresourcename'
+        resource_name = 'roomresourcename' # Needed because tastypie otherwise removes 'resource' from the name
         filtering = { 
             "slug": ALL,
             "name": ALL,
@@ -299,3 +299,44 @@ class Meta:
         }
 api.name.register(NomineePositionStateNameResource())
 
+
+
+class IprDisclosureStateNameResource(ModelResource):
+    class Meta:
+        queryset = IprDisclosureStateName.objects.all()
+        #resource_name = 'iprdisclosurestatename'
+        filtering = { 
+            "slug": ALL,
+            "name": ALL,
+            "desc": ALL,
+            "used": ALL,
+            "order": ALL,
+        }
+api.name.register(IprDisclosureStateNameResource())
+
+class IprEventTypeNameResource(ModelResource):
+    class Meta:
+        queryset = IprEventTypeName.objects.all()
+        #resource_name = 'ipreventtypename'
+        filtering = { 
+            "slug": ALL,
+            "name": ALL,
+            "desc": ALL,
+            "used": ALL,
+            "order": ALL,
+        }
+api.name.register(IprEventTypeNameResource())
+
+class IprLicenseTypeNameResource(ModelResource):
+    class Meta:
+        queryset = IprLicenseTypeName.objects.all()
+        #resource_name = 'iprlicensetypename'
+        filtering = { 
+            "slug": ALL,
+            "name": ALL,
+            "desc": ALL,
+            "used": ALL,
+            "order": ALL,
+        }
+api.name.register(IprLicenseTypeNameResource())
+