Merged in [18445] from rjsparks@nostrum.com:

    Provide a management command to inform github backups.
 - Legacy-Id: 18455
Note: SVN reference [18445] has been migrated to Git commit 30ac983

Author: levkowetz

ietf/doc/management/commands/find_github_backup_info.py

@@ -1,45 +1,74 @@
 # Copyright The IETF Trust 2020, All Rights Reserved
 
-from django.core.management.base import BaseCommand
-from django.db.models import F
+
+import github3
+
+from collections import Counter
+from urllib.parse import urlparse
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
 
 from ietf.doc.models import DocExtResource
 from ietf.group.models import GroupExtResource
 from ietf.person.models import PersonExtResource
 
+# TODO: Think more about submodules. This currently will only take top level repos, with the assumption that the clone will include arguments to grab all the submodules. 
+# As a consequence, we might end up pulling more than we need (or that the org or user expected)
+# Make sure this is what we want.
+
 class Command(BaseCommand):
-    help = ('Locate information about gihub repositories to backup')
+    help = ('Locate information about github repositories to backup')
+
+    def add_arguments(self, parser):
+        parser.add_argument('--verbose', dest='verbose', action='store_true', help='Show counts of types of repositories')
 
     def handle(self, *args, **options):
 
-        info_dict = {}
-
-
-        for repo in DocExtResource.objects.filter(name__slug='github_repo'):
-            if not repo.value.endswith('/'):
-                repo.value += '/'
-            if repo not in info_dict:
-                info_dict[repo.value] = []
-            for username in DocExtResource.objects.filter(name__slug='github_username', doc=F('doc')):
-                info_dict[repo.value].push(username.value)
-
-        for repo in GroupExtResource.objects.filter(name__slug='github_repo'):
-            if not repo.value.endswith('/'):
-                repo.value += '/'
-            if repo not in info_dict:
-                info_dict[repo.value] = []
-            for username in GroupExtResource.objects.filter(name__slug='github_username', group=F('group')):
-                info_dict[repo.value].push(username.value)
-
-        for repo in PersonExtResource.objects.filter(name__slug='github_repo'):
-            if not repo.value.endswith('/'):
-                repo.value += '/'
-            if repo not in info_dict:
-                info_dict[repo.value] = []
-            for username in PersonExtResource.objects.filter(name__slug='github_username', person=F('person')):
-                info_dict[repo.value].push(username.value)
-
-        #print (json.dumps(info_dict))
-        # For now, all we need are the repo names
-        for name in info_dict.keys():
-            print(name)
+        if not (hasattr(settings,'GITHUB_BACKUP_API_KEY') and settings.GITHUB_BACKUP_API_KEY):
+            raise CommandError("ERROR: can't find GITHUB_BACKUP_API_KEY") # TODO: at >= py3.1, use returncode
+
+        github = github3.login(token = settings.GITHUB_BACKUP_API_KEY)
+        owners = dict()
+        repos = set()
+
+        for cls in (DocExtResource, GroupExtResource, PersonExtResource):
+            for res in cls.objects.filter(name_id__in=('github_repo','github_org')):
+                path_parts = urlparse(res.value).path.strip('/').split('/')
+                if not path_parts or not path_parts[0]:
+                    continue
+
+                owner = path_parts[0]
+
+                if owner not in owners:
+                    try:
+                        gh_owner = github.user(username=owner)
+                        owners[owner] = gh_owner
+                    except github3.exceptions.NotFoundError:
+                        continue 
+
+                if gh_owner.type in ('User', 'Organization'):
+                    if len(path_parts) > 1:
+                        repo = path_parts[1]
+                        if (owner, repo) not in repos:
+                            try:
+                                _ = github.repository(owner,repo)
+                                repos.add( (owner, repo) )
+                            except github3.exceptions.NotFoundError:
+                                continue
+                    else:
+                        for repo in github.repositories_by(owner):
+                            repos.add( (owner, repo.name) )
+
+        owner_types = Counter([owners[owner].type for owner in owners])
+        if options['verbose']:
+            self.stdout.write("Owners:")
+            for key in owner_types:
+                self.stdout.write("    %s: %s"%(key,owner_types[key]))
+            self.stdout.write("Repositories: %d" % len(repos))
+            for repo in sorted(repos):
+                self.stdout.write("    https://github.com/%s/%s" % repo )
+        else:
+            for repo in sorted(repos):
+                self.stdout.write("%s/%s" % repo )
+

requirements.txt

@@ -27,6 +27,7 @@ django-widget-tweaks>=1.4.2
 docutils>=0.12,!=0.15
 factory-boy>=2.9.0,<3
 Faker>=0.8.8,!=0.8.9,!=0.8.10   # from factory-boy # Faker 0.8.9,0.8.10 sometimes return string names instead of unicode.
+github3.py>=1.2
 hashids>=1.1.0
 html2text>=2019.8.11
 html5lib>=1.0.1