test: run tests with external blobstore mgmt

ietf/settings_test.py

@@ -106,16 +106,24 @@ def tempdir_with_cleanup(**kwargs):
     },
 }
 
+# Configure storages for the blob store - use env settings if present. See the --no-manage-blobstore test option.
+_blob_store_endpoint_url = os.environ.get("DATATRACKER_BLOB_STORE_ENDPOINT_URL", "http://blobstore:9000")
+_blob_store_access_key = os.environ.get("DATATRACKER_BLOB_STORE_ACCESS_KEY", "minio_root")
+_blob_store_secret_key = os.environ.get("DATATRACKER_BLOB_STORE_SECRET_KEY", "minio_pass")
+_blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "test-")
+_blob_store_enable_profiling = (
+    os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true"
+)
 for storagename in MORE_STORAGE_NAMES:
     STORAGES[storagename] = {
         "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
         "OPTIONS": dict(
-            endpoint_url="http://blobstore:9000",
-            access_key="minio_root",
-            secret_key="minio_pass",
+            endpoint_url=_blob_store_endpoint_url,
+            access_key=_blob_store_access_key,
+            secret_key=_blob_store_secret_key,
             security_token=None,
             client_config=boto3.session.Config(signature_version="s3v4"),
-            verify=False,
-            bucket_name=f"test-{storagename}",
+            bucket_name=f"{_blob_store_bucket_prefix}{storagename}",
+            ietf_log_blob_timing=_blob_store_enable_profiling,
         ),
     }

ietf/utils/test_runner.py

@@ -723,9 +723,25 @@ def add_arguments(cls, parser):
         parser.add_argument('--rerun-until-failure',
             action='store_true', dest='rerun', default=False,
             help='Run the indicated tests in a loop until a failure occurs. ' )
-
-    def __init__(self, ignore_lower_coverage=False, skip_coverage=False, save_version_coverage=None, html_report=None, permit_mixed_migrations=None, show_logging=None, validate_html=None, validate_html_harder=None, rerun=None, **kwargs):
-        #
+        parser.add_argument('--no-manage-blobstore', action='store_false', dest='manage_blobstore',
+                            help='Disable creating/deleting test buckets in the blob store.'
+                                 'When this argument is used, a set of buckets with "test-" prefixed to their '
+                                 'names must already exist.')
+
+    def __init__(
+        self,
+        ignore_lower_coverage=False,
+        skip_coverage=False,
+        save_version_coverage=None,
+        html_report=None,
+        permit_mixed_migrations=None,
+        show_logging=None,
+        validate_html=None,
+        validate_html_harder=None,
+        rerun=None,
+        manage_blobstore=True,
+        **kwargs
+    ):    #
         self.ignore_lower_coverage = ignore_lower_coverage
         self.check_coverage = not skip_coverage
         self.save_version_coverage = save_version_coverage
@@ -754,7 +770,7 @@ def __init__(self, ignore_lower_coverage=False, skip_coverage=False, save_versio
         # specific classes necessary to get the right ordering:
         self.reorder_by = (PyFlakesTestCase, MyPyTest,) + self.reorder_by + (StaticLiveServerTestCase, TemplateTagTest, CoverageTest,)
         #self.buckets = set()
-        self.blobstoremanager = TestBlobstoreManager()
+        self.blobstoremanager = TestBlobstoreManager() if manage_blobstore else None
 
     def setup_test_environment(self, **kwargs):
         global template_coverage_collection
@@ -939,7 +955,8 @@ def setup_test_environment(self, **kwargs):
                 print(" (extra pedantically)")
                 self.vnu = start_vnu_server()
 
-        self.blobstoremanager.createTestBlobstores()
+        if self.blobstoremanager is not None:
+            self.blobstoremanager.createTestBlobstores()
 
         super(IetfTestRunner, self).setup_test_environment(**kwargs)
 
@@ -971,7 +988,8 @@ def teardown_test_environment(self, **kwargs):
             if self.vnu:
                 self.vnu.terminate()
 
-        self.blobstoremanager.destroyTestBlobstores()
+        if self.blobstoremanager is not None:
+            self.blobstoremanager.destroyTestBlobstores()
 
         super(IetfTestRunner, self).teardown_test_environment(**kwargs)