URLs -> Secrets (canada-ca#1275)

* Alter the use of URLs throughout scanner code to make use of secrets for flexible URL support

* Add default values for os.getenv()

Author: Ethanljf

app/bases/knative/config/autoscan.yaml

@@ -33,10 +33,15 @@ spec:
                     name: scanners
                     key: DB_HOST
               - name: DB_PORT
-                value: "5432"
+                value: "8529"
               - name: DB_NAME
                 valueFrom:
                   secretKeyRef:
                     name: scanners
                     key: DB_NAME
+              - name: SCAN_QUEUE_URL
+                valueFrom:
+                  secretKeyRef:
+                    name: scanners
+                    key: SCAN_QUEUE_URL
           restartPolicy: OnFailure

app/bases/knative/config/config.yaml

@@ -45,6 +45,11 @@ spec:
                 secretKeyRef:
                   name: scanners
                   key: DB_NAME
+            - name: RESULT_QUEUE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: scanners
+                  key: RESULT_QUEUE_URL
 
 ---
 
@@ -145,6 +150,11 @@ spec:
                 secretKeyRef:
                   name: scanners
                   key: DB_NAME
+            - name: RESULT_QUEUE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: scanners
+                  key: RESULT_QUEUE_URL
 
 ---
 
@@ -195,6 +205,11 @@ spec:
                 secretKeyRef:
                   name: scanners
                   key: DB_NAME
+            - name: RESULT_QUEUE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: scanners
+                  key: RESULT_QUEUE_URL
 
 ---
 

app/bases/knative/config/queues.yaml

@@ -21,6 +21,22 @@ spec:
       containers:
         - name: scan-queue
           image: gcr.io/track-compliance/services/scan-queue
+          env:
+            - name: HTTPS_URL
+              valueFrom:
+                secretKeyRef:
+                  name: scanners
+                  key: HTTPS_URL
+            - name: SSL_URL
+              valueFrom:
+                secretKeyRef:
+                  name: scanners
+                  key: SSL_URL
+            - name: DNS_URL
+              valueFrom:
+                secretKeyRef:
+                  name: scanners
+                  key: DNS_URL
 
 ---
 
@@ -47,3 +63,9 @@ spec:
       containers:
         - name: result-queue
           image: gcr.io/track-compliance/services/result-queue
+          env:
+            - name: PROCESSOR_URL
+              valueFrom:
+                secretKeyRef:
+                  name: scanners
+                  key: PROCESSOR_URL

services/auto-scan/autoscan.py

@@ -5,7 +5,6 @@
 import datetime
 import traceback
 from arango import ArangoClient
-from uuid import uuid4 as unique_id
 
 logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 
@@ -14,8 +13,7 @@
 DB_PORT = os.getenv("DB_PORT")
 DB_NAME = os.getenv("DB_NAME")
 DB_HOST = os.getenv("DB_HOST")
-
-QUEUE_URL = "http://scan-queue.scanners.svc.cluster.local"
+QUEUE_URL = os.getenv("SCAN_QUEUE_URL", "http://scan-queue.scanners.svc.cluster.local")
 
 
 def dispatch_https(domain, client):
@@ -46,11 +44,12 @@ def dispatch_dns(domain, client):
     client.post(QUEUE_URL + "/dns", json=payload)
 
 
-def scan(db_host, db_name, user_name, password, http_client=requests):
+def scan(db_host, db_port, db_name, user_name, password, http_client=requests):
     logging.info("Retrieving domains for scheduled scan...")
     try:
         # Establish DB connection
-        arango_client = ArangoClient(hosts=db_host)
+        connection_string = f"http://{db_host}:{db_port}"
+        arango_client = ArangoClient(hosts=connection_string)
         db = arango_client.db(db_name, username=user_name, password=password)
 
         logging.info("Querying domains...")
@@ -80,5 +79,5 @@ def scan(db_host, db_name, user_name, password, http_client=requests):
     return count
 
 if __name__ == "__main__":
-    dispatched_count = scan(DB_HOST, DB_NAME, DB_USER, DB_PASS)
+    dispatched_count = scan(DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASS)
     logging.info(f"Dispatched scans for {dispatched_count} domains.")

services/auto-scan/requirements.txt

@@ -2,4 +2,3 @@ pytest
 requests>=2.18.4
 python-arango
 pretend
-uuid

services/auto-scan/tests/test_autoscan.py

@@ -28,7 +28,7 @@ def test_dispatch():
     client_stub = stub(post=lambda url, json: None)
 
     dispatched = scan(
-        "http://testdb:8529", "test", "", "", http_client=client_stub
+        "testdb", 8529, "test", "", "", http_client=client_stub
     )
 
     assert dispatched == len(input_domains)

services/result-queue/result_queue.py

@@ -15,7 +15,7 @@
 logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 pool = ConnectionPool(host="127.0.0.1", port=6379, db=0)
 
-PROCESSOR_URL = "http://result-processor.scanners.svc.cluster.local"
+PROCESSOR_URL = os.getenv("PROCESSOR_URL", "http://result-processor.scanners.svc.cluster.local")
 
 redis = Redis(connection_pool=pool)
 https_queue = Queue("https", connection=redis)

services/scan-queue/scan_queue.py

@@ -15,9 +15,9 @@
 logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 pool = ConnectionPool(host="127.0.0.1", port=6379, db=0)
 
-HTTPS_URL = "http://https-scanner.scanners.svc.cluster.local"
-SSL_URL = "http://ssl-scanner.scanners.svc.cluster.local"
-DNS_URL = "http://dns-scanner.scanners.svc.cluster.local"
+HTTPS_URL = os.getenv("HTTPS_URL", "http://https-scanner.scanners.svc.cluster.local")
+SSL_URL = os.getenv("SSL_URL", "http://ssl-scanner.scanners.svc.cluster.local")
+DNS_URL = os.getenv("DNS_URL", "http://dns-scanner.scanners.svc.cluster.local")
 
 redis = Redis(connection_pool=pool)
 https_queue = Queue("https", connection=redis)

services/scanners/dns/dns_scanner.py

@@ -24,11 +24,11 @@
 
 logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 
-QUEUE_URL = "http://result-queue.scanners.svc.cluster.local/dns"
+QUEUE_URL = os.getenv("RESULT_QUEUE_URL", "http://result-queue.scanners.svc.cluster.local")
 
 
 def dispatch_results(payload, client):
-    client.post(QUEUE_URL, json=payload)
+    client.post(QUEUE_URL + "/dns", json=payload)
     logging.info("Scan results dispatched to result-processor")
 
 

services/scanners/https/https_scanner.py

@@ -18,11 +18,11 @@
 
 MIN_HSTS_AGE = 31536000  # one year
 
-QUEUE_URL = "http://result-queue.scanners.svc.cluster.local/https"
+QUEUE_URL = os.getenv("RESULT_QUEUE_URL", "http://result-queue.scanners.svc.cluster.local")
 
 
 def dispatch_results(payload, client):
-    client.post(QUEUE_URL, json=payload)
+    client.post(QUEUE_URL + "/https", json=payload)
     logging.info("Scan results dispatched to result-processor")