Skip to content

Commit 70bcf75

Browse files
EthanljfEthanljf
authored
Miscellaneous refactoring, tidying and dns scanner logic fix (canada-ca#1297)
* Miscellaneous refactoring, tidying and dns scanner logic fix * Processor test logic * Processor test logic
1 parent da8fcc7 commit 70bcf75

6 files changed

Lines changed: 12 additions & 77 deletions

File tree

‎app/bases/knative/config/config.yaml‎

Lines changed: 1 addition & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -23,28 +23,6 @@ spec:
2323
- name: https-scanner
2424
image: gcr.io/track-compliance/scanners/https # The URL to the image of the app
2525
env:
26-
- name: DB_USER
27-
valueFrom:
28-
secretKeyRef:
29-
name: scanners
30-
key: DB_USER
31-
- name: DB_PASS
32-
valueFrom:
33-
secretKeyRef:
34-
name: scanners
35-
key: DB_PASS
36-
- name: DB_HOST
37-
valueFrom:
38-
secretKeyRef:
39-
name: scanners
40-
key: DB_HOST
41-
- name: DB_PORT
42-
value: "5432"
43-
- name: DB_NAME
44-
valueFrom:
45-
secretKeyRef:
46-
name: scanners
47-
key: DB_NAME
4826
- name: RESULT_QUEUE_URL
4927
valueFrom:
5028
secretKeyRef:
@@ -94,7 +72,7 @@ spec:
9472
name: scanners
9573
key: DB_HOST
9674
- name: DB_PORT
97-
value: "5432"
75+
value: "8529"
9876
- name: DB_NAME
9977
valueFrom:
10078
secretKeyRef:
@@ -128,28 +106,6 @@ spec:
128106
- name: ssl-scanner
129107
image: gcr.io/track-compliance/scanners/ssl # The URL to the image of the app
130108
env:
131-
- name: DB_USER
132-
valueFrom:
133-
secretKeyRef:
134-
name: scanners
135-
key: DB_USER
136-
- name: DB_PASS
137-
valueFrom:
138-
secretKeyRef:
139-
name: scanners
140-
key: DB_PASS
141-
- name: DB_HOST
142-
valueFrom:
143-
secretKeyRef:
144-
name: scanners
145-
key: DB_HOST
146-
- name: DB_PORT
147-
value: "5432"
148-
- name: DB_NAME
149-
valueFrom:
150-
secretKeyRef:
151-
name: scanners
152-
key: DB_NAME
153109
- name: RESULT_QUEUE_URL
154110
valueFrom:
155111
secretKeyRef:
@@ -183,28 +139,6 @@ spec:
183139
- name: dns-scanner
184140
image: gcr.io/track-compliance/scanners/dns # The URL to the image of the app
185141
env:
186-
- name: DB_USER
187-
valueFrom:
188-
secretKeyRef:
189-
name: scanners
190-
key: DB_USER
191-
- name: DB_PASS
192-
valueFrom:
193-
secretKeyRef:
194-
name: scanners
195-
key: DB_PASS
196-
- name: DB_HOST
197-
valueFrom:
198-
secretKeyRef:
199-
name: scanners
200-
key: DB_HOST
201-
- name: DB_PORT
202-
value: "5432"
203-
- name: DB_NAME
204-
valueFrom:
205-
secretKeyRef:
206-
name: scanners
207-
key: DB_NAME
208142
- name: RESULT_QUEUE_URL
209143
valueFrom:
210144
secretKeyRef:

‎services/scanners/dns/dns_scanner.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929

3030
def dispatch_results(payload, client):
3131
client.post(QUEUE_URL + "/dns", json=payload)
32-
logging.info("Scan results dispatched to result-processor")
32+
logging.info("Scan results dispatched to result queue")
3333

3434

3535
async def scan_dmarc(domain):
@@ -49,7 +49,7 @@ async def scan_dmarc(domain):
4949
)
5050
return None
5151

52-
for rua in scan_result["dmarc"]["tags"]["rua"]["value"]:
52+
for rua in scan_result["dmarc"]["tags"].get("rua", {}).get("value", []):
5353
# Retrieve 'rua' tag address.
5454
rua_addr = rua["address"]
5555

@@ -87,7 +87,7 @@ async def scan_dmarc(domain):
8787
logging.error("Failed to validate rua address: %s" % str(e))
8888
rua["accepting"] = "undetermined"
8989

90-
for ruf in scan_result["dmarc"]["tags"]["ruf"]["value"]:
90+
for ruf in scan_result["dmarc"]["tags"].get("ruf", {}).get("value", []):
9191
# Retrieve 'ruf' tag address.
9292
ruf_addr = ruf["address"]
9393

‎services/scanners/https/https_scanner.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323

2424
def dispatch_results(payload, client):
2525
client.post(QUEUE_URL + "/https", json=payload)
26-
logging.info("Scan results dispatched to result-processor")
26+
logging.info("Scan results dispatched to result queue")
2727

2828

2929
def scan_https(domain):

‎services/scanners/results/result_processor.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -575,10 +575,11 @@ def insert_dns(report, tags, domain_key, db):
575575
PROCESS = {"https": process_https, "ssl": process_ssl, "dns": process_dns}
576576

577577

578-
def Server(db_host=DB_HOST, db_name=DB_NAME, db_user=DB_USER, db_pass=DB_PASS):
578+
def Server(db_host=DB_HOST, db_name=DB_NAME, db_user=DB_USER, db_pass=DB_PASS, db_port=DB_PORT):
579579

580580
# Establish DB connection
581-
arango_client = ArangoClient(hosts=db_host)
581+
connection_string = f"http://{db_host}:{db_port}"
582+
arango_client = ArangoClient(hosts=connection_string)
582583
db = arango_client.db(db_name, username=db_user, password=db_pass)
583584

584585
async def process(result_request):

‎services/scanners/results/tests/test_result_processor.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ def test_insert_https():
9494
domain_query = db.collection("domains").find({"domain": "cyber.gc.ca"}, limit=1)
9595
domain = domain_query.next()
9696
test_app = Server(
97-
db_host="http://testdb:8529", db_name="test", db_user="", db_pass=""
97+
db_host="testdb", db_name="test", db_user="", db_pass="", db_port=8529
9898
)
9999
test_client = TestClient(test_app)
100100

@@ -121,7 +121,7 @@ def test_insert_ssl():
121121
domain_query = db.collection("domains").find({"domain": "cyber.gc.ca"}, limit=1)
122122
domain = domain_query.next()
123123
test_app = Server(
124-
db_host="http://testdb:8529", db_name="test", db_user="", db_pass=""
124+
db_host="testdb", db_name="test", db_user="", db_pass="", db_port=8529
125125
)
126126
test_client = TestClient(test_app)
127127

@@ -148,7 +148,7 @@ def test_insert_dns():
148148
domain_query = db.collection("domains").find({"domain": "cyber.gc.ca"}, limit=1)
149149
domain = domain_query.next()
150150
test_app = Server(
151-
db_host="http://testdb:8529", db_name="test", db_user="", db_pass=""
151+
db_host="testdb", db_name="test", db_user="", db_pass="", db_port=8529
152152
)
153153
test_client = TestClient(test_app)
154154

‎services/scanners/ssl/ssl_scanner.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ class TlsVersionEnum(Enum):
4242

4343
def dispatch_results(payload, client):
4444
client.post(QUEUE_URL + "/ssl", json=payload)
45-
logging.info("Scan results dispatched to result-processor")
45+
logging.info("Scan results dispatched to result queue")
4646

4747

4848
def get_server_info(domain):

0 commit comments

Comments
 (0)