From 2497d416cbc98973ac9289d693b0f79e2868139a Mon Sep 17 00:00:00 2001 From: nsdeschenes Date: Wed, 26 May 2021 06:30:46 -0300 Subject: [PATCH 1/4] update core service create_collections --- services/core/core.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/services/core/core.py b/services/core/core.py index 452ca9f294..532607dc2e 100644 --- a/services/core/core.py +++ b/services/core/core.py @@ -111,7 +111,12 @@ def update_guidance( for entry in guidance_data: if entry["file"] == "scanSummaryCriteria.json": if not db.has_collection("scanSummaryCriteria"): - db.create_collection("scanSummaryCriteria") + db.create_collection( + name="scanSummaryCriteria", + replication_factor=2, + shard_count=6, + write_concern=1, + ) for criteria_type, criteria in entry["guidance"].items(): new_criteria = { "_key": criteria_type, @@ -153,7 +158,12 @@ def update_guidance( elif entry["file"] == "chartSummaryCriteria.json": if not db.has_collection("chartSummaryCriteria"): - db.create_collection("chartSummaryCriteria") + db.create_collection( + "chartSummaryCriteria", + replication_factor=2, + shard_count=6, + write_concern=1, + ) for criteria_type, criteria in entry["guidance"].items(): new_criteria = { "_key": criteria_type, @@ -191,7 +201,12 @@ def update_guidance( file_name = entry["file"].split(".json")[0] tag_type = file_name.split("tags_")[1] if not db.has_collection(f"{tag_type}GuidanceTags"): - db.create_collection(f"{tag_type}GuidanceTags") + db.create_collection( + f"{tag_type}GuidanceTags", + replication_factor=2, + shard_count=6, + write_concern=1, + ) for tag_key, tag_data in entry["guidance"].items(): new_tag = { "_key": tag_key, @@ -242,7 +257,12 @@ def update_scan_summaries(host=DB_HOST, name=DB_NAME, user=DB_USER, password=DB_ db = client.db(name, username=user, password=password) if not db.has_collection("scanSummaries"): - db.create_collection("scanSummaries") + db.create_collection( + "scanSummaries", + replication_factor=2, + shard_count=6, + write_concern=1, + ) for scan_type in SCAN_TYPES: scan_pass = 0 @@ -288,7 +308,12 @@ def update_chart_summaries(host=DB_HOST, name=DB_NAME, user=DB_USER, password=DB db = client.db(name, username=user, password=password) if not db.has_collection("chartSummaries"): - db.create_collection("chartSummaries") + db.create_collection( + "chartSummaries", + replication_factor=2, + shard_count=6, + write_concern=1, + ) for chart_type, scan_types in CHARTS.items(): pass_count = 0 From 9db065b98dcfa893b9725dbd4c6490d38a08a971 Mon Sep 17 00:00:00 2001 From: nsdeschenes Date: Wed, 26 May 2021 06:39:58 -0300 Subject: [PATCH 2/4] update replication factor to 3 --- services/core/core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/core/core.py b/services/core/core.py index 532607dc2e..d6ec3f6786 100644 --- a/services/core/core.py +++ b/services/core/core.py @@ -113,7 +113,7 @@ def update_guidance( if not db.has_collection("scanSummaryCriteria"): db.create_collection( name="scanSummaryCriteria", - replication_factor=2, + replication_factor=3, shard_count=6, write_concern=1, ) @@ -160,7 +160,7 @@ def update_guidance( if not db.has_collection("chartSummaryCriteria"): db.create_collection( "chartSummaryCriteria", - replication_factor=2, + replication_factor=3, shard_count=6, write_concern=1, ) @@ -203,7 +203,7 @@ def update_guidance( if not db.has_collection(f"{tag_type}GuidanceTags"): db.create_collection( f"{tag_type}GuidanceTags", - replication_factor=2, + replication_factor=3, shard_count=6, write_concern=1, ) @@ -259,7 +259,7 @@ def update_scan_summaries(host=DB_HOST, name=DB_NAME, user=DB_USER, password=DB_ if not db.has_collection("scanSummaries"): db.create_collection( "scanSummaries", - replication_factor=2, + replication_factor=3, shard_count=6, write_concern=1, ) @@ -310,7 +310,7 @@ def update_chart_summaries(host=DB_HOST, name=DB_NAME, user=DB_USER, password=DB if not db.has_collection("chartSummaries"): db.create_collection( "chartSummaries", - replication_factor=2, + replication_factor=3, shard_count=6, write_concern=1, ) From b2a58b1fba61aa491b2cccb23ff0e30494d97669 Mon Sep 17 00:00:00 2001 From: nsdeschenes Date: Wed, 26 May 2021 06:42:02 -0300 Subject: [PATCH 3/4] update database options --- services/super-admin/database-options.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/super-admin/database-options.js b/services/super-admin/database-options.js index aa55c910d5..7c84d1bcc5 100644 --- a/services/super-admin/database-options.js +++ b/services/super-admin/database-options.js @@ -3,14 +3,17 @@ const databaseOptions = ({ rootPass }) => [ { type: 'documentcollection', name: 'users', + options: { replicationFactor: 3, writeConcern: 1, numberOfShards: 6 }, }, { type: 'documentcollection', name: 'organizations', + options: { replicationFactor: 3, writeConcern: 1, numberOfShards: 6 }, }, { type: 'edgecollection', name: 'affiliations', + options: { replicationFactor: 3, writeConcern: 1, numberOfShards: 6 }, }, ] From 719cf8a8ebc51bf49eb3ba20a5a4700c98ab22a0 Mon Sep 17 00:00:00 2001 From: nsdeschenes Date: Wed, 26 May 2021 06:45:07 -0300 Subject: [PATCH 4/4] update dmarc-report database options --- services/dmarc-report/database-options.js | 86 ++--------------------- 1 file changed, 6 insertions(+), 80 deletions(-) diff --git a/services/dmarc-report/database-options.js b/services/dmarc-report/database-options.js index c97d3d2b6e..ca2ee724f8 100644 --- a/services/dmarc-report/database-options.js +++ b/services/dmarc-report/database-options.js @@ -1,108 +1,34 @@ const databaseOptions = ({ rootPass }) => [ { type: 'user', username: 'root', password: rootPass }, - { - type: 'documentcollection', - name: 'users', - }, { type: 'documentcollection', name: 'organizations', + options: { replicationFactor: 3, writeConcern: 1, numberOfShards: 6 }, }, { type: 'documentcollection', name: 'domains', - }, - { - type: 'documentcollection', - name: 'dkim', - }, - { - type: 'documentcollection', - name: 'dkimResults', - }, - { - type: 'documentcollection', - name: 'dmarc', - }, - { - type: 'documentcollection', - name: 'spf', - }, - { - type: 'documentcollection', - name: 'https', - }, - { - type: 'documentcollection', - name: 'ssl', - }, - { - type: 'documentcollection', - name: 'dkimGuidanceTags', - }, - { - type: 'documentcollection', - name: 'dmarcGuidanceTags', - }, - { - type: 'documentcollection', - name: 'spfGuidanceTags', - }, - { - type: 'documentcollection', - name: 'httpsGuidanceTags', - }, - { - type: 'documentcollection', - name: 'sslGuidanceTags', - }, - { - type: 'documentcollection', - name: 'chartSummaries', + options: { replicationFactor: 3, writeConcern: 1, numberOfShards: 6 }, }, { type: 'documentcollection', name: 'dmarcSummaries', - }, - { - type: 'edgecollection', - name: 'affiliations', + options: { replicationFactor: 3, writeConcern: 1, numberOfShards: 6 }, }, { type: 'edgecollection', name: 'claims', - }, - { - type: 'edgecollection', - name: 'domainsDKIM', - }, - { - type: 'edgecollection', - name: 'dkimToDkimResults', - }, - { - type: 'edgecollection', - name: 'domainsDMARC', - }, - { - type: 'edgecollection', - name: 'domainsSPF', - }, - { - type: 'edgecollection', - name: 'domainsHTTPS', - }, - { - type: 'edgecollection', - name: 'domainsSSL', + options: { replicationFactor: 3, writeConcern: 1, numberOfShards: 6 }, }, { type: 'edgecollection', name: 'ownership', + options: { replicationFactor: 3, writeConcern: 1, numberOfShards: 6 }, }, { type: 'edgecollection', name: 'domainsToDmarcSummaries', + options: { replicationFactor: 3, writeConcern: 1, numberOfShards: 6 }, }, ]