From 6daac59be35504509a75ce9661040ea3f7585e5d Mon Sep 17 00:00:00 2001 From: Ethan Foss Date: Thu, 24 Sep 2020 12:27:44 -0300 Subject: [PATCH 1/2] Update guidance table --- scripts/insert_csv.py | 13 ++++--- services/auto-scan/autoscan.py | 13 ++++++- .../dc6adc4d8944_update_guidance_table.py | 36 +++++++++++++++++++ services/db-migration/models/__init__.py | 10 ++---- services/scanners/results/result_processor.py | 13 ++++++- 5 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 services/db-migration/migrations/versions/dc6adc4d8944_update_guidance_table.py diff --git a/scripts/insert_csv.py b/scripts/insert_csv.py index 0fd51a0a14..28a916b4a5 100644 --- a/scripts/insert_csv.py +++ b/scripts/insert_csv.py @@ -210,19 +210,22 @@ "guidance", metadata, sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column("tag_id", sqlalchemy.String), sqlalchemy.Column("tag_name", sqlalchemy.String), sqlalchemy.Column("guidance", sqlalchemy.String), - sqlalchemy.Column("ref_links", sqlalchemy.String), + sqlalchemy.Column("ref_links", ARRAY(sqlalchemy.String)), ) -Classification = sqlalchemy.Table( - "classification", +Summaries = sqlalchemy.Table( + "summaries", metadata, sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True), - sqlalchemy.Column("UNCLASSIFIED", sqlalchemy.String), + sqlalchemy.Column("name", sqlalchemy.String), + sqlalchemy.Column("count", sqlalchemy.Integer), + sqlalchemy.Column("percentage", sqlalchemy.Float), + sqlalchemy.Column("type", sqlalchemy.String), ) - async def insert(): # Establish DB connection diff --git a/services/auto-scan/autoscan.py b/services/auto-scan/autoscan.py index 05efe1a5b1..a1cf81f5a7 100755 --- a/services/auto-scan/autoscan.py +++ b/services/auto-scan/autoscan.py @@ -214,9 +214,20 @@ "guidance", metadata, sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column("tag_id", sqlalchemy.String), sqlalchemy.Column("tag_name", sqlalchemy.String), sqlalchemy.Column("guidance", sqlalchemy.String), - sqlalchemy.Column("ref_links", sqlalchemy.String), + sqlalchemy.Column("ref_links", ARRAY(sqlalchemy.String)), +) + +Summaries = sqlalchemy.Table( + "summaries", + metadata, + sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column("name", sqlalchemy.String), + sqlalchemy.Column("count", sqlalchemy.Integer), + sqlalchemy.Column("percentage", sqlalchemy.Float), + sqlalchemy.Column("type", sqlalchemy.String), ) diff --git a/services/db-migration/migrations/versions/dc6adc4d8944_update_guidance_table.py b/services/db-migration/migrations/versions/dc6adc4d8944_update_guidance_table.py new file mode 100644 index 0000000000..7976f9e791 --- /dev/null +++ b/services/db-migration/migrations/versions/dc6adc4d8944_update_guidance_table.py @@ -0,0 +1,36 @@ +"""Update guidance table + +Revision ID: dc6adc4d8944 +Revises: 5f8c3a21331b +Create Date: 2020-09-24 12:04:03.960057 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import ARRAY + + +# revision identifiers, used by Alembic. +revision = 'dc6adc4d8944' +down_revision = '5f8c3a21331b' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_table("classification") + op.add_column( + "guidance", sa.Column("tag_id", sa.String(), index=True), + ) + op.alter_column("guidance", "ref_links", type_=ARRAY(sa.String())) + + +def downgrade(): + op.create_table( + "classification", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("UNCLASSIFIED", sa.String()), + ) + with op.batch_alter_table("guidance") as batch_op: + batch_op.drop_column("tag_id") + batch_op.alter_column("guidance", "ref_links", type_=sa.String()) diff --git a/services/db-migration/models/__init__.py b/services/db-migration/models/__init__.py index 001992abe9..a3e5f12c26 100755 --- a/services/db-migration/models/__init__.py +++ b/services/db-migration/models/__init__.py @@ -181,16 +181,10 @@ "guidance", metadata, sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column("tag_id", sqlalchemy.String), sqlalchemy.Column("tag_name", sqlalchemy.String), sqlalchemy.Column("guidance", sqlalchemy.String), - sqlalchemy.Column("ref_links", sqlalchemy.String), -) - -Classification = sqlalchemy.Table( - "classification", - metadata, - sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True), - sqlalchemy.Column("UNCLASSIFIED", sqlalchemy.String), + sqlalchemy.Column("ref_links", ARRAY(sqlalchemy.String)), ) Summaries = sqlalchemy.Table( diff --git a/services/scanners/results/result_processor.py b/services/scanners/results/result_processor.py index 0a34422ec1..46bee9c152 100755 --- a/services/scanners/results/result_processor.py +++ b/services/scanners/results/result_processor.py @@ -226,9 +226,20 @@ "guidance", metadata, sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column("tag_id", sqlalchemy.String), sqlalchemy.Column("tag_name", sqlalchemy.String), sqlalchemy.Column("guidance", sqlalchemy.String), - sqlalchemy.Column("ref_links", sqlalchemy.String), + sqlalchemy.Column("ref_links", ARRAY(sqlalchemy.String)), +) + +Summaries = sqlalchemy.Table( + "summaries", + metadata, + sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column("name", sqlalchemy.String), + sqlalchemy.Column("count", sqlalchemy.Integer), + sqlalchemy.Column("percentage", sqlalchemy.Float), + sqlalchemy.Column("type", sqlalchemy.String), ) logging.basicConfig(stream=sys.stdout, level=logging.INFO) From 4a7514c9c8d5071daaa63c4cbdc432febb3264a8 Mon Sep 17 00:00:00 2001 From: Ethan Foss Date: Thu, 24 Sep 2020 13:06:26 -0300 Subject: [PATCH 2/2] Revised version file --- .../versions/dc6adc4d8944_update_guidance_table.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/services/db-migration/migrations/versions/dc6adc4d8944_update_guidance_table.py b/services/db-migration/migrations/versions/dc6adc4d8944_update_guidance_table.py index 7976f9e791..36b149c7d4 100644 --- a/services/db-migration/migrations/versions/dc6adc4d8944_update_guidance_table.py +++ b/services/db-migration/migrations/versions/dc6adc4d8944_update_guidance_table.py @@ -20,9 +20,9 @@ def upgrade(): op.drop_table("classification") op.add_column( - "guidance", sa.Column("tag_id", sa.String(), index=True), + "guidance", sa.Column("tag_id", sa.String()), ) - op.alter_column("guidance", "ref_links", type_=ARRAY(sa.String())) + op.alter_column("guidance", "ref_links", type_=ARRAY(sa.String()), postgresql_using='ref_links::character varying[]') def downgrade(): @@ -31,6 +31,5 @@ def downgrade(): sa.Column("id", sa.Integer(), primary_key=True), sa.Column("UNCLASSIFIED", sa.String()), ) - with op.batch_alter_table("guidance") as batch_op: - batch_op.drop_column("tag_id") - batch_op.alter_column("guidance", "ref_links", type_=sa.String()) + op.drop_column("guidance", "tag_id") + op.alter_column("guidance", "ref_links", type_=sa.String())