Graphql Schema Introduction (canada-ca#155)

* Created new domain management type. Started initial work on new domains type

* Fix tests to reflect changes to domains

* Continued work on testing json stuff

* Created New README.md to give a examples on how to write schema objects

* Update README.md

Fixing formatting

* Update README.md

* Update README.md

* Created new schema directory structure. Skipped all resolvers tests, and commented them out from the schema. Waiting on db decision

* DMARC object type completed

* SPF, and the inital EmailScan objects are in working order

* SPF, and DMARC definitions completed

* Removed Bool values from domains table

* Update README.md

Added in .all() in missing cases

* Stated work on domain object type. Added fix to limit only related dmarc and spf scans results be returned

* Initial work on dkim object type has been completed

* Started work on www_scan and associated files, moved shared functions into seperate files in the functions folder

* Combined elements straight into object other than tags, due to their logic

* Started work on domain resolver

* Fix db migration after rebase

* Completed initial domain resolvers and corresponding tests, corrected dkim tag generation

* Fixed org id issue in the domain resolver

* Refactor test_domains_resolver.py to test_domains_resolver_access_control.py

* Started work on resolver README.md

* Fixed auth issue for user not having any roles in db

* Started work on graphql logger filter

* created ssl_tags file

* Added descriptions to domain(s) resolver

* Formatted excludes to be one field one row

* Completed first iteration of the resolver README.md

* Uncomment check

Author: IdezHD

api/Pipfile.lock

@@ -1,5 +1,7 @@
 from flask import Flask
 
+from app.logger import logger
+
 from db import (
     db,
     DB_NAME,

api/app/__init__.py

@@ -0,0 +1,66 @@
+import logging
+import logging.config
+from graphql import GraphQLError
+
+
+class GraphQLErrorLogFilter(logging.Filter):
+    def filter(self, record):
+        exc_type, exc, _ = record.exc_info
+        graphql_error = isinstance(exc, GraphQLError)
+        if graphql_error:
+            return False
+        return True
+
+
+class GraphQLLocatedLogFilter(logging.Filter):
+    def filter(self, record):
+        if 'graphql.error.located_error.GraphQLLocatedError:' in record.msg:
+            return False
+        return True
+
+
+class GraphQLTracebackLogFilter(logging.Filter):
+    def filter(self, record):
+        if 'graphql.execution.utils:Traceback (most recent call last):' in record.msg:
+            return False
+        return True
+
+
+logger_dict = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+        },
+    },
+    # Prevent graphql exception from displaying in console
+    'filters': {
+        'graphql_error_log_filter': {
+            '()': GraphQLErrorLogFilter,
+        },
+        'graphql_located_error_filter': {
+            '()': GraphQLLocatedLogFilter
+        },
+        'graphql_traceback_error_filter': {
+            '()': GraphQLTracebackLogFilter
+        }
+    },
+    'loggers': {
+        'graphql.execution.executor': {
+            'level': 'WARNING',
+            'handlers': ['console'],
+            'filters': [
+                'graphql_error_log_filter',
+                'graphql_located_error_filter',
+                'graphql_traceback_error_filter',
+            ],
+        },
+    },
+}
+
+logging.config.dictConfig(logger_dict)
+logger = logging.getLogger('custom')
+# logging.error('Error message')
+

api/app/logger.py

@@ -4,7 +4,7 @@
 from graphql import GraphQLError
 
 from functions.orm_to_dict import orm_to_dict
-from app import app
+from app import app, logger
 from models import User_affiliations, Organizations
 
 user_admin_perm = ['super_admin', 'admin']
@@ -20,12 +20,16 @@ def decode_auth_token(request):
     """
     auth_header = request.headers.get('Authorization')
     try:
-        payload = jwt.decode(auth_header, os.getenv('SUPER_SECRET_SALT'), algorithms=['HS256'])
+        payload = jwt.decode(
+            auth_header,
+            os.getenv('SUPER_SECRET_SALT'),
+            algorithms=['HS256']
+        )
         return payload['roles']
     except jwt.ExpiredSignatureError:
-        raise GraphQLError('Signature expired. Please login again')
+        raise GraphQLError('Signature expired, please login again')
     except jwt.InvalidTokenError:
-        raise GraphQLError('Invalid token. Please login again')
+        raise GraphQLError('Invalid token, please login again')
 
 
 def check_user_claims(user_claims):
@@ -36,14 +40,17 @@ def check_user_claims(user_claims):
     :param user_claims: A list of dicts that contain the users claims
     :return: Returns a valid list of user claims
     """
-    if user_claims:
+    user_roles = []
+    if user_claims[0] == 'none':
+        raise GraphQLError('Error, please contact your administrator for '
+                           'organization access.')
+    elif user_claims:
         user_id = user_claims[0]['user_id']
         with app.app_context():
             user_aff = User_affiliations.query.filter(
                 User_affiliations.user_id == user_id).all()
             user_aff = orm_to_dict(user_aff)
         if user_aff:
-            user_roles = []
             for select in user_aff:
                 temp_dict = {
                     'user_id': select['user_id'],
@@ -57,13 +64,12 @@ def check_user_claims(user_claims):
                           + list(
             itertools.filterfalse(lambda x: x in user_roles, user_claims))
         if user_claim_diff:
-            print(user_claim_diff)
             # User has a difference in their claims
-            raise GraphQLError("Error, Please sign in again.")
+            raise GraphQLError("Error, please sign in again.")
         else:
             return user_claims
     else:
-        raise GraphQLError("User has no claims")
+        raise GraphQLError("User has no claims, please sign in again")
 
 
 def require_token(method):
@@ -74,4 +80,5 @@ def wrapper(self, *args, **kwargs):
             kwargs['user_roles'] = user_claims
             return method(self, *args, **kwargs)
         raise GraphQLError(auth_resp)
+
     return wrapper

api/functions/auth_wrappers.py

@@ -22,6 +22,14 @@ def seed_org(db, app):
         db.session.add(org)
         db.session.commit()
 
+    org = Organizations(
+        id=3,
+        organization='ORG3',
+        description='Organization 3',
+    )
+    with app.app_context():
+        db.session.add(org)
+        db.session.commit()
 
 def remove_org(db, app):
     with app.app_context():

api/functions/db_seeding/organizations.py

@@ -0,0 +1,14 @@
+from sqlalchemy.orm import load_only
+
+from db import db
+from models import Scans, Domains
+
+
+def get_domain(self, info):
+    domain_id = db.session.query(Scans).filter(
+        Scans.id == self.id
+    ).options(load_only('domain_id')).first()
+    domain = db.session.query(Domains).filter(
+        Domains.id == domain_id.domain_id
+    ).options(load_only('domain')).first()
+    return domain.domain

api/functions/get_domain.py

@@ -0,0 +1,10 @@
+from sqlalchemy.orm import load_only
+
+from db import db
+from models import Scans, Domains
+
+def get_timestamp(self, info):
+    timestamp = db.session.query(Scans).filter(
+        Scans.id == self.id
+    ).options(load_only('scan_date')).first()
+    return timestamp.scan_date

api/functions/get_timestamp.py

@@ -55,7 +55,7 @@ def sign_in_user(user_name, password):
                 }
                 user_roles.append(temp_dict)
         else:
-            user_roles = 'none'
+            user_roles = ['none']
         try:
             payload = {
                 'exp': datetime.datetime.utcnow() + datetime.timedelta(days=0, seconds=1800),

api/functions/sign_in_user.py

@@ -0,0 +1,40 @@
+"""empty message
+
+Revision ID: c98ceb3d9163
+Revises: 48853738b19d
+Create Date: 2020-03-09 08:13:31.523128
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c98ceb3d9163'
+down_revision = '48853738b19d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('domains', 'scan_ssl')
+    op.drop_column('domains', 'scan_dmarc_psl')
+    op.drop_column('domains', 'scan_dkim')
+    op.drop_column('domains', 'scan_dmarc')
+    op.drop_column('domains', 'scan_https')
+    op.drop_column('domains', 'scan_mx')
+    op.drop_column('domains', 'scan_spf')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('domains', sa.Column('scan_spf', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    op.add_column('domains', sa.Column('scan_mx', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    op.add_column('domains', sa.Column('scan_https', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    op.add_column('domains', sa.Column('scan_dmarc', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    op.add_column('domains', sa.Column('scan_dkim', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    op.add_column('domains', sa.Column('scan_dmarc_psl', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    op.add_column('domains', sa.Column('scan_ssl', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    # ### end Alembic commands ###

api/migrations/versions/c98ceb3d9163_.py

@@ -13,13 +13,6 @@ class Domains(db.Model):
     id = Column(Integer, primary_key=True)
     domain = Column(String)
     last_run = Column(DateTime)
-    scan_spf = Column(Boolean)
-    scan_dmarc = Column(Boolean)
-    scan_dmarc_psl = Column(Boolean)
-    scan_mx = Column(Boolean)
-    scan_dkim = Column(Boolean)
-    scan_https = Column(Boolean)
-    scan_ssl = Column(Boolean)
     dmarc_phase = Column(Integer)
     organization_id = Column(Integer, ForeignKey('organizations.id'))
     organization = relationship("Organizations", back_populates="domains", cascade="all, delete")
@@ -92,51 +85,41 @@ class Scans(db.Model):
     scan_date = Column(DateTime)
     initiated_by = Column(Integer, ForeignKey('users.id'))
     domain = relationship("Domains", back_populates="scans", cascade="all, delete")
-    dmarc = relationship("Dmarc_scans", back_populates="dmarc_flagged_scan", cascade="all, delete")
-    dkim = relationship("Dkim_scans", back_populates="dkim_flagged_scan", cascade="all, delete")
-    spf = relationship("Spf_scans", back_populates="spf_flagged_scan", cascade="all, delete")
-    https = relationship("Https_scans", back_populates="https_flagged_scan", cascade="all, delete")
-    ssl = relationship("Ssl_scans", back_populates="ssl_flagged_scan", cascade="all, delete")
 
 
 class Dmarc_scans(db.Model):
     __tablename__ = 'dmarc_scans'
 
     id = Column(Integer, ForeignKey('scans.id'), primary_key=True)
     dmarc_scan = Column(JSONB)
-    dmarc_flagged_scan = relationship("Scans", back_populates="dmarc", cascade="all, delete")
 
 
 class Dkim_scans(db.Model):
     __tablename__ = 'dkim_scans'
 
     id = Column(Integer, ForeignKey('scans.id'), primary_key=True)
     dkim_scan = Column(JSONB)
-    dkim_flagged_scan = relationship("Scans", back_populates="dkim", cascade="all, delete")
 
 
 class Spf_scans(db.Model):
     __tablename__ = 'spf_scans'
 
     id = Column(Integer, ForeignKey('scans.id'), primary_key=True)
     spf_scan = Column(JSONB)
-    spf_flagged_scan = relationship("Scans", back_populates="spf", cascade="all, delete")
 
 
 class Https_scans(db.Model):
     __tablename__ = 'https_scans'
 
     id = Column(Integer, ForeignKey('scans.id'), primary_key=True)
     https_scan = Column(JSONB)
-    https_flagged_scan = relationship("Scans", back_populates="https", cascade="all, delete")
 
 
 class Ssl_scans(db.Model):
     __tablename__ = 'ssl_scans'
 
     id = Column(Integer, ForeignKey('scans.id'), primary_key=True)
     ssl_scan = Column(JSONB)
-    ssl_flagged_scan = relationship("Scans", back_populates="ssl", cascade="all, delete")
 
 
 class Ciphers(db.Model):