Create Data Access Layer for Cosmos SQL API

.env.template

@@ -1,5 +1,14 @@
-# Package where the app is located
+# API
+## Package where the app is located
 export FLASK_APP=time_tracker_api
 
-# The database connection URI. Check out the README.md for more details
-DATABASE_URI=mssql+pyodbc://<user>:<password>@time-tracker-srv.database.windows.net/<database>?driver\=ODBC Driver 17 for SQL Server
+# Common attributes
+## In case you use an Azure SQL database, you must specify the database connection URI. Check out the README.md for more details
+#export DATABASE_URI=mssql+pyodbc://<user>:<password>@time-tracker-srv.database.windows.net/<database>?driver\=ODBC Driver 17 for SQL Server
+
+## For Azure Cosmos DB
+export DATABASE_ACCOUNT_URI=https://<project_db_name>.documents.azure.com:443
+export DATABASE_MASTER_KEY=<db_master_key>
+export DATABASE_NAME=<db_name>
+### or
+# export DATABASE_URI=AccountEndpoint=<ACCOUNT_URI>;AccountKey=<ACCOUNT_KEY>

Dockerfile

@@ -8,7 +8,7 @@ COPY . .
 
 RUN apk update \
 	&& apk add --no-cache $buildDeps gcc unixodbc-dev \
-	&& pip3 install --no-cache-dir -r requirements/prod.txt \	
+	&& pip3 install --no-cache-dir -r requirements/time_tracker_api/prod.txt \
 	&& curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.2.1-1_amd64.apk \
 	&& curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_17.5.2.1-1_amd64.apk \
 	&& curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.2.1-1_amd64.sig \
@@ -27,4 +27,4 @@ ENV FLASK_APP time_tracker_api
 
 EXPOSE 5000
 
-CMD ["gunicorn", "-b 0.0.0.0:5000", "run:app"]
+CMD ["gunicorn", "-b 0.0.0.0:5000", "api:app"]

README.md

@@ -1,6 +1,6 @@
 # time-tracker-api
 
-The API of the TSheets killer app.
+This is the mono-repository for the backend services and their common codebase
 
 ## Getting started
 Follow the following instructions to get the project ready to use ASAP.
@@ -30,10 +30,16 @@ automatically [pip](https://pip.pypa.io/en/stable/) as well.
 
 - Install the requirements:
     ```
-    python3 -m pip install -r requirements/<stage>.txt
+    python3 -m pip install -r requirements/<app>/<stage>.txt
     ```
 
-    The `stage` can be `dev` or `prod`. 
+    Where <app> is one of the executable app namespace, e.g. `time_tracker_api`.
+    The `stage` can be 
+
+    * `dev`: Used for working locally
+    * `prod`: For anything deployed
+
+
     Remember to do it with Python 3.
 
 
@@ -94,7 +100,7 @@ The [integrations tests](https://en.wikipedia.org/wiki/Integration_testing) veri
 are working well together. These are the default tests we should run:
 
 ```dotenv
-python3 -m pytest -v --ignore=tests/sql_repository_test.py
+python3 -m pytest -v --ignore=tests/commons/data_access_layer/azure/sql_repository_test.py
 ```
 
 As you may have noticed we are ignoring the tests related with the repository.

commons/data_access_layer/cosmos_db.py

@@ -0,0 +1,149 @@
+import dataclasses
+import uuid
+from typing import Callable
+
+import azure.cosmos.cosmos_client as cosmos_client
+import azure.cosmos.exceptions as exceptions
+from azure.cosmos import ContainerProxy
+from flask import Flask
+
+
+class CosmosDBFacade:
+    def __init__(self, app: Flask):  # pragma: no cover
+        self.app = app
+
+        db_uri = app.config.get('DATABASE_URI')
+        if db_uri is None:
+            app.logger.warn("DATABASE_URI was not found. Looking for alternative variables.")
+            account_uri = app.config.get('DATABASE_ACCOUNT_URI')
+            if account_uri is None:
+                raise EnvironmentError("DATABASE_ACCOUNT_URI is not defined in the environment")
+
+            master_key = app.config.get('DATABASE_MASTER_KEY')
+            if master_key is None:
+                raise EnvironmentError("DATABASE_MASTER_KEY is not defined in the environment")
+
+            self.client = cosmos_client.CosmosClient(account_uri, {'masterKey': master_key},
+                                                     user_agent="CosmosDBDotnetQuickstart",
+                                                     user_agent_overwrite=True)
+        else:
+            self.client = cosmos_client.CosmosClient.from_connection_string(db_uri)
+
+        db_id = app.config.get('DATABASE_NAME')
+        if db_id is None:
+            raise EnvironmentError("DATABASE_NAME is not defined in the environment")
+
+        self.db = self.client.get_database_client(db_id)
+
+    def create_container(self, container_definition: dict):
+        try:
+            return self.db.create_container(**container_definition)
+
+        except exceptions.CosmosResourceExistsError:  # pragma: no cover
+            self.app.logger.info('Container with id \'{0}\' was found'.format(container_definition["id"]))
+
+    def delete_container(self, container_id: str):
+        try:
+            return self.db.delete_container(container_id)
+
+        except exceptions.CosmosHttpResponseError:  # pragma: no cover
+            self.app.logger.info('Container with id \'{0}\' was not deleted'.format(container_id))
+
+
+cosmos_helper: CosmosDBFacade = None
+
+
+class CosmosDBModel():
+    def __init__(self, data):
+        names = set([f.name for f in dataclasses.fields(self)])
+        for k, v in data.items():
+            if k in names:
+                setattr(self, k, v)
+
+
+class CosmosDBRepository:
+    def __init__(self, container_id: str,
+                 mapper: Callable = None,
+                 custom_cosmos_helper: CosmosDBFacade = None):
+        global cosmos_helper
+        self.cosmos_helper = custom_cosmos_helper or cosmos_helper
+        if self.cosmos_helper is None:  # pragma: no cover
+            raise ValueError("The cosmos_db module has not been initialized!")
+        self.mapper = mapper
+        self.container: ContainerProxy = self.cosmos_helper.db.get_container_client(container_id)
+
+    @classmethod
+    def from_definition(cls, container_definition: dict,
+                        mapper: Callable = None,
+                        custom_cosmos_helper: CosmosDBFacade = None):
+        return cls(container_definition['id'], mapper, custom_cosmos_helper)
+
+    def create(self, data: dict, mapper: Callable = None):
+        function_mapper = self.get_mapper_or_dict(mapper)
+        return function_mapper(self.container.create_item(body=data))
+
+    def find(self, id: str, partition_key_value, visible_only=True, mapper: Callable = None):
+        found_item = self.container.read_item(id, partition_key_value)
+        function_mapper = self.get_mapper_or_dict(mapper)
+        return function_mapper(self.check_visibility(found_item, visible_only))
+
+    def find_all(self, partition_key_value: str, max_count=None, offset=0,
+                 visible_only=True, mapper: Callable = None):
+        # TODO Use the tenant_id param and change container alias
+        max_count = self.get_page_size_or(max_count)
+        result = self.container.query_items(
+            query="""
+            SELECT * FROM c WHERE c.tenant_id=@tenant_id AND {visibility_condition}
+            OFFSET @offset LIMIT @max_count
+            """.format(visibility_condition=self.create_sql_condition_for_visibility(visible_only)),
+            parameters=[
+                {"name": "@tenant_id", "value": partition_key_value},
+                {"name": "@offset", "value": offset},
+                {"name": "@max_count", "value": max_count},
+            ],
+            partition_key=partition_key_value,
+            max_item_count=max_count)
+
+        function_mapper = self.get_mapper_or_dict(mapper)
+        return list(map(function_mapper, result))
+
+    def partial_update(self, id: str, changes: dict, partition_key_value: str,
+                       visible_only=True, mapper: Callable = None):
+        item_data = self.find(id, partition_key_value, visible_only=visible_only)
+        item_data.update(changes)
+        return self.update(id, item_data, mapper=mapper)
+
+    def update(self, id: str, item_data: dict, mapper: Callable = None):
+        function_mapper = self.get_mapper_or_dict(mapper)
+        return function_mapper(self.container.replace_item(id, body=item_data))
+
+    def delete(self, id: str, partition_key_value: str, mapper: Callable = None):
+        return self.partial_update(id, {
+            'deleted': str(uuid.uuid4())
+        }, partition_key_value, visible_only=True, mapper=mapper)
+
+    def check_visibility(self, item, throw_not_found_if_deleted):
+        if throw_not_found_if_deleted and item.get('deleted') is not None:
+            raise exceptions.CosmosResourceNotFoundError(message='Deleted item',
+                                                         status_code=404)
+
+        return item
+
+    def create_sql_condition_for_visibility(self, visible_only: bool, container_name='c') -> str:
+        if visible_only:
+            # We are considering that `deleted == null` is not a choice
+            return 'NOT IS_DEFINED(%s.deleted)' % container_name
+        return 'true'
+
+    def get_mapper_or_dict(self, alternative_mapper: Callable) -> Callable:
+        return alternative_mapper or self.mapper or dict
+
+    def get_page_size_or(self, custom_page_size: int) -> int:
+        # TODO The default value should be taken from the Azure Feature Manager
+        # or any other repository for the settings
+        return custom_page_size or 100
+
+
+def init_app(app: Flask) -> None:
+    global cosmos_helper
+    cosmos_helper = CosmosDBFacade(app)

time_tracker_api/sql_repository.py → commons/data_access_layer/sql.py

@@ -14,7 +14,7 @@ def handle_commit_issues(f):
     def rollback_if_necessary(*args, **kw):
         try:
             return f(*args, **kw)
-        except:
+        except: # pragma: no cover
             db.session.rollback()
             raise
 
@@ -92,7 +92,7 @@ def delete(self, id):
         self.repository.remove(id)
 
 
-class SQLSeeder(Seeder):
+class SQLSeeder(Seeder): # pragma: no cover
     def run(self):
         print("Provisioning database...")
         db.create_all()

requirements/azure_cosmos.txt

@@ -0,0 +1,14 @@
+# requirements/azure_cosmos.txt
+
+# For Cosmos DB
+
+# Azure Cosmos DB official library
+azure-core==1.1.1
+azure-cosmos==4.0.0b6
+certifi==2019.11.28
+chardet==3.0.4
+idna==2.8
+six==1.13.0
+urllib3==1.25.7
+virtualenv==16.7.9
+virtualenv-clone==0.5.3

requirements/commons.txt

@@ -0,0 +1,9 @@
+# requirements/commons.txt
+
+# For Common dependencies
+
+# Handling requests
+requests==2.23.0
+
+# To create sample content in tests and API documentation
+Faker==4.0.2

requirements/sql_db.txt

@@ -0,0 +1,12 @@
+# requirements/sql_db.txt
+
+# For SQL database (MS SQL)
+
+
+# SQL Server driver
+pyodbc==4.0.30
+
+# ORM
+SQLAlchemy==1.3.15
+SQLAlchemy-Utils==0.36.3
+flask_sqlalchemy==2.4.1

requirements/dev.txt → requirements/time_tracker_api/dev.txt

@@ -1,5 +1,6 @@
-# requirements/dev.txt
+# requirements/time_tracker_api/dev.txt
 
+# Include the prod resources
 -r prod.txt
 
 # For development
@@ -9,7 +10,6 @@ pytest==5.2.0
 
 # Mocking
 pytest-mock==2.0.0
-Faker==4.0.2
 
 # Coverage
 coverage==4.5.1

requirements/prod.txt → requirements/time_tracker_api/prod.txt

@@ -1,4 +1,9 @@
-# requirements/prod.txt
+# requirements/time_tracker_api/prod.txt
+
+# Dependencies
+-r ../commons.txt
+-r ../azure_cosmos.txt
+-r ../sql_db.txt
 
 # For production releases
 
@@ -15,21 +20,9 @@ gunicorn==20.0.4
 #Swagger support for Restful API
 flask-restplus==0.12.1
 
-#Mocking
-Faker==4.0.2
-
 #CLI support
 Flask-Script==2.0.6
 
-# SQL database (MS SQL)
-pyodbc==4.0.30
-SQLAlchemy==1.3.15
-SQLAlchemy-Utils==0.36.3
-flask_sqlalchemy==2.4.1
-
-# Handling requests
-requests==2.23.0
-
 # The Debug Toolbar
 Flask-DebugToolbar==0.11.0
 

setup.cfg

@@ -6,3 +6,4 @@ addopts = -p no:warnings
 branch = True
 source =
     time_tracker_api
+    commons