Merge pull request #76 from ioet/feature/replace-sql-models-for-cosmosdb-ones

Create models in Cosmos DB SQL API for new DAL

Author: EliuX

README.md

@@ -1,10 +1,15 @@
 # time-tracker-api
 
+[![Build status](https://dev.azure.com/IOET-DevOps/TimeTracker-API/_apis/build/status/TimeTracker-API%20-%20CI)](https://dev.azure.com/IOET-DevOps/TimeTracker-API/_build/latest?definitionId=1)
+
 This is the mono-repository for the backend services and their common codebase
 
+
+
 ## Getting started
 Follow the following instructions to get the project ready to use ASAP.
 
+
 ### Requirements
 Be sure you have installed in your system
 
@@ -42,35 +47,6 @@ automatically [pip](https://pip.pypa.io/en/stable/) as well.
     
     Remember to do it with Python 3.
     
-    
-- Install the [Microsoft ODBC Driver for SQL Server](https://docs.microsoft.com/en-us/sql/connect/odbc/microsoft-odbc-driver-for-sql-server?view=sql-server-ver15) 
-in your operative system. Then you have to check out what is the name of the SQL Driver installation. 
-Check it out with:
-
-```bash
-vim /usr/local/etc/odbcinst.ini
-```
-
-It may display something like
-
-```.ini
-[ODBC Driver 17 for SQL Server] 
-Description=Microsoft ODBC Driver 17 for SQL Server 
-Driver=/usr/local/lib/libmsodbcsql.17.dylib 
-UsageCount=2
-```
-
-Then specify the driver name, in this case _DBC Driver 17 for SQL Server_ in the `SQL_DATABASE_URI`, e.g.:
-
-```.dotenv
-SQL_DATABASE_URI=mssql+pyodbc://<user>:<password>@time-tracker-srv.database.windows.net/<database>?driver\=ODBC Driver 17 for SQL Server
-```
-
-To troubleshoot issues regarding this part please check out:
-- [Install the Microsoft ODBC driver for SQL Server (macOS)](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/install-microsoft-odbc-driver-sql-server-macos?view=sql-server-ver15).
-- Github issue [odbcinst: SQLRemoveDriver failed with Unable to find component name](https://github.com/Microsoft/homebrew-mssql-preview/issues/2).
-- Stack overflow solution to [Can't open lib 'ODBC Driver 13 for SQL Server'? Sym linking issue?](https://stackoverflow.com/questions/44527452/cant-open-lib-odbc-driver-13-for-sql-server-sym-linking-issue).
-
 ### How to use it
 - Set the env var `FLASK_APP` to `time_tracker_api` and start the app:
 
@@ -89,6 +65,13 @@ To troubleshoot issues regarding this part please check out:
 a link to the swagger.json with the definition of the api.
 
 
+### Important notes
+Due to the used technology and particularities on the implementation of this API, it is important that you respect the
+following notes regarding to the manipulation of the data from and towards the API:
+
+- The [recommended](https://docs.microsoft.com/en-us/azure/cosmos-db/working-with-dates#storing-datetimes) format for 
+DateTime strings in Azure Cosmos DB is `YYYY-MM-DDThh:mm:ss.fffffffZ` which follows the ISO 8601 **UTC standard**.
+
 ## Development
 
 ### Test

cli.py

@@ -46,23 +46,6 @@ def gen_postman_collection(filename='timetracker-api-postman-collection.json',
     save_data(parsed_json, filename)
 
 
-@cli_manager.command
-def seed():
-    from time_tracker_api.database import seeder as seed
-    seed()
-
-
-@cli_manager.command
-def re_create_db():
-    print('This is going to drop all tables and seed again the database')
-    confirm_answer = input('Do you confirm (Y) you want to remove all your data?\n')
-    if confirm_answer.upper() == 'Y':
-        from time_tracker_api.database import seeder
-        seeder.fresh()
-    else:
-        print('\nThis action was cancelled!')
-
-
 def save_data(data: str, filename: str) -> None:
     """ Save text content to a file """
     if filename:

commons/data_access_layer/cosmos_db.py

@@ -1,12 +1,17 @@
 import dataclasses
 import logging
 import uuid
+from datetime import datetime
 from typing import Callable
 
 import azure.cosmos.cosmos_client as cosmos_client
 import azure.cosmos.exceptions as exceptions
 from azure.cosmos import ContainerProxy, PartitionKey
 from flask import Flask
+from werkzeug.exceptions import HTTPException
+
+from commons.data_access_layer.database import CRUDDao
+from time_tracker_api.security import current_user_tenant_id, current_user_id
 
 
 class CosmosDBFacade:
@@ -72,12 +77,14 @@ class CosmosDBRepository:
     def __init__(self, container_id: str,
                  partition_key_attribute: str,
                  mapper: Callable = None,
+                 order_fields: list = [],
                  custom_cosmos_helper: CosmosDBFacade = None):
         global cosmos_helper
         self.cosmos_helper = custom_cosmos_helper or cosmos_helper
         if self.cosmos_helper is None:  # pragma: no cover
             raise ValueError("The cosmos_db module has not been initialized!")
         self.mapper = mapper
+        self.order_fields = order_fields
         self.container: ContainerProxy = self.cosmos_helper.db.get_container_client(container_id)
         self.partition_key_attribute: str = partition_key_attribute
 
@@ -90,7 +97,29 @@ def from_definition(cls, container_definition: dict,
                    mapper=mapper,
                    custom_cosmos_helper=custom_cosmos_helper)
 
+    @staticmethod
+    def create_sql_condition_for_visibility(visible_only: bool, container_name='c') -> str:
+        if visible_only:
+            # We are considering that `deleted == null` is not a choice
+            return 'AND NOT IS_DEFINED(%s.deleted)' % container_name
+        return ''
+
+    @staticmethod
+    def create_sql_condition_for_owner_id(owner_id: str, container_name='c') -> str:
+        if owner_id:
+            return 'AND %s.owner_id=@owner_id' % container_name
+        return ''
+
+    @staticmethod
+    def check_visibility(item, throw_not_found_if_deleted):
+        if throw_not_found_if_deleted and item.get('deleted') is not None:
+            raise exceptions.CosmosResourceNotFoundError(message='Deleted item',
+                                                         status_code=404)
+
+        return item
+
     def create(self, data: dict, mapper: Callable = None):
+        self.on_create(data)
         function_mapper = self.get_mapper_or_dict(mapper)
         return function_mapper(self.container.create_item(body=data))
 
@@ -99,20 +128,24 @@ def find(self, id: str, partition_key_value, visible_only=True, mapper: Callable
         function_mapper = self.get_mapper_or_dict(mapper)
         return function_mapper(self.check_visibility(found_item, visible_only))
 
-    def find_all(self, partition_key_value: str, max_count=None, offset=0,
+    def find_all(self, partition_key_value: str, owner_id=None, max_count=None, offset=0,
                  visible_only=True, mapper: Callable = None):
         # TODO Use the tenant_id param and change container alias
         max_count = self.get_page_size_or(max_count)
         result = self.container.query_items(
             query="""
-            SELECT * FROM c WHERE c.{partition_key_attribute}=@partition_key_value AND {visibility_condition}
+            SELECT * FROM c WHERE c.{partition_key_attribute}=@partition_key_value
+            {owner_condition} {visibility_condition}  {order_clause}
             OFFSET @offset LIMIT @max_count
             """.format(partition_key_attribute=self.partition_key_attribute,
-                       visibility_condition=self.create_sql_condition_for_visibility(visible_only)),
+                       visibility_condition=self.create_sql_condition_for_visibility(visible_only),
+                       owner_condition=self.create_sql_condition_for_owner_id(owner_id),
+                       order_clause=self.create_sql_order_clause()),
             parameters=[
                 {"name": "@partition_key_value", "value": partition_key_value},
                 {"name": "@offset", "value": offset},
                 {"name": "@max_count", "value": max_count},
+                {"name": "@owner_id", "value": owner_id},
             ],
             partition_key=partition_key_value,
             max_item_count=max_count)
@@ -122,11 +155,12 @@ def find_all(self, partition_key_value: str, max_count=None, offset=0,
 
     def partial_update(self, id: str, changes: dict, partition_key_value: str,
                        visible_only=True, mapper: Callable = None):
-        item_data = self.find(id, partition_key_value, visible_only=visible_only)
+        item_data = self.find(id, partition_key_value, visible_only=visible_only, mapper=dict)
         item_data.update(changes)
         return self.update(id, item_data, mapper=mapper)
 
     def update(self, id: str, item_data: dict, mapper: Callable = None):
+        self.on_update(item_data)
         function_mapper = self.get_mapper_or_dict(mapper)
         return function_mapper(self.container.replace_item(id, body=item_data))
 
@@ -138,19 +172,6 @@ def delete(self, id: str, partition_key_value: str, mapper: Callable = None):
     def delete_permanently(self, id: str, partition_key_value: str) -> None:
         self.container.delete_item(id, partition_key_value)
 
-    def check_visibility(self, item, throw_not_found_if_deleted):
-        if throw_not_found_if_deleted and item.get('deleted') is not None:
-            raise exceptions.CosmosResourceNotFoundError(message='Deleted item',
-                                                         status_code=404)
-
-        return item
-
-    def create_sql_condition_for_visibility(self, visible_only: bool, container_name='c') -> str:
-        if visible_only:
-            # We are considering that `deleted == null` is not a choice
-            return 'NOT IS_DEFINED(%s.deleted)' % container_name
-        return 'true'
-
     def get_mapper_or_dict(self, alternative_mapper: Callable) -> Callable:
         return alternative_mapper or self.mapper or dict
 
@@ -159,6 +180,68 @@ def get_page_size_or(self, custom_page_size: int) -> int:
         # or any other repository for the settings
         return custom_page_size or 100
 
+    def on_create(self, new_item_data: dict):
+        if new_item_data.get('id') is None:
+            new_item_data['id'] = str(uuid.uuid4())
+
+    def on_update(self, update_item_data: dict):
+        pass
+
+    def create_sql_order_clause(self):
+        if len(self.order_fields) > 0:
+            return "ORDER BY c.{}".format(", c.".join(self.order_fields))
+        else:
+            return ""
+
+
+class CosmosDBDao(CRUDDao):
+    def __init__(self, repository: CosmosDBRepository):
+        self.repository = repository
+
+    @property
+    def partition_key_value(self):
+        return current_user_tenant_id()
+
+    def get_all(self) -> list:
+        tenant_id: str = self.partition_key_value
+        owner_id = current_user_id()
+        return self.repository.find_all(partition_key_value=tenant_id, owner_id=owner_id)
+
+    def get(self, id):
+        tenant_id: str = self.partition_key_value
+        return self.repository.find(id, partition_key_value=tenant_id)
+
+    def create(self, data: dict):
+        data[self.repository.partition_key_attribute] = self.partition_key_value
+        data['owner_id'] = current_user_id()
+        return self.repository.create(data)
+
+    def update(self, id, data: dict):
+        return self.repository.partial_update(id,
+                                              changes=data,
+                                              partition_key_value=self.partition_key_value)
+
+    def delete(self, id):
+        tenant_id: str = current_user_tenant_id()
+        self.repository.delete(id, partition_key_value=tenant_id)
+
+
+class CustomError(HTTPException):
+    def __init__(self, status_code: int, description: str = None):
+        self.code = status_code
+        self.description = description
+
+
+def current_datetime():
+    return datetime.utcnow()
+
+
+def datetime_str(value: datetime):
+    if value is not None:
+        return value.isoformat()
+    else:
+        return None
+
 
 def init_app(app: Flask) -> None:
     global cosmos_helper

time_tracker_api/database.py renamed to commons/data_access_layer/database.py

@@ -35,45 +35,16 @@ def delete(self, id):
         raise NotImplementedError  # pragma: no cover
 
 
-class Seeder(abc.ABC):
-    @abc.abstractmethod
-    def run(self):
-        raise NotImplementedError  # pragma: no cover
-
-    @abc.abstractmethod
-    def fresh(self):
-        raise NotImplementedError  # pragma: no cover
-
-    def __call__(self, *args, **kwargs):
-        self.run()  # pragma: no cover
-
-
-seeder: Seeder = None
-
-
 def init_app(app: Flask) -> None:
-    init_sql(app)
+    init_sql(app)  # TODO Delete after the migration to Cosmos DB has finished.
+    init_cosmos_db(app)
 
 
 def init_sql(app: Flask) -> None:
-    from commons.data_access_layer.sql import init_app, SQLSeeder
+    from commons.data_access_layer.sql import init_app
     init_app(app)
-    global seeder
-    seeder = SQLSeeder()
 
 
 def init_cosmos_db(app: Flask) -> None:
-    # from commons.data_access_layer.azure.cosmos_db import cosmos_helper
-    class CosmosSeeder(Seeder):
-        def run(self):
-            print("Provisioning namespace(database)...")
-            # cosmos_helper.create_container()
-            print("Database seeded!")
-
-        def fresh(self):
-            print("Removing namespace(database)...")
-            # cosmos_helper.remove_container()
-            self.run()
-
-    global seeder
-    seeder = CosmosSeeder()
+    from commons.data_access_layer.cosmos_db import init_app
+    init_app(app)

commons/data_access_layer/sql.py

@@ -3,7 +3,7 @@
 from flask import Flask
 from flask_sqlalchemy import SQLAlchemy
 
-from time_tracker_api.database import CRUDDao, Seeder, ID_MAX_LENGTH
+from commons.data_access_layer.database import CRUDDao, ID_MAX_LENGTH
 from time_tracker_api.security import current_user_id
 
 db: SQLAlchemy = None
@@ -14,7 +14,7 @@ def handle_commit_issues(f):
     def rollback_if_necessary(*args, **kw):
         try:
             return f(*args, **kw)
-        except: # pragma: no cover
+        except:  # pragma: no cover
             db.session.rollback()
             raise
 
@@ -90,16 +90,3 @@ def update(self, id, data: dict):
 
     def delete(self, id):
         self.repository.remove(id)
-
-
-class SQLSeeder(Seeder): # pragma: no cover
-    def run(self):
-        print("Provisioning database...")
-        db.create_all()
-        print("Database seeded!")
-
-    def fresh(self):
-        print("Removing all existing data...")
-        db.drop_all()
-
-        self.run()

migrations/__init__.py

@@ -74,4 +74,4 @@ def remove_migration(self, name):
         self.repository.delete_permanently(name, self.app_id)
 
 
-configure(storage=CosmosDBStorage("migrations", "time-tracker-api"))
+configure(storage=CosmosDBStorage("migration", "time-tracker-api"))

requirements/azure_cosmos.txt

@@ -11,4 +11,7 @@ idna==2.8
 six==1.13.0
 urllib3==1.25.7
 virtualenv==16.7.9
-virtualenv-clone==0.5.3
+virtualenv-clone==0.5.3
+
+# Dataclasses
+dataclasses==0.6

requirements/migrations.txt

@@ -3,4 +3,4 @@
 # For running any kind of data migration
 
 # Migration tool
-migrate-anything==0.1.6
+migrate-anything==0.1.6