From 6f83435db6307c8f36de20f9105d3e0de6289da1 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Fri, 5 Nov 2021 15:27:19 +0000 Subject: [PATCH 01/16] feat: file stream from azure blob storage --- commons/data_access_layer/file_stream.py | 27 +++++++++++++++++++ .../data_access_layer/file_stream_test.py | 15 +++++++++++ 2 files changed, 42 insertions(+) create mode 100644 commons/data_access_layer/file_stream.py create mode 100644 tests/commons/data_access_layer/file_stream_test.py diff --git a/commons/data_access_layer/file_stream.py b/commons/data_access_layer/file_stream.py new file mode 100644 index 00000000..a705c061 --- /dev/null +++ b/commons/data_access_layer/file_stream.py @@ -0,0 +1,27 @@ +import os +from azure.storage.blob.blockblobservice import BlockBlobService + +ACCOUNT_KEY = os.environ.get('AZURE_STORAGE_ACCOUNT_KEY') + +class FileStream: + def __init__(self, account_name:str, container_name:str): + """ + Initialize the FileStream object. which is used to get the file stream from Azure Blob Storage. + `account_name`: The name of the Azure Storage account. + `container_name`: The name of the Azure Storage container. + """ + self.account_name = account_name + self.container_name = container_name + self.blob_service = BlockBlobService(account_name=self.account_name, account_key=ACCOUNT_KEY) + + def get_file_stream(self, filename:str): + import tempfile + try: + local_file = tempfile.NamedTemporaryFile() + self.blob_service.get_blob_to_stream(self.container_name, filename, stream=local_file) + + local_file.seek(0) + return local_file + except Exception as e: + print(e) + return None \ No newline at end of file diff --git a/tests/commons/data_access_layer/file_stream_test.py b/tests/commons/data_access_layer/file_stream_test.py new file mode 100644 index 00000000..01d3580c --- /dev/null +++ b/tests/commons/data_access_layer/file_stream_test.py @@ -0,0 +1,15 @@ +import json + +from commons.data_access_layer.file_stream import FileStream + +fs = FileStream("storagefiles2","ioetfiles") + +def test_get_file_stream_return_file_when_enter_file_name(): + result = fs.get_file_stream("activity.json") + + assert len(json.load(result)) == 15 + +def test_get_file_stream_return_None_when_not_enter_file_name_or_incorrect_name(): + result = fs.get_file_stream("") + + assert result == None \ No newline at end of file From ee8a966aac46d5b8b3b8a12460af59788649d982 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Fri, 5 Nov 2021 15:29:02 +0000 Subject: [PATCH 02/16] refactor: add new python package in dev.txt --- requirements/time_tracker_api/dev.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements/time_tracker_api/dev.txt b/requirements/time_tracker_api/dev.txt index 9657c071..b7a6d667 100644 --- a/requirements/time_tracker_api/dev.txt +++ b/requirements/time_tracker_api/dev.txt @@ -19,4 +19,7 @@ coverage==4.5.1 # CLI tools PyInquirer==1.0.3 pyfiglet==0.7 -factory_boy==3.2.0 \ No newline at end of file +factory_boy==3.2.0 + +# azure blob storage +azure-storage-blob==2.1.0 \ No newline at end of file From be714fd4490e21746e1e01393a582585b7f1f94a Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Fri, 5 Nov 2021 15:37:04 +0000 Subject: [PATCH 03/16] feat: implement new methods to read files from blob storage --- .../activities/activities_model.py | 333 +++++++++--------- .../activities/activities_namespace.py | 244 ++++++------- 2 files changed, 294 insertions(+), 283 deletions(-) diff --git a/time_tracker_api/activities/activities_model.py b/time_tracker_api/activities/activities_model.py index cbfd0d20..08d594a4 100644 --- a/time_tracker_api/activities/activities_model.py +++ b/time_tracker_api/activities/activities_model.py @@ -1,161 +1,172 @@ -from dataclasses import dataclass - -from azure.cosmos import PartitionKey - -from commons.data_access_layer.cosmos_db import ( - CosmosDBModel, - CosmosDBDao, - CosmosDBRepository, -) -from time_tracker_api.database import CRUDDao, APICosmosDBDao -from typing import List, Callable -from commons.data_access_layer.database import EventContext -from utils.enums.status import Status -from utils.query_builder import CosmosDBQueryBuilder - - -class ActivityDao(CRUDDao): - pass - - -container_definition = { - 'id': 'activity', - 'partition_key': PartitionKey(path='/tenant_id'), - 'unique_key_policy': { - 'uniqueKeys': [ - {'paths': ['/name', '/deleted']}, - ] - }, -} - - -@dataclass() -class ActivityCosmosDBModel(CosmosDBModel): - id: str - name: str - description: str - deleted: str - status: str - tenant_id: str - - def __init__(self, data): - super(ActivityCosmosDBModel, self).__init__(data) # pragma: no cover - - def __repr__(self): - return '' % self.name # pragma: no cover - - def __str___(self): - return "the activity \"%s\"" % self.name # pragma: no cover - - -class ActivityCosmosDBRepository(CosmosDBRepository): - def __init__(self): - CosmosDBRepository.__init__( - self, - container_id=container_definition['id'], - partition_key_attribute='tenant_id', - mapper=ActivityCosmosDBModel, - ) - - def find_all_with_id_in_list( - self, - event_context: EventContext, - activity_ids: List[str], - visible_only=True, - mapper: Callable = None, - ): - query_builder = ( - CosmosDBQueryBuilder() - .add_sql_in_condition('id', activity_ids) - .add_sql_visibility_condition(visible_only) - .build() - ) - query_str = query_builder.get_query() - - tenant_id_value = self.find_partition_key_value(event_context) - result = self.container.query_items( - query=query_str, - partition_key=tenant_id_value, - ) - - function_mapper = self.get_mapper_or_dict(mapper) - return list(map(function_mapper, result)) - - def find_all( - self, - event_context: EventContext, - conditions: dict = None, - activities_id: List = None, - visible_only=True, - max_count=None, - offset=0, - mapper: Callable = None, - ): - query_builder = ( - CosmosDBQueryBuilder() - .add_sql_in_condition('id', activities_id) - .add_sql_where_equal_condition(conditions) - .add_sql_visibility_condition(visible_only) - .add_sql_limit_condition(max_count) - .add_sql_offset_condition(offset) - .build() - ) - - query_str = query_builder.get_query() - tenant_id_value = self.find_partition_key_value(event_context) - params = query_builder.get_parameters() - - result = self.container.query_items( - query=query_str, - parameters=params, - partition_key=tenant_id_value, - ) - function_mapper = self.get_mapper_or_dict(mapper) - return list(map(function_mapper, result)) - - -class ActivityCosmosDBDao(APICosmosDBDao, ActivityDao): - def __init__(self, repository): - CosmosDBDao.__init__(self, repository) - - def get_all_with_id_in_list( - self, - activity_ids, - ): - event_ctx = self.create_event_context("read-many") - return self.repository.find_all_with_id_in_list( - event_ctx, - activity_ids, - ) - - def get_all( - self, - conditions: dict = None, - activities_id: List = None, - max_count=None, - visible_only=True, - ) -> list: - event_ctx = self.create_event_context("read-many") - max_count = self.repository.get_page_size_or(max_count) - - activities = self.repository.find_all( - event_context=event_ctx, - conditions=conditions, - activities_id=activities_id, - visible_only=visible_only, - max_count=max_count, - ) - return activities - - def create(self, activity_payload: dict): - event_ctx = self.create_event_context('create') - activity_payload['status'] = Status.ACTIVE.value - return self.repository.create( - data=activity_payload, event_context=event_ctx - ) - - -def create_dao() -> ActivityDao: - repository = ActivityCosmosDBRepository() - - return ActivityCosmosDBDao(repository) +from dataclasses import dataclass + +from azure.cosmos import PartitionKey + +from commons.data_access_layer.cosmos_db import ( + CosmosDBModel, + CosmosDBDao, + CosmosDBRepository, +) +from time_tracker_api.database import CRUDDao, APICosmosDBDao +from typing import List, Callable +from commons.data_access_layer.database import EventContext +from utils.enums.status import Status +from utils.query_builder import CosmosDBQueryBuilder +from commons.data_access_layer.file_stream import FileStream + +class ActivityDao(CRUDDao): + pass + + +container_definition = { + 'id': 'activity', + 'partition_key': PartitionKey(path='/tenant_id'), + 'unique_key_policy': { + 'uniqueKeys': [ + {'paths': ['/name', '/deleted']}, + ] + }, +} + + +@dataclass() +class ActivityCosmosDBModel(CosmosDBModel): + id: str + name: str + description: str + deleted: str + status: str + tenant_id: str + + def __init__(self, data): + super(ActivityCosmosDBModel, self).__init__(data) # pragma: no cover + + def __repr__(self): + return '' % self.name # pragma: no cover + + def __str___(self): + return "the activity \"%s\"" % self.name # pragma: no cover + + +class ActivityCosmosDBRepository(CosmosDBRepository): + def __init__(self): + CosmosDBRepository.__init__( + self, + container_id=container_definition['id'], + partition_key_attribute='tenant_id', + mapper=ActivityCosmosDBModel, + ) + + def find_all_with_id_in_list( + self, + event_context: EventContext, + activity_ids: List[str], + visible_only=True, + mapper: Callable = None, + ): + query_builder = ( + CosmosDBQueryBuilder() + .add_sql_in_condition('id', activity_ids) + .add_sql_visibility_condition(visible_only) + .build() + ) + query_str = query_builder.get_query() + + tenant_id_value = self.find_partition_key_value(event_context) + result = self.container.query_items( + query=query_str, + partition_key=tenant_id_value, + ) + + function_mapper = self.get_mapper_or_dict(mapper) + return list(map(function_mapper, result)) + + def find_all( + self, + event_context: EventContext, + conditions: dict = None, + activities_id: List = None, + visible_only=True, + max_count=None, + offset=0, + mapper: Callable = None, + ): + query_builder = ( + CosmosDBQueryBuilder() + .add_sql_in_condition('id', activities_id) + .add_sql_where_equal_condition(conditions) + .add_sql_visibility_condition(visible_only) + .add_sql_limit_condition(max_count) + .add_sql_offset_condition(offset) + .build() + ) + + query_str = query_builder.get_query() + tenant_id_value = self.find_partition_key_value(event_context) + params = query_builder.get_parameters() + + result = self.container.query_items( + query=query_str, + parameters=params, + partition_key=tenant_id_value, + ) + function_mapper = self.get_mapper_or_dict(mapper) + return list(map(function_mapper, result)) + + def find_all_from_blob_storage(self,conditions: dict = None, mapper: Callable = None): + import json + fs = FileStream("storagefiles2","ioetfiles") + result = fs.get_file_stream("activity.json") + + function_mapper = self.get_mapper_or_dict(mapper) + return list(map(function_mapper, json.load(result))) + +class ActivityCosmosDBDao(APICosmosDBDao, ActivityDao): + def __init__(self, repository): + CosmosDBDao.__init__(self, repository) + + def get_all_with_id_in_list( + self, + activity_ids, + ): + event_ctx = self.create_event_context("read-many") + return self.repository.find_all_with_id_in_list( + event_ctx, + activity_ids, + ) + + def get_all_old( + self, + conditions: dict = None, + activities_id: List = None, + max_count=None, + visible_only=True, + ) -> list: + event_ctx = self.create_event_context("read-many") + max_count = self.repository.get_page_size_or(max_count) + + activities = self.repository.find_all( + event_context=event_ctx, + conditions=conditions, + activities_id=activities_id, + visible_only=visible_only, + max_count=max_count, + ) + return activities + + def get_all(self,conditions: dict = None) -> list: + activities = self.repository.find_all_from_blob_storage() + return activities + + def create(self, activity_payload: dict): + event_ctx = self.create_event_context('create') + activity_payload['status'] = Status.ACTIVE.value + return self.repository.create( + data=activity_payload, event_context=event_ctx + ) + + +def create_dao() -> ActivityDao: + repository = ActivityCosmosDBRepository() + + return ActivityCosmosDBDao(repository) diff --git a/time_tracker_api/activities/activities_namespace.py b/time_tracker_api/activities/activities_namespace.py index df3c104a..6b8b59ea 100644 --- a/time_tracker_api/activities/activities_namespace.py +++ b/time_tracker_api/activities/activities_namespace.py @@ -1,122 +1,122 @@ -from faker import Faker -from flask_restplus import fields, Resource -from flask_restplus._http import HTTPStatus - -from time_tracker_api.activities.activities_model import create_dao -from time_tracker_api.api import ( - common_fields, - api, - remove_required_constraint, - NullableString, -) -from utils.enums.status import Status - -faker = Faker() - -ns = api.namespace( - 'activities', description='Namespace of the API for activities' -) - -# Activity Model -activity_input = ns.model( - 'ActivityInput', - { - 'name': fields.String( - required=True, - title='Name', - max_length=50, - description='Canonical name of the activity', - example=faker.word(['Development', 'Training']), - ), - 'description': NullableString( - title='Description', - required=False, - description='Comments about the activity', - example=faker.paragraph(), - ), - 'status': fields.String( - required=False, - title='Status', - description='Status active or inactive activities', - example=Faker().words( - 2, - [ - Status.ACTIVE.value, - Status.INACTIVE.value, - ], - unique=True, - ), - ), - }, -) - -activity_response_fields = {} -activity_response_fields.update(common_fields) - -activity = ns.inherit('Activity', activity_input, activity_response_fields) - -activity_dao = create_dao() - -list_activities_attribs_parser = ns.parser() -list_activities_attribs_parser.add_argument( - 'status', - required=False, - store_missing=False, - help="(Filter) Permits to get a list of active or inactive activities.", - location='args', -) - - -@ns.route('') -class Activities(Resource): - @ns.doc('list_activities') - @ns.marshal_list_with(activity) - @ns.expect(list_activities_attribs_parser) - def get(self): - """List all activities""" - conditions = list_activities_attribs_parser.parse_args() - return activity_dao.get_all(conditions=conditions) - - @ns.doc('create_activity') - @ns.response(HTTPStatus.CONFLICT, 'This activity already exists') - @ns.response( - HTTPStatus.BAD_REQUEST, - 'Invalid format or structure of the attributes of the activity', - ) - @ns.expect(activity_input) - @ns.marshal_with(activity, code=HTTPStatus.CREATED) - def post(self): - """Create an activity""" - return activity_dao.create(ns.payload), HTTPStatus.CREATED - - -@ns.route('/') -@ns.response(HTTPStatus.NOT_FOUND, 'Activity not found') -@ns.response(HTTPStatus.UNPROCESSABLE_ENTITY, 'The id has an invalid format') -@ns.param('id', 'The activity identifier') -class Activity(Resource): - @ns.doc('get_activity') - @ns.marshal_with(activity) - def get(self, id): - """Get an activity""" - return activity_dao.get(id) - - @ns.doc('update_activity') - @ns.expect(remove_required_constraint(activity_input)) - @ns.response( - HTTPStatus.BAD_REQUEST, 'Invalid format or structure of the activity' - ) - @ns.response( - HTTPStatus.CONFLICT, 'An activity already exists with this new data' - ) - @ns.marshal_with(activity) - def put(self, id): - """Update an activity""" - return activity_dao.update(id, ns.payload) - - @ns.doc('delete_activity') - @ns.response(HTTPStatus.NO_CONTENT, 'Activity deleted successfully') - def delete(self, id): - """Delete an activity""" - activity_dao.update(id, {'status': Status.INACTIVE.value}) - return None, HTTPStatus.NO_CONTENT +from faker import Faker +from flask_restplus import fields, Resource +from flask_restplus._http import HTTPStatus + +from time_tracker_api.activities.activities_model import create_dao +from time_tracker_api.api import ( + common_fields, + api, + remove_required_constraint, + NullableString, +) +from utils.enums.status import Status + +faker = Faker() + +ns = api.namespace( + 'activities', description='Namespace of the API for activities' +) + +# Activity Model +activity_input = ns.model( + 'ActivityInput', + { + 'name': fields.String( + required=True, + title='Name', + max_length=50, + description='Canonical name of the activity', + example=faker.word(['Development', 'Training']), + ), + 'description': NullableString( + title='Description', + required=False, + description='Comments about the activity', + example=faker.paragraph(), + ), + 'status': fields.String( + required=False, + title='Status', + description='Status active or inactive activities', + example=Faker().words( + 2, + [ + Status.ACTIVE.value, + Status.INACTIVE.value, + ], + unique=True, + ), + ), + }, +) + +activity_response_fields = {} +activity_response_fields.update(common_fields) + +activity = ns.inherit('Activity', activity_input, activity_response_fields) + +activity_dao = create_dao() + +list_activities_attribs_parser = ns.parser() +list_activities_attribs_parser.add_argument( + 'status', + required=False, + store_missing=False, + help="(Filter) Permits to get a list of active or inactive activities.", + location='args', +) + + +@ns.route('') +class Activities(Resource): + @ns.doc('list_activities') + @ns.marshal_list_with(activity) + @ns.expect(list_activities_attribs_parser) + def get(self): + """List all activities""" + conditions = list_activities_attribs_parser.parse_args() + return activity_dao.get_all(conditions=conditions) + + @ns.doc('create_activity') + @ns.response(HTTPStatus.CONFLICT, 'This activity already exists') + @ns.response( + HTTPStatus.BAD_REQUEST, + 'Invalid format or structure of the attributes of the activity', + ) + @ns.expect(activity_input) + @ns.marshal_with(activity, code=HTTPStatus.CREATED) + def post(self): + """Create an activity""" + return activity_dao.create(ns.payload), HTTPStatus.CREATED + + +@ns.route('/') +@ns.response(HTTPStatus.NOT_FOUND, 'Activity not found') +@ns.response(HTTPStatus.UNPROCESSABLE_ENTITY, 'The id has an invalid format') +@ns.param('id', 'The activity identifier') +class Activity(Resource): + @ns.doc('get_activity') + @ns.marshal_with(activity) + def get(self, id): + """Get an activity""" + return activity_dao.get(id) + + @ns.doc('update_activity') + @ns.expect(remove_required_constraint(activity_input)) + @ns.response( + HTTPStatus.BAD_REQUEST, 'Invalid format or structure of the activity' + ) + @ns.response( + HTTPStatus.CONFLICT, 'An activity already exists with this new data' + ) + @ns.marshal_with(activity) + def put(self, id): + """Update an activity""" + return activity_dao.update(id, ns.payload) + + @ns.doc('delete_activity') + @ns.response(HTTPStatus.NO_CONTENT, 'Activity deleted successfully') + def delete(self, id): + """Delete an activity""" + activity_dao.update(id, {'status': Status.INACTIVE.value}) + return None, HTTPStatus.NO_CONTENT From da9c1b342344c454d1f61aa71f445da9ca17530c Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Fri, 5 Nov 2021 23:22:13 +0000 Subject: [PATCH 04/16] feat: implemented the reading of the blob storage to the endpoint activity --- .../activities/activities_model.py | 20 +++++++++++-------- .../activities/activities_namespace.py | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/time_tracker_api/activities/activities_model.py b/time_tracker_api/activities/activities_model.py index 08d594a4..adf36aa1 100644 --- a/time_tracker_api/activities/activities_model.py +++ b/time_tracker_api/activities/activities_model.py @@ -113,13 +113,16 @@ def find_all( function_mapper = self.get_mapper_or_dict(mapper) return list(map(function_mapper, result)) - def find_all_from_blob_storage(self,conditions: dict = None, mapper: Callable = None): + def find_all_from_blob_storage(self, event_context: EventContext, mapper: Callable = None): + tenant_id_value = self.find_partition_key_value(event_context) + function_mapper = self.get_mapper_or_dict(mapper) + if tenant_id_value is None: + return [] + import json - fs = FileStream("storagefiles2","ioetfiles") + fs = FileStream("storageaccounteystr82c5","tt-common-files") result = fs.get_file_stream("activity.json") - - function_mapper = self.get_mapper_or_dict(mapper) - return list(map(function_mapper, json.load(result))) + return list(map(function_mapper, json.load(result))) if result is not None else [] class ActivityCosmosDBDao(APICosmosDBDao, ActivityDao): def __init__(self, repository): @@ -154,9 +157,10 @@ def get_all_old( ) return activities - def get_all(self,conditions: dict = None) -> list: - activities = self.repository.find_all_from_blob_storage() - return activities + def get_all(self, conditions: dict = None) -> list: + event_ctx = self.create_event_context("read-many") + activities = self.repository.find_all_from_blob_storage(event_context=event_ctx) + return activities def create(self, activity_payload: dict): event_ctx = self.create_event_context('create') diff --git a/time_tracker_api/activities/activities_namespace.py b/time_tracker_api/activities/activities_namespace.py index 6b8b59ea..d6d4e807 100644 --- a/time_tracker_api/activities/activities_namespace.py +++ b/time_tracker_api/activities/activities_namespace.py @@ -75,7 +75,7 @@ class Activities(Resource): def get(self): """List all activities""" conditions = list_activities_attribs_parser.parse_args() - return activity_dao.get_all(conditions=conditions) + return activity_dao.get_all(conditions) @ns.doc('create_activity') @ns.response(HTTPStatus.CONFLICT, 'This activity already exists') From 36cd17d2d3853019d062c153a0f14c0e8583f155 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Mon, 8 Nov 2021 03:12:28 +0000 Subject: [PATCH 05/16] fix: TT-384 Change blob storage connection input names --- tests/commons/data_access_layer/file_stream_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commons/data_access_layer/file_stream_test.py b/tests/commons/data_access_layer/file_stream_test.py index 01d3580c..bab4c160 100644 --- a/tests/commons/data_access_layer/file_stream_test.py +++ b/tests/commons/data_access_layer/file_stream_test.py @@ -2,10 +2,10 @@ from commons.data_access_layer.file_stream import FileStream -fs = FileStream("storagefiles2","ioetfiles") +fs = FileStream("storageaccounteystr82c5","tt-common-files") def test_get_file_stream_return_file_when_enter_file_name(): - result = fs.get_file_stream("activity.json") + result = fs.get_file_stream("activity_test.json") assert len(json.load(result)) == 15 From 52a4359d84434dfa9086c809d966a31578296404 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Mon, 8 Nov 2021 03:15:58 +0000 Subject: [PATCH 06/16] fix: TT-384 Add the file name as a parameter of the function --- time_tracker_api/activities/activities_model.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/time_tracker_api/activities/activities_model.py b/time_tracker_api/activities/activities_model.py index adf36aa1..4969bce3 100644 --- a/time_tracker_api/activities/activities_model.py +++ b/time_tracker_api/activities/activities_model.py @@ -113,7 +113,12 @@ def find_all( function_mapper = self.get_mapper_or_dict(mapper) return list(map(function_mapper, result)) - def find_all_from_blob_storage(self, event_context: EventContext, mapper: Callable = None): + def find_all_from_blob_storage( + self, + event_context: EventContext, + mapper: Callable = None, + file_name: str = "activity.json", + ): tenant_id_value = self.find_partition_key_value(event_context) function_mapper = self.get_mapper_or_dict(mapper) if tenant_id_value is None: @@ -121,7 +126,7 @@ def find_all_from_blob_storage(self, event_context: EventContext, mapper: Callab import json fs = FileStream("storageaccounteystr82c5","tt-common-files") - result = fs.get_file_stream("activity.json") + result = fs.get_file_stream(file_name) return list(map(function_mapper, json.load(result))) if result is not None else [] class ActivityCosmosDBDao(APICosmosDBDao, ActivityDao): From 94d196a1fd7c34ebd9a5c39c8ea502041214a4a0 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Mon, 8 Nov 2021 03:20:28 +0000 Subject: [PATCH 07/16] test: TT-384 Add a tests to obtain activities from blob storage, endpoint and repository --- .../activities/activities_model_test.py | 13 +++++++++++++ .../activities/activities_namespace_test.py | 14 +++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/time_tracker_api/activities/activities_model_test.py b/tests/time_tracker_api/activities/activities_model_test.py index c1a1b243..77ead767 100644 --- a/tests/time_tracker_api/activities/activities_model_test.py +++ b/tests/time_tracker_api/activities/activities_model_test.py @@ -64,3 +64,16 @@ def test_create_activity_should_add_active_status( activity_repository_create_mock.assert_called_with( data=expect_argument, event_context=ANY ) + + +def test_find_all_from_blob_storage( + event_context: EventContext, + activity_repository: ActivityCosmosDBRepository, +): + activity_repository.container = Mock() + + result = activity_repository.find_all_from_blob_storage( + event_context=event_context, + file_name="activity_test.json" + ) + assert len(result) == 15 diff --git a/tests/time_tracker_api/activities/activities_namespace_test.py b/tests/time_tracker_api/activities/activities_namespace_test.py index a2b9ab20..d9a13a53 100644 --- a/tests/time_tracker_api/activities/activities_namespace_test.py +++ b/tests/time_tracker_api/activities/activities_namespace_test.py @@ -1,5 +1,5 @@ from unittest.mock import ANY - +import pytest from faker import Faker from flask import json from flask.testing import FlaskClient @@ -18,6 +18,14 @@ fake_activity = ({"id": fake.random_int(1, 9999)}).update(valid_activity_data) +def test_get_all_activities_return_list_activities_when_send_get_request( + client: FlaskClient, valid_header: dict +): + response = client.get( + "/activities", headers=valid_header, follow_redirects=True + ) + + assert HTTPStatus.OK == response.status_code def test_create_activity_should_succeed_with_valid_request( client: FlaskClient, mocker: MockFixture, valid_header: dict @@ -55,7 +63,7 @@ def test_create_activity_should_reject_bad_request( assert HTTPStatus.BAD_REQUEST == response.status_code repository_create_mock.assert_not_called() - +@pytest.mark.skip(reason="There is currently no way to test this. Getting the value of the azure blob storage") def test_list_all_active( client: FlaskClient, mocker: MockFixture, valid_header: dict ): @@ -81,7 +89,7 @@ def test_list_all_active( max_count=ANY, ) - +@pytest.mark.skip(reason="There is currently no way to test this. Getting the value of the azure blob storage") def test_list_all_active_activities( client: FlaskClient, mocker: MockFixture, valid_header: dict ): From 169853a8df89d6579ce9d437258b219b18de8268 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Mon, 8 Nov 2021 15:38:47 +0000 Subject: [PATCH 08/16] fix: TT-384 revert changes --- .../data_access_layer/file_stream_test.py | 4 +- .../activities/activities_model_test.py | 13 - .../activities/activities_namespace_test.py | 14 +- .../activities/activities_model.py | 342 +++++++++--------- .../activities/activities_namespace.py | 244 ++++++------- 5 files changed, 288 insertions(+), 329 deletions(-) diff --git a/tests/commons/data_access_layer/file_stream_test.py b/tests/commons/data_access_layer/file_stream_test.py index bab4c160..01d3580c 100644 --- a/tests/commons/data_access_layer/file_stream_test.py +++ b/tests/commons/data_access_layer/file_stream_test.py @@ -2,10 +2,10 @@ from commons.data_access_layer.file_stream import FileStream -fs = FileStream("storageaccounteystr82c5","tt-common-files") +fs = FileStream("storagefiles2","ioetfiles") def test_get_file_stream_return_file_when_enter_file_name(): - result = fs.get_file_stream("activity_test.json") + result = fs.get_file_stream("activity.json") assert len(json.load(result)) == 15 diff --git a/tests/time_tracker_api/activities/activities_model_test.py b/tests/time_tracker_api/activities/activities_model_test.py index 77ead767..c1a1b243 100644 --- a/tests/time_tracker_api/activities/activities_model_test.py +++ b/tests/time_tracker_api/activities/activities_model_test.py @@ -64,16 +64,3 @@ def test_create_activity_should_add_active_status( activity_repository_create_mock.assert_called_with( data=expect_argument, event_context=ANY ) - - -def test_find_all_from_blob_storage( - event_context: EventContext, - activity_repository: ActivityCosmosDBRepository, -): - activity_repository.container = Mock() - - result = activity_repository.find_all_from_blob_storage( - event_context=event_context, - file_name="activity_test.json" - ) - assert len(result) == 15 diff --git a/tests/time_tracker_api/activities/activities_namespace_test.py b/tests/time_tracker_api/activities/activities_namespace_test.py index d9a13a53..a2b9ab20 100644 --- a/tests/time_tracker_api/activities/activities_namespace_test.py +++ b/tests/time_tracker_api/activities/activities_namespace_test.py @@ -1,5 +1,5 @@ from unittest.mock import ANY -import pytest + from faker import Faker from flask import json from flask.testing import FlaskClient @@ -18,14 +18,6 @@ fake_activity = ({"id": fake.random_int(1, 9999)}).update(valid_activity_data) -def test_get_all_activities_return_list_activities_when_send_get_request( - client: FlaskClient, valid_header: dict -): - response = client.get( - "/activities", headers=valid_header, follow_redirects=True - ) - - assert HTTPStatus.OK == response.status_code def test_create_activity_should_succeed_with_valid_request( client: FlaskClient, mocker: MockFixture, valid_header: dict @@ -63,7 +55,7 @@ def test_create_activity_should_reject_bad_request( assert HTTPStatus.BAD_REQUEST == response.status_code repository_create_mock.assert_not_called() -@pytest.mark.skip(reason="There is currently no way to test this. Getting the value of the azure blob storage") + def test_list_all_active( client: FlaskClient, mocker: MockFixture, valid_header: dict ): @@ -89,7 +81,7 @@ def test_list_all_active( max_count=ANY, ) -@pytest.mark.skip(reason="There is currently no way to test this. Getting the value of the azure blob storage") + def test_list_all_active_activities( client: FlaskClient, mocker: MockFixture, valid_header: dict ): diff --git a/time_tracker_api/activities/activities_model.py b/time_tracker_api/activities/activities_model.py index 4969bce3..cbfd0d20 100644 --- a/time_tracker_api/activities/activities_model.py +++ b/time_tracker_api/activities/activities_model.py @@ -1,181 +1,161 @@ -from dataclasses import dataclass - -from azure.cosmos import PartitionKey - -from commons.data_access_layer.cosmos_db import ( - CosmosDBModel, - CosmosDBDao, - CosmosDBRepository, -) -from time_tracker_api.database import CRUDDao, APICosmosDBDao -from typing import List, Callable -from commons.data_access_layer.database import EventContext -from utils.enums.status import Status -from utils.query_builder import CosmosDBQueryBuilder -from commons.data_access_layer.file_stream import FileStream - -class ActivityDao(CRUDDao): - pass - - -container_definition = { - 'id': 'activity', - 'partition_key': PartitionKey(path='/tenant_id'), - 'unique_key_policy': { - 'uniqueKeys': [ - {'paths': ['/name', '/deleted']}, - ] - }, -} - - -@dataclass() -class ActivityCosmosDBModel(CosmosDBModel): - id: str - name: str - description: str - deleted: str - status: str - tenant_id: str - - def __init__(self, data): - super(ActivityCosmosDBModel, self).__init__(data) # pragma: no cover - - def __repr__(self): - return '' % self.name # pragma: no cover - - def __str___(self): - return "the activity \"%s\"" % self.name # pragma: no cover - - -class ActivityCosmosDBRepository(CosmosDBRepository): - def __init__(self): - CosmosDBRepository.__init__( - self, - container_id=container_definition['id'], - partition_key_attribute='tenant_id', - mapper=ActivityCosmosDBModel, - ) - - def find_all_with_id_in_list( - self, - event_context: EventContext, - activity_ids: List[str], - visible_only=True, - mapper: Callable = None, - ): - query_builder = ( - CosmosDBQueryBuilder() - .add_sql_in_condition('id', activity_ids) - .add_sql_visibility_condition(visible_only) - .build() - ) - query_str = query_builder.get_query() - - tenant_id_value = self.find_partition_key_value(event_context) - result = self.container.query_items( - query=query_str, - partition_key=tenant_id_value, - ) - - function_mapper = self.get_mapper_or_dict(mapper) - return list(map(function_mapper, result)) - - def find_all( - self, - event_context: EventContext, - conditions: dict = None, - activities_id: List = None, - visible_only=True, - max_count=None, - offset=0, - mapper: Callable = None, - ): - query_builder = ( - CosmosDBQueryBuilder() - .add_sql_in_condition('id', activities_id) - .add_sql_where_equal_condition(conditions) - .add_sql_visibility_condition(visible_only) - .add_sql_limit_condition(max_count) - .add_sql_offset_condition(offset) - .build() - ) - - query_str = query_builder.get_query() - tenant_id_value = self.find_partition_key_value(event_context) - params = query_builder.get_parameters() - - result = self.container.query_items( - query=query_str, - parameters=params, - partition_key=tenant_id_value, - ) - function_mapper = self.get_mapper_or_dict(mapper) - return list(map(function_mapper, result)) - - def find_all_from_blob_storage( - self, - event_context: EventContext, - mapper: Callable = None, - file_name: str = "activity.json", - ): - tenant_id_value = self.find_partition_key_value(event_context) - function_mapper = self.get_mapper_or_dict(mapper) - if tenant_id_value is None: - return [] - - import json - fs = FileStream("storageaccounteystr82c5","tt-common-files") - result = fs.get_file_stream(file_name) - return list(map(function_mapper, json.load(result))) if result is not None else [] - -class ActivityCosmosDBDao(APICosmosDBDao, ActivityDao): - def __init__(self, repository): - CosmosDBDao.__init__(self, repository) - - def get_all_with_id_in_list( - self, - activity_ids, - ): - event_ctx = self.create_event_context("read-many") - return self.repository.find_all_with_id_in_list( - event_ctx, - activity_ids, - ) - - def get_all_old( - self, - conditions: dict = None, - activities_id: List = None, - max_count=None, - visible_only=True, - ) -> list: - event_ctx = self.create_event_context("read-many") - max_count = self.repository.get_page_size_or(max_count) - - activities = self.repository.find_all( - event_context=event_ctx, - conditions=conditions, - activities_id=activities_id, - visible_only=visible_only, - max_count=max_count, - ) - return activities - - def get_all(self, conditions: dict = None) -> list: - event_ctx = self.create_event_context("read-many") - activities = self.repository.find_all_from_blob_storage(event_context=event_ctx) - return activities - - def create(self, activity_payload: dict): - event_ctx = self.create_event_context('create') - activity_payload['status'] = Status.ACTIVE.value - return self.repository.create( - data=activity_payload, event_context=event_ctx - ) - - -def create_dao() -> ActivityDao: - repository = ActivityCosmosDBRepository() - - return ActivityCosmosDBDao(repository) +from dataclasses import dataclass + +from azure.cosmos import PartitionKey + +from commons.data_access_layer.cosmos_db import ( + CosmosDBModel, + CosmosDBDao, + CosmosDBRepository, +) +from time_tracker_api.database import CRUDDao, APICosmosDBDao +from typing import List, Callable +from commons.data_access_layer.database import EventContext +from utils.enums.status import Status +from utils.query_builder import CosmosDBQueryBuilder + + +class ActivityDao(CRUDDao): + pass + + +container_definition = { + 'id': 'activity', + 'partition_key': PartitionKey(path='/tenant_id'), + 'unique_key_policy': { + 'uniqueKeys': [ + {'paths': ['/name', '/deleted']}, + ] + }, +} + + +@dataclass() +class ActivityCosmosDBModel(CosmosDBModel): + id: str + name: str + description: str + deleted: str + status: str + tenant_id: str + + def __init__(self, data): + super(ActivityCosmosDBModel, self).__init__(data) # pragma: no cover + + def __repr__(self): + return '' % self.name # pragma: no cover + + def __str___(self): + return "the activity \"%s\"" % self.name # pragma: no cover + + +class ActivityCosmosDBRepository(CosmosDBRepository): + def __init__(self): + CosmosDBRepository.__init__( + self, + container_id=container_definition['id'], + partition_key_attribute='tenant_id', + mapper=ActivityCosmosDBModel, + ) + + def find_all_with_id_in_list( + self, + event_context: EventContext, + activity_ids: List[str], + visible_only=True, + mapper: Callable = None, + ): + query_builder = ( + CosmosDBQueryBuilder() + .add_sql_in_condition('id', activity_ids) + .add_sql_visibility_condition(visible_only) + .build() + ) + query_str = query_builder.get_query() + + tenant_id_value = self.find_partition_key_value(event_context) + result = self.container.query_items( + query=query_str, + partition_key=tenant_id_value, + ) + + function_mapper = self.get_mapper_or_dict(mapper) + return list(map(function_mapper, result)) + + def find_all( + self, + event_context: EventContext, + conditions: dict = None, + activities_id: List = None, + visible_only=True, + max_count=None, + offset=0, + mapper: Callable = None, + ): + query_builder = ( + CosmosDBQueryBuilder() + .add_sql_in_condition('id', activities_id) + .add_sql_where_equal_condition(conditions) + .add_sql_visibility_condition(visible_only) + .add_sql_limit_condition(max_count) + .add_sql_offset_condition(offset) + .build() + ) + + query_str = query_builder.get_query() + tenant_id_value = self.find_partition_key_value(event_context) + params = query_builder.get_parameters() + + result = self.container.query_items( + query=query_str, + parameters=params, + partition_key=tenant_id_value, + ) + function_mapper = self.get_mapper_or_dict(mapper) + return list(map(function_mapper, result)) + + +class ActivityCosmosDBDao(APICosmosDBDao, ActivityDao): + def __init__(self, repository): + CosmosDBDao.__init__(self, repository) + + def get_all_with_id_in_list( + self, + activity_ids, + ): + event_ctx = self.create_event_context("read-many") + return self.repository.find_all_with_id_in_list( + event_ctx, + activity_ids, + ) + + def get_all( + self, + conditions: dict = None, + activities_id: List = None, + max_count=None, + visible_only=True, + ) -> list: + event_ctx = self.create_event_context("read-many") + max_count = self.repository.get_page_size_or(max_count) + + activities = self.repository.find_all( + event_context=event_ctx, + conditions=conditions, + activities_id=activities_id, + visible_only=visible_only, + max_count=max_count, + ) + return activities + + def create(self, activity_payload: dict): + event_ctx = self.create_event_context('create') + activity_payload['status'] = Status.ACTIVE.value + return self.repository.create( + data=activity_payload, event_context=event_ctx + ) + + +def create_dao() -> ActivityDao: + repository = ActivityCosmosDBRepository() + + return ActivityCosmosDBDao(repository) diff --git a/time_tracker_api/activities/activities_namespace.py b/time_tracker_api/activities/activities_namespace.py index d6d4e807..df3c104a 100644 --- a/time_tracker_api/activities/activities_namespace.py +++ b/time_tracker_api/activities/activities_namespace.py @@ -1,122 +1,122 @@ -from faker import Faker -from flask_restplus import fields, Resource -from flask_restplus._http import HTTPStatus - -from time_tracker_api.activities.activities_model import create_dao -from time_tracker_api.api import ( - common_fields, - api, - remove_required_constraint, - NullableString, -) -from utils.enums.status import Status - -faker = Faker() - -ns = api.namespace( - 'activities', description='Namespace of the API for activities' -) - -# Activity Model -activity_input = ns.model( - 'ActivityInput', - { - 'name': fields.String( - required=True, - title='Name', - max_length=50, - description='Canonical name of the activity', - example=faker.word(['Development', 'Training']), - ), - 'description': NullableString( - title='Description', - required=False, - description='Comments about the activity', - example=faker.paragraph(), - ), - 'status': fields.String( - required=False, - title='Status', - description='Status active or inactive activities', - example=Faker().words( - 2, - [ - Status.ACTIVE.value, - Status.INACTIVE.value, - ], - unique=True, - ), - ), - }, -) - -activity_response_fields = {} -activity_response_fields.update(common_fields) - -activity = ns.inherit('Activity', activity_input, activity_response_fields) - -activity_dao = create_dao() - -list_activities_attribs_parser = ns.parser() -list_activities_attribs_parser.add_argument( - 'status', - required=False, - store_missing=False, - help="(Filter) Permits to get a list of active or inactive activities.", - location='args', -) - - -@ns.route('') -class Activities(Resource): - @ns.doc('list_activities') - @ns.marshal_list_with(activity) - @ns.expect(list_activities_attribs_parser) - def get(self): - """List all activities""" - conditions = list_activities_attribs_parser.parse_args() - return activity_dao.get_all(conditions) - - @ns.doc('create_activity') - @ns.response(HTTPStatus.CONFLICT, 'This activity already exists') - @ns.response( - HTTPStatus.BAD_REQUEST, - 'Invalid format or structure of the attributes of the activity', - ) - @ns.expect(activity_input) - @ns.marshal_with(activity, code=HTTPStatus.CREATED) - def post(self): - """Create an activity""" - return activity_dao.create(ns.payload), HTTPStatus.CREATED - - -@ns.route('/') -@ns.response(HTTPStatus.NOT_FOUND, 'Activity not found') -@ns.response(HTTPStatus.UNPROCESSABLE_ENTITY, 'The id has an invalid format') -@ns.param('id', 'The activity identifier') -class Activity(Resource): - @ns.doc('get_activity') - @ns.marshal_with(activity) - def get(self, id): - """Get an activity""" - return activity_dao.get(id) - - @ns.doc('update_activity') - @ns.expect(remove_required_constraint(activity_input)) - @ns.response( - HTTPStatus.BAD_REQUEST, 'Invalid format or structure of the activity' - ) - @ns.response( - HTTPStatus.CONFLICT, 'An activity already exists with this new data' - ) - @ns.marshal_with(activity) - def put(self, id): - """Update an activity""" - return activity_dao.update(id, ns.payload) - - @ns.doc('delete_activity') - @ns.response(HTTPStatus.NO_CONTENT, 'Activity deleted successfully') - def delete(self, id): - """Delete an activity""" - activity_dao.update(id, {'status': Status.INACTIVE.value}) - return None, HTTPStatus.NO_CONTENT +from faker import Faker +from flask_restplus import fields, Resource +from flask_restplus._http import HTTPStatus + +from time_tracker_api.activities.activities_model import create_dao +from time_tracker_api.api import ( + common_fields, + api, + remove_required_constraint, + NullableString, +) +from utils.enums.status import Status + +faker = Faker() + +ns = api.namespace( + 'activities', description='Namespace of the API for activities' +) + +# Activity Model +activity_input = ns.model( + 'ActivityInput', + { + 'name': fields.String( + required=True, + title='Name', + max_length=50, + description='Canonical name of the activity', + example=faker.word(['Development', 'Training']), + ), + 'description': NullableString( + title='Description', + required=False, + description='Comments about the activity', + example=faker.paragraph(), + ), + 'status': fields.String( + required=False, + title='Status', + description='Status active or inactive activities', + example=Faker().words( + 2, + [ + Status.ACTIVE.value, + Status.INACTIVE.value, + ], + unique=True, + ), + ), + }, +) + +activity_response_fields = {} +activity_response_fields.update(common_fields) + +activity = ns.inherit('Activity', activity_input, activity_response_fields) + +activity_dao = create_dao() + +list_activities_attribs_parser = ns.parser() +list_activities_attribs_parser.add_argument( + 'status', + required=False, + store_missing=False, + help="(Filter) Permits to get a list of active or inactive activities.", + location='args', +) + + +@ns.route('') +class Activities(Resource): + @ns.doc('list_activities') + @ns.marshal_list_with(activity) + @ns.expect(list_activities_attribs_parser) + def get(self): + """List all activities""" + conditions = list_activities_attribs_parser.parse_args() + return activity_dao.get_all(conditions=conditions) + + @ns.doc('create_activity') + @ns.response(HTTPStatus.CONFLICT, 'This activity already exists') + @ns.response( + HTTPStatus.BAD_REQUEST, + 'Invalid format or structure of the attributes of the activity', + ) + @ns.expect(activity_input) + @ns.marshal_with(activity, code=HTTPStatus.CREATED) + def post(self): + """Create an activity""" + return activity_dao.create(ns.payload), HTTPStatus.CREATED + + +@ns.route('/') +@ns.response(HTTPStatus.NOT_FOUND, 'Activity not found') +@ns.response(HTTPStatus.UNPROCESSABLE_ENTITY, 'The id has an invalid format') +@ns.param('id', 'The activity identifier') +class Activity(Resource): + @ns.doc('get_activity') + @ns.marshal_with(activity) + def get(self, id): + """Get an activity""" + return activity_dao.get(id) + + @ns.doc('update_activity') + @ns.expect(remove_required_constraint(activity_input)) + @ns.response( + HTTPStatus.BAD_REQUEST, 'Invalid format or structure of the activity' + ) + @ns.response( + HTTPStatus.CONFLICT, 'An activity already exists with this new data' + ) + @ns.marshal_with(activity) + def put(self, id): + """Update an activity""" + return activity_dao.update(id, ns.payload) + + @ns.doc('delete_activity') + @ns.response(HTTPStatus.NO_CONTENT, 'Activity deleted successfully') + def delete(self, id): + """Delete an activity""" + activity_dao.update(id, {'status': Status.INACTIVE.value}) + return None, HTTPStatus.NO_CONTENT From a6ee691b795bfb719ea84f5a09b78acc4488cded Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Mon, 8 Nov 2021 16:11:24 +0000 Subject: [PATCH 09/16] test: TT-384 Change blob storage connection input names --- tests/commons/data_access_layer/file_stream_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commons/data_access_layer/file_stream_test.py b/tests/commons/data_access_layer/file_stream_test.py index 01d3580c..bab4c160 100644 --- a/tests/commons/data_access_layer/file_stream_test.py +++ b/tests/commons/data_access_layer/file_stream_test.py @@ -2,10 +2,10 @@ from commons.data_access_layer.file_stream import FileStream -fs = FileStream("storagefiles2","ioetfiles") +fs = FileStream("storageaccounteystr82c5","tt-common-files") def test_get_file_stream_return_file_when_enter_file_name(): - result = fs.get_file_stream("activity.json") + result = fs.get_file_stream("activity_test.json") assert len(json.load(result)) == 15 From 054798b568c99e3faa576f65e9726d031480ad48 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Mon, 8 Nov 2021 16:30:37 +0000 Subject: [PATCH 10/16] feat: TT-384 implemented the reading of the storage blob to the endpoint and repository --- .../activities/activities_model.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/time_tracker_api/activities/activities_model.py b/time_tracker_api/activities/activities_model.py index cbfd0d20..ddc1d4db 100644 --- a/time_tracker_api/activities/activities_model.py +++ b/time_tracker_api/activities/activities_model.py @@ -12,7 +12,7 @@ from commons.data_access_layer.database import EventContext from utils.enums.status import Status from utils.query_builder import CosmosDBQueryBuilder - +from commons.data_access_layer.file_stream import FileStream class ActivityDao(CRUDDao): pass @@ -113,6 +113,21 @@ def find_all( function_mapper = self.get_mapper_or_dict(mapper) return list(map(function_mapper, result)) + def find_all_from_blob_storage( + self, + event_context: EventContext, + mapper: Callable = None, + file_name: str = "activity.json", + ): + tenant_id_value = self.find_partition_key_value(event_context) + function_mapper = self.get_mapper_or_dict(mapper) + if tenant_id_value is None: + return [] + + import json + fs = FileStream("storageaccounteystr82c5","tt-common-files") + result = fs.get_file_stream(file_name) + return list(map(function_mapper, json.load(result))) if result is not None else [] class ActivityCosmosDBDao(APICosmosDBDao, ActivityDao): def __init__(self, repository): @@ -128,7 +143,7 @@ def get_all_with_id_in_list( activity_ids, ) - def get_all( + def get_all_old( self, conditions: dict = None, activities_id: List = None, @@ -147,6 +162,11 @@ def get_all( ) return activities + def get_all(self, conditions: dict = None) -> list: + event_ctx = self.create_event_context("read-many") + activities = self.repository.find_all_from_blob_storage(event_context=event_ctx) + return activities + def create(self, activity_payload: dict): event_ctx = self.create_event_context('create') activity_payload['status'] = Status.ACTIVE.value From afd537def26539c8a324f3afb5bb01d6eb5e7478 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Mon, 8 Nov 2021 16:39:44 +0000 Subject: [PATCH 11/16] test: TT-384 Add a tests to obtain activities from blob storage, endpoint and repository --- .../activities/activities_model_test.py | 12 ++++++++++++ .../activities/activities_namespace_test.py | 13 +++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/time_tracker_api/activities/activities_model_test.py b/tests/time_tracker_api/activities/activities_model_test.py index c1a1b243..aec15ef2 100644 --- a/tests/time_tracker_api/activities/activities_model_test.py +++ b/tests/time_tracker_api/activities/activities_model_test.py @@ -64,3 +64,15 @@ def test_create_activity_should_add_active_status( activity_repository_create_mock.assert_called_with( data=expect_argument, event_context=ANY ) + +def test_find_all_from_blob_storage( + event_context: EventContext, + activity_repository: ActivityCosmosDBRepository, +): + activity_repository.container = Mock() + + result = activity_repository.find_all_from_blob_storage( + event_context=event_context, + file_name="activity_test.json" + ) + assert len(result) == 15 diff --git a/tests/time_tracker_api/activities/activities_namespace_test.py b/tests/time_tracker_api/activities/activities_namespace_test.py index a2b9ab20..4b8b2746 100644 --- a/tests/time_tracker_api/activities/activities_namespace_test.py +++ b/tests/time_tracker_api/activities/activities_namespace_test.py @@ -4,6 +4,7 @@ from flask import json from flask.testing import FlaskClient from flask_restplus._http import HTTPStatus +import pytest from pytest_mock import MockFixture from utils.enums.status import Status @@ -18,6 +19,14 @@ fake_activity = ({"id": fake.random_int(1, 9999)}).update(valid_activity_data) +def test_get_all_activities_return_list_activities_when_send_get_request( + client: FlaskClient, valid_header: dict +): + response = client.get( + "/activities", headers=valid_header, follow_redirects=True + ) + + assert HTTPStatus.OK == response.status_code def test_create_activity_should_succeed_with_valid_request( client: FlaskClient, mocker: MockFixture, valid_header: dict @@ -55,7 +64,7 @@ def test_create_activity_should_reject_bad_request( assert HTTPStatus.BAD_REQUEST == response.status_code repository_create_mock.assert_not_called() - +@pytest.mark.skip(reason="There is currently no way to test this. Getting the value of the azure blob storage") def test_list_all_active( client: FlaskClient, mocker: MockFixture, valid_header: dict ): @@ -81,7 +90,7 @@ def test_list_all_active( max_count=ANY, ) - +@pytest.mark.skip(reason="There is currently no way to test this. Getting the value of the azure blob storage") def test_list_all_active_activities( client: FlaskClient, mocker: MockFixture, valid_header: dict ): From f9532ea8d5d4356ebd360f532dce116444fd15d9 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Tue, 9 Nov 2021 21:23:46 +0000 Subject: [PATCH 12/16] test: TT-384 changed test name with correct formatting --- .../commons/data_access_layer/file_stream_test.py | 4 ++-- .../activities/activities_model_test.py | 14 +++++++++++++- .../activities/activities_namespace_test.py | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/commons/data_access_layer/file_stream_test.py b/tests/commons/data_access_layer/file_stream_test.py index bab4c160..a3119774 100644 --- a/tests/commons/data_access_layer/file_stream_test.py +++ b/tests/commons/data_access_layer/file_stream_test.py @@ -4,12 +4,12 @@ fs = FileStream("storageaccounteystr82c5","tt-common-files") -def test_get_file_stream_return_file_when_enter_file_name(): +def test__get_file_stream__return_file_content__when_enter_file_name(): result = fs.get_file_stream("activity_test.json") assert len(json.load(result)) == 15 -def test_get_file_stream_return_None_when_not_enter_file_name_or_incorrect_name(): +def test__get_file_stream__return_None__when_not_enter_file_name_or_incorrect_name(): result = fs.get_file_stream("") assert result == None \ No newline at end of file diff --git a/tests/time_tracker_api/activities/activities_model_test.py b/tests/time_tracker_api/activities/activities_model_test.py index aec15ef2..66e08ed7 100644 --- a/tests/time_tracker_api/activities/activities_model_test.py +++ b/tests/time_tracker_api/activities/activities_model_test.py @@ -65,7 +65,7 @@ def test_create_activity_should_add_active_status( data=expect_argument, event_context=ANY ) -def test_find_all_from_blob_storage( +def test__find_all_from_blob_storage__return_list__when_send_event_context_and_correct_file_name( event_context: EventContext, activity_repository: ActivityCosmosDBRepository, ): @@ -76,3 +76,15 @@ def test_find_all_from_blob_storage( file_name="activity_test.json" ) assert len(result) == 15 + +def test__find_all_from_blob_storage__return_empty_list__when_send_event_context_and_incorrect_file_name( + event_context: EventContext, + activity_repository: ActivityCosmosDBRepository, +): + activity_repository.container = Mock() + + result = activity_repository.find_all_from_blob_storage( + event_context=event_context, + file_name="incorrect.json" + ) + assert result == [] \ No newline at end of file diff --git a/tests/time_tracker_api/activities/activities_namespace_test.py b/tests/time_tracker_api/activities/activities_namespace_test.py index 4b8b2746..86e34691 100644 --- a/tests/time_tracker_api/activities/activities_namespace_test.py +++ b/tests/time_tracker_api/activities/activities_namespace_test.py @@ -19,7 +19,7 @@ fake_activity = ({"id": fake.random_int(1, 9999)}).update(valid_activity_data) -def test_get_all_activities_return_list_activities_when_send_get_request( +def test__get_all_activities__return_response__when_send_activities_get_request( client: FlaskClient, valid_header: dict ): response = client.get( From e3480cc5479870218ffdc168d7ed3c231f1ad7b5 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Wed, 10 Nov 2021 16:57:07 +0000 Subject: [PATCH 13/16] refactor: TT-384 change import to global and name method --- time_tracker_api/activities/activities_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/time_tracker_api/activities/activities_model.py b/time_tracker_api/activities/activities_model.py index ddc1d4db..ddb46411 100644 --- a/time_tracker_api/activities/activities_model.py +++ b/time_tracker_api/activities/activities_model.py @@ -1,5 +1,6 @@ from dataclasses import dataclass +import json from azure.cosmos import PartitionKey from commons.data_access_layer.cosmos_db import ( @@ -124,7 +125,6 @@ def find_all_from_blob_storage( if tenant_id_value is None: return [] - import json fs = FileStream("storageaccounteystr82c5","tt-common-files") result = fs.get_file_stream(file_name) return list(map(function_mapper, json.load(result))) if result is not None else [] @@ -143,7 +143,7 @@ def get_all_with_id_in_list( activity_ids, ) - def get_all_old( + def get_all_v1( self, conditions: dict = None, activities_id: List = None, From abe096ee7d6d0dcf2846734d141a39827ca0aa7a Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Wed, 10 Nov 2021 17:19:42 +0000 Subject: [PATCH 14/16] refactor: change import json to global --- time_tracker_api/activities/activities_model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/time_tracker_api/activities/activities_model.py b/time_tracker_api/activities/activities_model.py index ddc1d4db..7002be1d 100644 --- a/time_tracker_api/activities/activities_model.py +++ b/time_tracker_api/activities/activities_model.py @@ -1,5 +1,5 @@ from dataclasses import dataclass - +import json from azure.cosmos import PartitionKey from commons.data_access_layer.cosmos_db import ( @@ -124,7 +124,6 @@ def find_all_from_blob_storage( if tenant_id_value is None: return [] - import json fs = FileStream("storageaccounteystr82c5","tt-common-files") result = fs.get_file_stream(file_name) return list(map(function_mapper, json.load(result))) if result is not None else [] From f2a48a029cee3ea59f94107cce7300487dbdedf9 Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Fri, 12 Nov 2021 00:19:54 +0000 Subject: [PATCH 15/16] fix: TT-384 add package azure blob storage to prod.txt --- requirements/time_tracker_api/prod.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements/time_tracker_api/prod.txt b/requirements/time_tracker_api/prod.txt index 77ed3a0a..dd6df0df 100644 --- a/requirements/time_tracker_api/prod.txt +++ b/requirements/time_tracker_api/prod.txt @@ -44,4 +44,7 @@ azure-functions-worker==1.1.9 # Time utils pytz==2019.3 -python-dateutil==2.8.1 \ No newline at end of file +python-dateutil==2.8.1 + +# azure blob storage +azure-storage-blob==2.1.0 \ No newline at end of file From f0867d01148e84a7803c9dd6ba94da6bbe0e097a Mon Sep 17 00:00:00 2001 From: Jipson Murillo <38593785+Jobzi@users.noreply.github.com> Date: Fri, 12 Nov 2021 20:06:08 +0000 Subject: [PATCH 16/16] test: TT-384 reset original endpoint --- time_tracker_api/activities/activities_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/time_tracker_api/activities/activities_model.py b/time_tracker_api/activities/activities_model.py index 3daa35fb..9643586a 100644 --- a/time_tracker_api/activities/activities_model.py +++ b/time_tracker_api/activities/activities_model.py @@ -142,7 +142,7 @@ def get_all_with_id_in_list( activity_ids, ) - def get_all_v1( + def get_all( self, conditions: dict = None, activities_id: List = None, @@ -161,7 +161,7 @@ def get_all_v1( ) return activities - def get_all(self, conditions: dict = None) -> list: + def get_all_qa(self, conditions: dict = None) -> list: event_ctx = self.create_event_context("read-many") activities = self.repository.find_all_from_blob_storage(event_context=event_ctx) return activities