Tt 265 cosmosdb emulator (#300)

* feat: TT-265 Support for docker-compose and cosmosdb emulator

Co-authored-by: lenshinoda <[email protected]>

Author: diegotony

.env.template

@@ -20,4 +20,7 @@ export MS_CLIENT_ID=
 export MS_SCOPE=
 export MS_SECRET=
 export MS_ENDPOINT=
-export AZURE_APP_CONFIGURATION_CONNECTION_STRING=
+export AZURE_APP_CONFIGURATION_CONNECTION_STRING=
+
+## If you are running on Docker compose:
+export REQUESTS_CA_BUNDLE=/etc/ssl/certs/

.gitignore

@@ -39,3 +39,6 @@ migration_status.csv
 
 # windows env variables
 env.*
+
+# SSL certificate for cosmos emulator
+emulatorcert.crt

Dockerfile.dev

@@ -0,0 +1,28 @@
+FROM python:3.9-alpine
+ARG buildDeps='g++ gnupg libffi-dev openssl-dev musl-dev cargo'
+WORKDIR /usr/src/app
+COPY . .
+RUN apk update \
+	&& apk add --no-cache $buildDeps gcc unixodbc-dev  ca-certificates curl \
+	&& pip3 install --no-cache-dir -r requirements/time_tracker_api/dev.txt \
+	&& curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.2.1-1_amd64.apk \
+	&& curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_17.5.2.1-1_amd64.apk \
+	&& curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.2.1-1_amd64.sig \
+	&& curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_17.5.2.1-1_amd64.sig \
+	&& curl https://packages.microsoft.com/keys/microsoft.asc  | gpg --import - \
+	&& gpg --verify msodbcsql17_17.5.2.1-1_amd64.sig msodbcsql17_17.5.2.1-1_amd64.apk \
+	&& gpg --verify mssql-tools_17.5.2.1-1_amd64.sig mssql-tools_17.5.2.1-1_amd64.apk \
+	&& apk add --no-cache --allow-untrusted msodbcsql17_17.5.2.1-1_amd64.apk \
+	&& apk add --no-cache --allow-untrusted mssql-tools_17.5.2.1-1_amd64.apk \
+	&& rm msodbcsql17_17.5.2.1-1_amd64.apk mssql-tools_17.5.2.1-1_amd64.apk msodbcsql17_17.5.2.1-1_amd64.sig mssql-tools_17.5.2.1-1_amd64.sig \
+	&& apk del $buildDeps \
+	&& rm -rfv /root/.cache/pip/* && \
+find /usr/local \( -type d -a -name test -o -name tests \) -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) -exec rm -rfv '{}' \+
+RUN apk add --no-cache --upgrade bash
+ENV FLASK_APP time_tracker_api
+ENV FLASK_RUN_HOST 0.0.0.0
+
+EXPOSE 5000
+
+CMD ["flask","run"]
+

README.md

@@ -438,6 +438,13 @@ They will be automatically run during the Continuous Deployment process.
 Shared file with all the Feature Toggles we create, so we can have a history of them
 [Feature Toggles dictionary](https://github.com/ioet/time-tracker-ui/wiki/Feature-Toggles-dictionary)
 
+## Support for docker-compose and cosmosdb emulator
+
+To run the dev enviroment in docker-compose:
+```bash
+docker-compose up
+```
+
 ## More information about the project
 
 [Starting in Time Tracker](https://github.com/ioet/time-tracker-ui/wiki/Time-tracker)

cosmosdb-emulator/entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+until curl -ksf "${DATABASE_ACCOUNT_URI}/_explorer/emulator.pem" -o 'cosmosdb-emulator/emulatorcert.crt'; do
+    echo "Waiting for Cosmosdb to start..."
+    sleep 10
+done
+
+echo "Container cosmosemulator started."
+
+echo "Importing SSL..."
+cp cosmosdb-emulator/emulatorcert.crt /usr/local/share/ca-certificates/
+cp cosmosdb-emulator/emulatorcert.crt /usr/share/ca-certificates/
+update-ca-certificates --fresh
+echo "Importing Containers..."
+export REQUESTS_CA_BUNDLE=/etc/ssl/certs/
+python3 ./cosmosdb-emulator/init_emulator_db.py
+echo "Installation succeed!!"
+
+echo "Starting Flask!!"
+flask run

cosmosdb-emulator/init_emulator.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+containerId=$(docker ps --all | grep 'Time-Tracker-Cosmos-Db' | awk '{print $1}')
+if [ -z "$containerId" ]; then
+    ipaddr="`ifconfig | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}' | head -n 1`"
+    containerId=$(docker create -p 8081:8081 -p 10251:10251 -p 10252:10252 -p 10253:10253 -p 10254:10254  -m 3g --cpus=2.0 --name=Time-Tracker-Cosmos-Db -e AZURE_COSMOS_EMULATOR_PARTITION_COUNT=10 -e AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE=true -e AZURE_COSMOS_EMULATOR_IP_ADDRESS_OVERRIDE=$ipaddr -it mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator)
+    echo "##vso[task.setvariable variable=cosmosDbContainerId]$containerId"> /dev/tty
+fi
+docker start $containerId
+
+until curl -ksf "127.0.0.1:8081/_explorer/emulator.pem" -o 'cosmosdb-emulator/emulatorcert.crt'; do
+    echo "Waiting for Cosmosdb to start..."
+    sleep 10
+done
+
+echo "Container cosmosemulator started."
+
+echo "Checking SSL"
+isInstalled=$( awk -v cmd='openssl x509 -noout -subject' '/BEGIN/{close(cmd)};{print | cmd}' < /etc/ssl/certs/ca-certificates.crt | grep host ) || :
+
+echo "ps"
+echo "$isInstalled"
+
+if [ -z "$isInstalled" ]; then
+    echo "Importing SSL..."
+    cp cosmosdb-emulator/emulatorcert.crt /usr/local/share/ca-certificates/
+    cp cosmosdb-emulator/emulatorcert.crt /usr/share/ca-certificates/
+    update-ca-certificates --fresh
+    echo "Importing Containers..."
+    export REQUESTS_CA_BUNDLE=/etc/ssl/certs/
+    python3 ./cosmosdb-emulator/init_emulator_db.py
+    echo "Installation succeed!!"
+fi
+
+echo "Starting Flask!!" 
+flask run

cosmosdb-emulator/init_emulator_db.py

@@ -0,0 +1,49 @@
+from azure.cosmos import exceptions, CosmosClient, PartitionKey
+import os, sys
+  
+sys.path.append("/usr/src/app")
+
+DATABASE_ACCOUNT_URI = os.environ.get('DATABASE_ACCOUNT_URI')
+DATABASE_MASTER_KEY = os.environ.get('DATABASE_MASTER_KEY')
+
+endpoint = DATABASE_ACCOUNT_URI
+key = DATABASE_MASTER_KEY
+
+# <create_cosmos_client>
+client = CosmosClient(endpoint, key)
+# <create_database_if_not_exists>
+database_name = 'time-tracker-db'
+database = client.create_database_if_not_exists(id=database_name)
+# </create_containers_if_not_exists>
+
+print("Creating TimeTracker initial initial database schema...")
+
+try:
+    print('- Project')
+    from time_tracker_api.projects.projects_model import container_definition as project_definition
+    database.create_container_if_not_exists(**project_definition)
+
+    print('- Project type')
+    from time_tracker_api.project_types.project_types_model import container_definition as project_type_definition
+    database.create_container_if_not_exists(**project_type_definition)
+
+    print('- Activity')
+    from time_tracker_api.activities.activities_model import container_definition as activity_definition
+    database.create_container_if_not_exists(**activity_definition)
+
+    print('- Customer')
+    from time_tracker_api.customers.customers_model import container_definition as customer_definition
+    database.create_container_if_not_exists(**customer_definition)
+
+    print('- Time entry')
+    from time_tracker_api.time_entries.time_entries_model import container_definition as time_entry_definition
+    database.create_container_if_not_exists(**time_entry_definition)
+
+    print('- Technology')
+    from time_tracker_api.technologies.technologies_model import container_definition as technologies_definition
+    database.create_container_if_not_exists(**technologies_definition)
+except exceptions.CosmosResourceExistsError as e:
+    print("Unexpected error while creating initial database schema: %s" % e.message)
+
+print("Done!")
+

docker-compose.yml

@@ -0,0 +1,50 @@
+version: '3.8'
+
+services:
+  api:
+    container_name: "time-tracker-backend_api"
+    build: 
+        context: .
+        dockerfile: ./Dockerfile.dev
+    ports:
+      - 5000:5000
+    env_file:
+      - .env
+    volumes: 
+      - .:/usr/src/app
+    # depends_on:
+    #   - cosmosdb
+    entrypoint:
+      - /bin/sh
+      - ./cosmosdb-emulator/entrypoint.sh
+    # networks:
+    #   services_net:
+    #     ipv4_address: 172.20.0.77
+
+  cosmosdb:
+    container_name: "azurecosmosemulator"
+    hostname: "azurecosmosemulator"
+    image: 'mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator'
+    tty: true
+    mem_limit: 3GB
+    ports:
+        - '8081:8081'
+        - '10251:10251'
+        - '10252:10252'
+        - '10253:10253'
+        - '10254:10254'
+    environment:
+        - AZURE_COSMOS_EMULATOR_PARTITION_COUNT=7
+        - AZURE_COSMOS_EMULATOR_ARGS=/alternativenames=azurecosmosemulator
+
+    # networks:
+    #   services_net:
+    #     ipv4_address: 172.20.0.78
+
+
+# networks:
+#   services_net:
+#     ipam:
+#       driver: default
+#       config:
+#         - subnet: 172.20.0.0/16

run-test.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+PYTHON_COMMAND="pip install azure-functions"
+PYTHON_TEST="python3 -m pytest -v --ignore=tests/commons/data_access_layer/azure/sql_repository_test.py"
+API_CONTAINER_NAME="time-tracker-backend_api"
+execute(){
+    
+    docker exec -ti $API_CONTAINER_NAME sh -c "$PYTHON_COMMAND"
+    docker exec -ti $API_CONTAINER_NAME sh -c "awk -v cmd='openssl x509 -noout -subject' '/BEGIN/{close(cmd)};{print | cmd}' < /etc/ssl/certs/ca-certificates.crt | grep host"
+    docker exec -ti $API_CONTAINER_NAME sh -c "env"
+    docker exec -ti $API_CONTAINER_NAME sh -c "$PYTHON_TEST"
+
+}
+
+
+execute