From 8cb7719117e80e0e03e53ee1a2cf32f99c4bdc77 Mon Sep 17 00:00:00 2001 From: Mike Williamson Date: Fri, 11 Jun 2021 16:23:57 -0400 Subject: [PATCH] Add staging overlays This commit adds the overlays needed for an AKS based staging environment. This should be enough to get Tracker going: ```sh make secrets # env=dev is the default make deploy env=staging ``` --- app/staging/README.md | 55 +++++++++++++++++++ app/staging/arangodb-deployment.yaml | 17 ++++++ app/staging/certificate.yaml | 20 +++++++ app/staging/knative/config/queues.yaml | 49 +++++++++++++++++ app/staging/kustomization.yaml | 23 ++++++++ app/staging/publicgateway.yaml | 38 +++++++++++++ app/staging/storage-classes.yaml | 50 +++++++++++++++++ deploy/staging/app-kustomization.yaml | 9 +++ deploy/staging/kustomization.yaml | 7 +++ deploy/staging/platform-kustomization.yaml | 8 +++ platform/staging/README.md | 3 + platform/staging/domain-config.yaml | 14 +++++ .../staging/istio-ingressgateway-service.yaml | 7 +++ platform/staging/kustomization.yaml | 18 ++++++ 14 files changed, 318 insertions(+) create mode 100644 app/staging/README.md create mode 100644 app/staging/arangodb-deployment.yaml create mode 100644 app/staging/certificate.yaml create mode 100644 app/staging/knative/config/queues.yaml create mode 100644 app/staging/kustomization.yaml create mode 100644 app/staging/publicgateway.yaml create mode 100644 app/staging/storage-classes.yaml create mode 100644 deploy/staging/app-kustomization.yaml create mode 100644 deploy/staging/kustomization.yaml create mode 100644 deploy/staging/platform-kustomization.yaml create mode 100644 platform/staging/README.md create mode 100644 platform/staging/domain-config.yaml create mode 100644 platform/staging/istio-ingressgateway-service.yaml create mode 100644 platform/staging/kustomization.yaml diff --git a/app/staging/README.md b/app/staging/README.md new file mode 100644 index 000000000..d0bcaa701 --- /dev/null +++ b/app/staging/README.md @@ -0,0 +1,55 @@ +# Staging on AKS + +The purpose of this overlay is to bring up Tracker on AKS in a Staging config. The config here is just a draft. We're just getting started! + +If you need some dev credentials for this test cluster, you can generate them with `make credentials mode=dev`. + +## Bringing up a cluster on AKS + +Normally we would create the cluster with `make cluster`, but we haven't gotten an AKS equivalent worked out yet. In the absence of that, we'll just assume you already have a 6 node cluster somehow, and `kubectl` configured to talk to it. + +## Installing Tracker + +Just run the following commands. + +```sh +make secrets env=aks +# If either of these fail, just run them again +make platform env=staging +make app env=staging +``` +Or as an alternate path, you could install flux and let it do the work for you. + +```sh +make secrets env=aks +make deploy env=staging +``` + +If you want to watch the pod creation process, you can do it with this: + +```sh +watch kubectl get po -A +``` + +That will bring the cluster up with a self-signed certificate. To connect to it, we just need the external IP. + +```sh +kubectl get svc -n istio-system istio-ingressgateway +``` + +Connecting to both `https://` and `https:///graphql` should succeed. Reaching the frontend and API respectively. + +## Loading data + +The database isn't exposed to the outside world, so loading data requires you to forward the database ports to your local machine. + +```sh +kubectl port-forward -n db svc/arangodb 8529:8529 +``` + +With that port forwarding in place, you can now load/dump with the following commands: + +```sh +make backup to=~/track_dmarc-$(date --iso-8601) +make restore from=~/track_dmarc-2021-05-12 +``` diff --git a/app/staging/arangodb-deployment.yaml b/app/staging/arangodb-deployment.yaml new file mode 100644 index 000000000..f337749af --- /dev/null +++ b/app/staging/arangodb-deployment.yaml @@ -0,0 +1,17 @@ +kind: ArangoDeployment +apiVersion: database.arangodb.com/v1alpha +metadata: + name: arangodb + namespace: db +spec: + agents: + volumeClaimTemplate: + spec: + storageClassName: slow-retain + persistentVolumeReclaimPolicy: Retain + dbservers: + volumeClaimTemplate: + spec: + storageClassName: fast-retain + persistentVolumeReclaimPolicy: Retain + diff --git a/app/staging/certificate.yaml b/app/staging/certificate.yaml new file mode 100644 index 000000000..07455fde6 --- /dev/null +++ b/app/staging/certificate.yaml @@ -0,0 +1,20 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + creationTimestamp: null + name: ingress-cert + namespace: istio-system +spec: + commonName: tracker.canada.ca + dnsNames: + - tracker.canada.ca + - suivi.canada.ca + issuerRef: + kind: Issuer + name: letsencrypt-staging + privateKey: + algorithm: RSA + encoding: PKCS8 # ITSP.40.062 6.2 Signature Algorithms + size: 4096 + secretName: tracker-credential +status: {} diff --git a/app/staging/knative/config/queues.yaml b/app/staging/knative/config/queues.yaml new file mode 100644 index 000000000..06758d6fe --- /dev/null +++ b/app/staging/knative/config/queues.yaml @@ -0,0 +1,49 @@ +apiVersion: serving.knative.dev/v1 # Current version of Knative +kind: Service +metadata: + name: scan-queue + namespace: scanners + labels: + app: scanners +spec: + template: + metadata: + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '9090' + # Knative concurrency-based autoscaling (default). + autoscaling.knative.dev/class: kpa.autoscaling.knative.dev + autoscaling.knative.dev/metric: concurrency + autoscaling.knative.dev/minScale: "1" + autoscaling.knative.dev/maxScale: "2" + spec: + timeoutSeconds: 500 + containers: + - name: scan-queue + image: gcr.io/track-compliance/services/scan-queue + +--- + +apiVersion: serving.knative.dev/v1 # Current version of Knative +kind: Service +metadata: + name: result-queue + namespace: scanners + labels: + app: scanners +spec: + template: + metadata: + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '9090' + # Knative concurrency-based autoscaling (default). + autoscaling.knative.dev/class: kpa.autoscaling.knative.dev + autoscaling.knative.dev/metric: concurrency + autoscaling.knative.dev/minScale: "1" + autoscaling.knative.dev/maxScale: "2" + spec: + timeoutSeconds: 500 + containers: + - name: result-queue + image: gcr.io/track-compliance/services/result-queue diff --git a/app/staging/kustomization.yaml b/app/staging/kustomization.yaml new file mode 100644 index 000000000..d47874cb8 --- /dev/null +++ b/app/staging/kustomization.yaml @@ -0,0 +1,23 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../bases +- storage-classes.yaml +patchesStrategicMerge: +- arangodb-deployment.yaml +- certificate.yaml +- publicgateway.yaml +- knative/config/queues.yaml +replicas: +- count: 2 + name: tracker-frontend +- count: 2 + name: tracker-api +- count: 1 + name: arango-deployment-replication-operator +- count: 1 + name: arango-deployment-operator +- count: 1 + name: arango-storage-operator +components: +- ../namespaces diff --git a/app/staging/publicgateway.yaml b/app/staging/publicgateway.yaml new file mode 100644 index 000000000..1882cd7a0 --- /dev/null +++ b/app/staging/publicgateway.yaml @@ -0,0 +1,38 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: publicgateway + namespace: istio-system + labels: + istio: publicgateway +spec: + selector: + istio: ingressgateway + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" + tls: + httpsRedirect: true + - port: + number: 443 + name: https + protocol: HTTPS + hosts: + - "*" + tls: + mode: SIMPLE + credentialName: ingress-cert-frompfx + privateKey: sds + serverCertificate: sds + minProtocolVersion: TLSV1_2 + cipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 diff --git a/app/staging/storage-classes.yaml b/app/staging/storage-classes.yaml new file mode 100644 index 000000000..642902e86 --- /dev/null +++ b/app/staging/storage-classes.yaml @@ -0,0 +1,50 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: fast-retain +provisioner: kubernetes.io/azure-disk +reclaimPolicy: Retain +parameters: + storageaccounttype: Premium_LRS + kind: Managed +volumeBindingMode: WaitForFirstConsumer + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: slow-retain +provisioner: kubernetes.io/azure-disk +reclaimPolicy: Retain +parameters: + storageaccounttype: Standard_LRS + kind: Managed +volumeBindingMode: WaitForFirstConsumer + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: fast-delete +provisioner: kubernetes.io/azure-disk +reclaimPolicy: Delete +parameters: + storageaccounttype: Premium_LRS + kind: Managed +volumeBindingMode: WaitForFirstConsumer + +--- + +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: slow-delete +provisioner: kubernetes.io/azure-disk +reclaimPolicy: Delete +parameters: + storageaccounttype: Standard_LRS + kind: Managed +volumeBindingMode: WaitForFirstConsumer +--- diff --git a/deploy/staging/app-kustomization.yaml b/deploy/staging/app-kustomization.yaml new file mode 100644 index 000000000..8e27b08d9 --- /dev/null +++ b/deploy/staging/app-kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 +kind: Kustomization +metadata: + name: app + namespace: flux-system +spec: + path: ./app/staging + diff --git a/deploy/staging/kustomization.yaml b/deploy/staging/kustomization.yaml new file mode 100644 index 000000000..3669131f6 --- /dev/null +++ b/deploy/staging/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../bases +patchesStrategicMerge: +- app-kustomization.yaml +- platform-kustomization.yaml diff --git a/deploy/staging/platform-kustomization.yaml b/deploy/staging/platform-kustomization.yaml new file mode 100644 index 000000000..07afe60ba --- /dev/null +++ b/deploy/staging/platform-kustomization.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 +kind: Kustomization +metadata: + name: platform + namespace: flux-system +spec: + path: ./platform/staging diff --git a/platform/staging/README.md b/platform/staging/README.md new file mode 100644 index 000000000..28d52543d --- /dev/null +++ b/platform/staging/README.md @@ -0,0 +1,3 @@ +# AKS + +This directory contains the config variations needed to get the platform stuff running on AKS. diff --git a/platform/staging/domain-config.yaml b/platform/staging/domain-config.yaml new file mode 100644 index 000000000..557cb0cf2 --- /dev/null +++ b/platform/staging/domain-config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-domain + namespace: knative-serving +data: + # svc.cluster.local will be used for routes having app=prod. + svc.cluster.local: | + selector: + app: scanners + # Default value for domain, for routes that does not have app=prod labels. + # Although it will match all routes, it is the least-specific rule so it + # will only be used if no other domain matches. + example.com: "" diff --git a/platform/staging/istio-ingressgateway-service.yaml b/platform/staging/istio-ingressgateway-service.yaml new file mode 100644 index 000000000..bbd69e756 --- /dev/null +++ b/platform/staging/istio-ingressgateway-service.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-ingressgateway + namespace: istio-system + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" diff --git a/platform/staging/kustomization.yaml b/platform/staging/kustomization.yaml new file mode 100644 index 000000000..214729728 --- /dev/null +++ b/platform/staging/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../base +- domain-config.yaml +components: +- ../components/knative +patchesJSON6902: +- target: + group: install.istio.io + version: v1alpha1 + kind: IstioOperator + name: istio-controlplane + namespace: istio-operator + patch: |- + - op: add + path: /spec/components/ingressGateways/0/k8s/service/loadBalancerIP + value: 10.58.10.58