ci: merge main to release

.github/workflows/build.yml

@@ -156,109 +156,221 @@ jobs:
       with:
         fetch-depth: 1
         fetch-tags: false
-
-    - name: Setup Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: '16'
-
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.x'
-
-    - name: Download a Coverage Results
-      if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v4.1.8
-      with:
-        name: coverage
 
-    - name: Make Release Build
-      env:
-        DEBIAN_FRONTEND: noninteractive
-        BROWSERSLIST_IGNORE_OLD_DATA: 1
-      run: |
-        echo "PKG_VERSION: $PKG_VERSION"
-        echo "GITHUB_SHA: $GITHUB_SHA"
-        echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
-        echo "Running frontend build script..."
-        echo "Compiling native node packages..."
-        yarn rebuild
-        echo "Packaging static assets..."
-        yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/
-        yarn legacy:build
-        echo "Setting version $PKG_VERSION..."
-        sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py
-        sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py
-        sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py
-
-    - name: Set Production Flags
-      if: ${{ env.SHOULD_DEPLOY == 'true' }}
+    - name: Launch build VM
+      id: azlaunch
       run: |
-        echo "Setting production flags in settings.py..."
-        sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py
-
-    - name: Make Release Tarball
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      run: |
-        echo "Build release tarball..."
-        mkdir -p /home/runner/work/release
-        tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt .
+        echo "Authenticating to Azure..."
+        az login --service-principal -u ${{ secrets.AZ_BUILD_APP_ID }} -p ${{ secrets.AZ_BUILD_PWD }} --tenant ${{ secrets.AZ_BUILD_TENANT_ID }}
+        echo "Creating VM..."
+        vminfo=$(az vm create \
+          --resource-group ghaDatatracker \
+          --name tmpGhaBuildVM \
+          --image Ubuntu2204 \
+          --admin-username azureuser \
+          --generate-ssh-keys \
+          --priority Spot \
+          --size Standard_D8ads_v5 \
+          --max-price -1 \
+          --ephemeral-os-disk \
+          --os-disk-size-gb 100 \
+          --eviction-policy Delete \
+          --nic-delete-option Delete \
+          --output tsv \
+          --query "publicIpAddress")
+        echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT"
+        echo "VM Public IP: $vminfo"
+        cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key
+        ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
 
-    - name: Collect + Push Statics
+    - name: Remote SSH into Build VM
+      uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1
       env:
-        DEBIAN_FRONTEND: noninteractive
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_ACTOR: ${{ github.actor }}
+        GITHUB_SHA: ${{ github.sha }}
+        GITHUB_REF_NAME: ${{ github.ref_name }}
+        GITHUB_RUN_ID: ${{ github.run_id }}
         AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }}
         AWS_DEFAULT_REGION: auto
         AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }}
+        PKG_VERSION: ${{ env.PKG_VERSION }}
+        SHOULD_DEPLOY: ${{ env.SHOULD_DEPLOY }}
+        SKIP_TESTS: ${{ github.event.inputs.skiptests }}
+        DEBIAN_FRONTEND: noninteractive
+        BROWSERSLIST_IGNORE_OLD_DATA: 1
+      with:
+        host: ${{ steps.azlaunch.outputs.ipaddr }}
+        port: 22
+        username: azureuser
+        command_timeout: 60m
+        key_path: ${{ github.workspace }}/prvkey.key
+        envs: GITHUB_TOKEN,GITHUB_ACTOR,GITHUB_SHA,GITHUB_REF_NAME,GITHUB_RUN_ID,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_DEFAULT_REGION,AWS_ENDPOINT_URL,PKG_VERSION,SHOULD_DEPLOY,SKIP_TESTS,DEBIAN_FRONTEND,BROWSERSLIST_IGNORE_OLD_DATA
+        script_stop: true
+        script: |
+          export DEBIAN_FRONTEND=noninteractive
+          lsb_release -a
+          sudo apt-get update
+          sudo apt-get upgrade -y
+          sudo apt-get install wget unzip curl -y
+
+          echo "=========================================================================="
+          echo "Installing Docker..."
+          echo "=========================================================================="
+          curl -fsSL https://get.docker.com -o get-docker.sh
+          sudo sh get-docker.sh
+          sudo docker buildx create \
+            --name container-builder \
+            --driver docker-container \
+            --bootstrap --use
+
+          echo "=========================================================================="
+          echo "Login to ghcr.io..."
+          echo "=========================================================================="
+          echo $GITHUB_TOKEN | sudo docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
+
+          echo "=========================================================================="
+          echo "Installing GH CLI..."
+          echo "=========================================================================="
+          sudo mkdir -p -m 755 /etc/apt/keyrings \
+          	&& wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
+          	&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
+          	&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+          	&& sudo apt update \
+          	&& sudo apt install gh -y
+
+          echo "=========================================================================="
+          echo "Installing AWS CLI..."
+          echo "=========================================================================="
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install
+
+          echo "=========================================================================="
+          echo "Install Node.js..."
+          echo "=========================================================================="
+          curl -fsSL https://deb.nodesource.com/setup_18.x -o nodesource_setup.sh
+          sudo bash nodesource_setup.sh
+          sudo apt-get install -y nodejs
+          sudo corepack enable
+
+          echo "=========================================================================="
+          echo "Install Python 3.x..."
+          echo "=========================================================================="
+          sudo apt-get install python3 python3-dev -y
+          python3 --version
+
+          echo "=========================================================================="
+          echo "Clone project..."
+          echo "=========================================================================="
+          sudo mkdir -p /workspace
+          sudo chown azureuser /workspace
+          cd /workspace
+          gh repo clone ietf-tools/datatracker -- --depth=1 --no-tags
+          cd datatracker
+
+          if [ "$SKIP_TESTS" = "false" ] || [ "$GITHUB_REF_NAME" = "release" ] ; then
+            echo "=========================================================================="
+            echo "Downloading coverage..."
+            echo "=========================================================================="
+            gh run download $GITHUB_RUN_ID -n coverage
+          fi
+
+          echo "=========================================================================="
+          echo "Building project..."
+          echo "=========================================================================="
+          echo "PKG_VERSION: $PKG_VERSION"
+          echo "GITHUB_SHA: $GITHUB_SHA"
+          echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
+          echo "Running frontend build script..."
+          echo "Compiling native node packages..."
+          yarn rebuild
+          echo "Packaging static assets..."
+          yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/
+          yarn legacy:build
+          echo "Setting version $PKG_VERSION..."
+          sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py
+          sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py
+          sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py
+
+          if [ "$SHOULD_DEPLOY" = "true" ] ; then
+            echo "=========================================================================="
+            echo "Setting production flags in settings.py..."
+            echo "=========================================================================="
+            sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py
+          fi
+
+          echo "=========================================================================="
+          echo "Build release tarball..."
+          echo "=========================================================================="
+          mkdir -p /workspace/release
+          tar -czf /workspace/release.tar.gz -X dev/build/exclude-patterns.txt .
+
+          echo "=========================================================================="
+          echo "Collecting statics..."
+          echo "=========================================================================="
+          sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh
+          echo "Pushing statics..."
+          cd static
+          aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
+          cd ..
+
+          echo "=========================================================================="
+          echo "Augment dockerignore for docker image build..."
+          echo "=========================================================================="
+          cat >> .dockerignore <<EOL
+          .devcontainer
+          .github
+          .vscode
+          k8s
+          playwright
+          svn-history
+          docker-compose.yml
+          EOL
+
+          echo "=========================================================================="
+          echo "Building Images..."
+          echo "=========================================================================="
+          sudo docker buildx build --file dev/build/Dockerfile --platform linux/amd64,linux/arm64 --tag ghcr.io/ietf-tools/datatracker:$PKG_VERSION --push .
+
+    - name: Fetch release tarball
       run: |
-        echo "Collecting statics..."
-        docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh
-        echo "Pushing statics..."
-        cd static
-        aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
+        mkdir -p /home/runner/work/release
+        chmod 0600 ${{ github.workspace }}/prvkey.key
+        scp -i ${{ github.workspace }}/prvkey.key azureuser@${{ steps.azlaunch.outputs.ipaddr }}:/workspace/release.tar.gz /home/runner/work/release/release.tar.gz
 
-    - name: Augment dockerignore for docker image build
-      env:
-        DEBIAN_FRONTEND: noninteractive
+    - name: Destroy Build VM + resources
+      if: always()
+      shell: pwsh
       run: |
-        cat >> .dockerignore <<EOL
-        .devcontainer
-        .github
-        .vscode
-        k8s
-        playwright
-        svn-history
-        docker-compose.yml
-        EOL
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+        echo "Destroying VM..."        
+        az vm delete -g ghaDatatracker -n tmpGhaBuildVM --yes --force-deletion true
 
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v3
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
+        $resourceOrderRemovalOrder = [ordered]@{
+            "Microsoft.Compute/virtualMachines" = 0
+            "Microsoft.Compute/disks" = 1
+            "Microsoft.Network/networkInterfaces" = 2
+            "Microsoft.Network/publicIpAddresses" = 3
+            "Microsoft.Network/networkSecurityGroups" = 4
+            "Microsoft.Network/virtualNetworks" = 5
+        }
+        echo "Fetching remaining resources..."
+        $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
 
-    - name: Build Release Docker Image
-      uses: docker/build-push-action@v6
-      env:
-        DOCKER_BUILD_SUMMARY: false
-      with:
-        context: .
-        file: dev/build/Dockerfile
-        platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }}
-        push: true
-        tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
+        $orderedResources = $resources 
+            | Sort-Object @{
+                Expression = {$resourceOrderRemovalOrder[$_.type]}
+                Descending = $False
+            }
+
+        echo "Deleting remaining resources..."
+        $orderedResources | ForEach-Object {
+            az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
+        }
+
+        echo "Logout from Azure..."
+        az logout
 
     - name: Update CHANGELOG
       id: changelog
@@ -270,6 +382,12 @@ jobs:
         toTag: ${{ env.TO_TAG }}
         writeToFile: false
 
+    - name: Download Coverage Results
+      if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
+      uses: actions/download-artifact@v4.1.8
+      with:
+        name: coverage
+
     - name: Prepare Coverage Action
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
       working-directory: ./dev/coverage-action
@@ -335,7 +453,7 @@ jobs:
     steps:      
     - name: Notify on Slack (Success)
       if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
-      uses: slackapi/slack-github-action@v1.26.0
+      uses: slackapi/slack-github-action@v1.27.0
       with:
         channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
         payload: |
@@ -358,7 +476,7 @@ jobs:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
     - name: Notify on Slack (Failure)
       if: ${{ contains(join(needs.*.result, ','), 'failure') }}
-      uses: slackapi/slack-github-action@v1.26.0
+      uses: slackapi/slack-github-action@v1.27.0
       with:
         channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
         payload: |

.github/workflows/tests-az.yml

@@ -38,7 +38,7 @@ jobs:
         ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
 
     - name: Remote SSH into VM
-      uses: appleboy/ssh-action@029f5b4aeeeb58fdfe1410a5d17f967dacf36262
+      uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:

.github/workflows/tests.yml

@@ -59,7 +59,7 @@ jobs:
         path: geckodriver.log
 
     - name: Upload Coverage Results to Codecov
-      uses: codecov/codecov-action@v4.5.0
+      uses: codecov/codecov-action@v4.6.0
       with:
         files: coverage.xml
 

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2008-2023, The IETF Trust
+Copyright (c) 2008-2024, The IETF Trust
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without