diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f88c7c22e3..8b137891791 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,475 +1 @@ -name: Build and Release -run-name: ${{ github.ref_name == 'release' && '[Prod]' || '[Dev]' }} Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} -on: - push: - branches: [release] - - workflow_dispatch: - inputs: - deploy: - description: 'Deploy to K8S' - default: 'Skip' - required: true - type: choice - options: - - Skip - - Staging Only - - Staging + Prod - sandbox: - description: 'Deploy to Sandbox' - default: true - required: true - type: boolean - sandboxNoDbRefresh: - description: 'Sandbox Disable Daily DB Refresh' - default: false - required: true - type: boolean - skiptests: - description: 'Skip Tests' - default: false - required: true - type: boolean - skiparm: - description: 'Skip ARM64 Build' - default: false - required: true - type: boolean - ignoreLowerCoverage: - description: 'Ignore Lower Coverage' - default: false - required: true - type: boolean - updateCoverage: - description: 'Update Baseline Coverage' - default: false - required: true - type: boolean - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # ----------------------------------------------------------------- - # PREPARE - # ----------------------------------------------------------------- - prepare: - name: Prepare Release - runs-on: ubuntu-latest - outputs: - should_deploy: ${{ steps.buildvars.outputs.should_deploy }} - pkg_version: ${{ steps.buildvars.outputs.pkg_version }} - from_tag: ${{ steps.semver.outputs.nextStrict }} - to_tag: ${{ steps.semver.outputs.current }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - fetch-tags: false - - - name: Get Next Version (Prod) - if: ${{ github.ref_name == 'release' }} - id: semver - uses: ietf-tools/semver-action@v1 - with: - token: ${{ github.token }} - branch: release - skipInvalidTags: true - - - name: Get Dev Version - if: ${{ github.ref_name != 'release' }} - id: semverdev - uses: ietf-tools/semver-action@v1 - with: - token: ${{ github.token }} - branch: release - skipInvalidTags: true - noVersionBumpBehavior: 'current' - noNewCommitBehavior: 'current' - - - name: Set Release Flag - if: ${{ github.ref_name == 'release' }} - run: | - echo "IS_RELEASE=true" >> $GITHUB_ENV - - - name: Create Draft Release - uses: ncipollo/release-action@v1.14.0 - if: ${{ github.ref_name == 'release' }} - with: - prerelease: true - draft: false - commit: ${{ github.sha }} - tag: ${{ steps.semver.outputs.nextStrict }} - name: ${{ steps.semver.outputs.nextStrict }} - body: '*pending*' - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Build Variables - id: buildvars - run: | - if [[ $IS_RELEASE ]]; then - echo "Using AUTO SEMVER mode: ${{ steps.semver.outputs.nextStrict }}" - echo "should_deploy=true" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT - echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" - else - echo "Using TEST mode: ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" - echo "should_deploy=false" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT - echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" - fi - - # ----------------------------------------------------------------- - # TESTS - # ----------------------------------------------------------------- - - tests: - name: Run Tests - uses: ./.github/workflows/tests.yml - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - needs: [prepare] - with: - ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }} - - # ----------------------------------------------------------------- - # RELEASE - # ----------------------------------------------------------------- - release: - name: Make Release - if: ${{ !failure() && !cancelled() }} - needs: [tests, prepare] - runs-on: ubuntu-latest - permissions: - contents: write - packages: write - env: - SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}} - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - FROM_TAG: ${{needs.prepare.outputs.from_tag}} - TO_TAG: ${{needs.prepare.outputs.to_tag}} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - fetch-tags: false - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '16' - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Download a Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.1.8 - with: - name: coverage - - - name: Make Release Build - env: - DEBIAN_FRONTEND: noninteractive - BROWSERSLIST_IGNORE_OLD_DATA: 1 - run: | - echo "PKG_VERSION: $PKG_VERSION" - echo "GITHUB_SHA: $GITHUB_SHA" - echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" - echo "Running frontend build script..." - echo "Compiling native node packages..." - yarn rebuild - echo "Packaging static assets..." - yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/ - yarn legacy:build - echo "Setting version $PKG_VERSION..." - sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py - sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py - sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py - - - name: Set Production Flags - if: ${{ env.SHOULD_DEPLOY == 'true' }} - run: | - echo "Setting production flags in settings.py..." - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py - - - name: Make Release Tarball - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Build release tarball..." - mkdir -p /home/runner/work/release - tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . - - - name: Collect + Push Statics - env: - DEBIAN_FRONTEND: noninteractive - AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} - AWS_DEFAULT_REGION: auto - AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} - run: | - echo "Collecting statics..." - docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh - echo "Pushing statics..." - cd static - aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors - - - name: Augment dockerignore for docker image build - env: - DEBIAN_FRONTEND: noninteractive - run: | - cat >> .dockerignore < by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>", - "attachments": [ - { - "color": "28a745", - "fields": [ - { - "title": "Status", - "short": true, - "value": "Completed" - } - ] - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }} - - name: Notify on Slack (Failure) - if: ${{ contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v1.26.0 - with: - channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - payload: | - { - "text": "Datatracker Build by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>", - "attachments": [ - { - "color": "a82929", - "fields": [ - { - "title": "Status", - "short": true, - "value": "Failed" - } - ] - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }} - - # ----------------------------------------------------------------- - # SANDBOX - # ----------------------------------------------------------------- - sandbox: - name: Deploy to Sandbox - if: ${{ !failure() && !cancelled() && github.event.inputs.sandbox == 'true' }} - needs: [prepare, release] - runs-on: [self-hosted, dev-server] - environment: - name: sandbox - env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - - steps: - - uses: actions/checkout@v4 - - - name: Download a Release Artifact - uses: actions/download-artifact@v4.1.8 - with: - name: release-${{ env.PKG_VERSION }} - - - name: Deploy to containers - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Reset production flags in settings.py..." - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = True/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'development'/" ietf/settings.py - echo "Install Deploy to Container CLI dependencies..." - cd dev/deploy-to-container - npm ci - cd ../.. - echo "Start Deploy..." - node ./dev/deploy-to-container/cli.js --branch ${{ github.ref_name }} --domain dev.ietf.org --appversion ${{ env.PKG_VERSION }} --commit ${{ github.sha }} --ghrunid ${{ github.run_id }} --nodbrefresh ${{ github.event.inputs.sandboxNoDbRefresh }} - - - name: Cleanup old docker resources - env: - DEBIAN_FRONTEND: noninteractive - run: | - docker image prune -a -f - - # ----------------------------------------------------------------- - # STAGING - # ----------------------------------------------------------------- - staging: - name: Deploy to Staging - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging Only' || github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, release] - runs-on: ubuntu-latest - environment: - name: staging - env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - - steps: - - name: Deploy to staging - uses: the-actions-org/workflow-dispatch@v4 - with: - workflow: deploy.yml - repo: ietf-tools/infra-k8s - ref: main - token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' - wait-for-completion: true - wait-for-completion-timeout: 10m - wait-for-completion-interval: 30s - display-workflow-run-url: false - - # ----------------------------------------------------------------- - # PROD - # ----------------------------------------------------------------- - prod: - name: Deploy to Production - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, staging] - runs-on: ubuntu-latest - environment: - name: production - env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - - steps: - - name: Deploy to production - uses: the-actions-org/workflow-dispatch@v4 - with: - workflow: deploy.yml - repo: ietf-tools/infra-k8s - ref: main - token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' - wait-for-completion: true - wait-for-completion-timeout: 10m - wait-for-completion-interval: 30s - display-workflow-run-url: false diff --git a/README.md b/README.md index 133d08f5e79..e69de29bb2d 100644 --- a/README.md +++ b/README.md @@ -1,296 +0,0 @@ -
- -IETF Datatracker - -[![Release](https://img.shields.io/github/release/ietf-tools/datatracker.svg?style=flat&maxAge=300)](https://github.com/ietf-tools/datatracker/releases) -[![License](https://img.shields.io/github/license/ietf-tools/datatracker)](https://github.com/ietf-tools/datatracker/blob/main/LICENSE) -[![Code Coverage](https://codecov.io/gh/ietf-tools/datatracker/branch/feat/bs5/graph/badge.svg?token=V4DXB0Q28C)](https://codecov.io/gh/ietf-tools/datatracker) -[![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites) -[![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites) -[![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites) -[![MariaDB Version](https://img.shields.io/badge/postgres-16-blue?logo=postgresql&logoColor=white)](#prerequisites) - -##### The day-to-day front-end to the IETF database for people who work on IETF standards. - -
- -- [**Production Website**](https://datatracker.ietf.org) -- [Changelog](https://github.com/ietf-tools/datatracker/releases) -- [Contributing](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md) -- [Getting Started](#getting-started) - *[ tl;dr ](#the-tldr-to-get-going)* - - [Creating a Fork](#creating-a-fork) - - [Git Cloning Tips](#git-cloning-tips) - - [Docker Dev Environment](docker/README.md) -- [Database & Assets](#database--assets) -- [Old Datatracker Branches](https://github.com/ietf-tools/old-datatracker-branches/branches/all) -- [Frontend Development](#frontend-development) - - [Intro](#intro) - - [Vite](#vite-vue-3) - - [Parcel](#parcel-legacyjquery) - - [Bootstrap](#bootstrap) - - [Serving Static Files via CDN](#serving-static-files-via-cdn) - - [Handling of External Javascript and CSS Components](#handling-of-external-javascript-and-css-components) - - [Handling of Internal Static Files](#handling-of-internal-static-files) - - [Changes to Template Files](#changes-to-template-files) - - [Deployment](#deployment) -- [Running Tests](#running-tests) - - [Python](#python-tests) - - [Frontend](#frontend-tests) - - [Diff Tool](#diff-tool) - ---- - -### Getting Started - -This project is following the standard **Git Feature Workflow** development model. Learn about all the various steps of the development workflow, from creating a fork to submitting a pull request, in the [Contributing](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md) guide. - -> Make sure to read the [Styleguides](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md#styleguides) section to ensure a cohesive code format across the project. - -You can submit bug reports, enhancement and new feature requests in the [discussions](https://github.com/ietf-tools/datatracker/discussions) area. Accepted tickets will be converted to issues. - -#### Creating a Fork - -Click the Fork button in the top-right corner of the repository to create a personal copy that you can work on. - -> Note that some GitHub Actions might be enabled by default in your fork. You should disable them by going to **Settings** > **Actions** > **General** and selecting **Disable actions** (then Save). - -#### Git Cloning Tips - -As outlined in the [Contributing](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md) guide, you will first want to create a fork of the datatracker project in your personal GitHub account before cloning it. - -Windows developers: [Start with WSL2 from the beginning](https://github.com/ietf-tools/.github/blob/main/docs/windows-dev.md). - -Because of the extensive history of this project, cloning the datatracker project locally can take a long time / disk space. You can speed up the cloning process by limiting the history depth, for example *(replace `USERNAME` with your GitHub username)*: - -- To fetch only up to the 10 latest commits: - ```sh - git clone --depth=10 https://github.com/USERNAME/datatracker.git - ``` -- To fetch only up to a specific date: - ```sh - git clone --shallow-since=DATE https://github.com/USERNAME/datatracker.git - ``` - -#### The tl;dr to get going - -Note that you will have to have cloned the datatracker code locally - please read the above sections. - -Datatracker development is performed using Docker containers. You will need to be able to run docker (and docker-compose) on your machine to effectively develop. It is possible to get a purely native install working, but it is _very complicated_ and typically takes a first time datatracker developer a full day of setup, where the docker setup completes in a small number of minutes. - -Many developers are using [VS Code](https://code.visualstudio.com/) and taking advantage of VS Code's ability to start a project in a set of containers. If you are using VS Code, simply start VS Code in your clone and inside VS Code choose `Restart in container`. - -If VS Code is not available to you, in your clone, type `cd docker; ./run` - -Once the containers are started, run the tests to make sure your checkout is a good place to start from (all tests should pass - if any fail, ask for help at tools-develop@). Inside the app container's shell type: -```sh -ietf/manage.py test --settings=settings_test -``` - -Note that we recently moved the datatracker onto PostgreSQL - you may still find older documentation that suggests testing with settings_sqlitetest. That will no longer work. - -For a more detailed description of getting going, see [docker/README.md](docker/README.md). - -#### Overview of the datatracker models - -A beginning of a [walkthrough of the datatracker models](https://notes.ietf.org/iab-aid-datatracker-database-overview) was prepared for the IAB AID workshop. - -#### Docker Dev Environment - -In order to simplify and reduce the time required for setup, a preconfigured docker environment is available. - -Read the [Docker Dev Environment](docker/README.md) guide to get started. - -### Database & Assets - -Nightly database dumps of the datatracker are available as Docker images: `ghcr.io/ietf-tools/datatracker-db:latest` - -> Note that to update the database in your dev environment to the latest version, you should run the `docker/cleandb` script. - -### Frontend Development - -#### Intro - -We now use `yarn` to manage assets for the Datatracker, and `vite`/`parcel` to package them. `yarn` maintains its `node` packages under the `.yarn` directory. - -The datatracker uses 2 different build systems, depending on the use case: -- [**Vite**](https://vitejs.dev/) for Vue 3 pages / components -- [**Parcel**](https://parceljs.org/) for legacy pages / jQuery - -#### Vite *(Vue 3)* - -Pages will gradually be updated to Vue 3 components. These components are located under the `/client` directory. - -Each Vue 3 app has its own sub-directory. For example, the agenda app is located under `/client/agenda`. - -The datatracker makes use of the Django-Vite plugin to point to either the Vite.js server or the precompiled production files. The `DJANGO_VITE_DEV_MODE` flag, found in the `ietf/settings_local.py` file determines whether the Vite.js server is used or not. - -In development mode, you must start the Vite.js development server, in addition to the usual Datatracker server: - -```sh -yarn dev -``` - -Any changes made to the files under `/client` will automatically trigger a hot-reload of the modified components. - -To generate production assets, run the build command: - -```sh -yarn build -``` - -This will create packages under `ietf/static/dist-neue`, which are then served by the Django development server, and which must be uploaded to the CDN. - -#### Parcel *(Legacy/jQuery)* - -The Datatracker includes these packages from the various Javascript and CSS files in `ietf/static/js` and `ietf/static/css` respectively, bundled using Parcel. -Static images are likewise in `ietf/static/images`. - -Whenever changes are made to the files under `ietf/static`, you must re-run the build command to package them: - -``` shell -yarn legacy:build -``` - -This will create packages under `ietf/static/dist/ietf`, which are then served by the Django development server, and which must be uploaded to the CDN. - -#### Bootstrap - -The "new" datatracker uses Twitter Bootstrap for the UI. - -Get familiar with and use those UI elements, CSS classes, etc. instead of cooking up your own. - -Some ground rules: - -- Think hard before tweaking the bootstrap CSS, it will make it harder to upgrade to future releases. -- No `