diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5a91b21b635..ef1146c45d6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,91 +7,81 @@ "workspaceFolder": "/workspace", "shutdownAction": "stopCompose", "postCreateCommand": "/docker-init.sh", + "postStartCommand": "/docker-start.sh", "containerEnv": { "EDITOR_VSCODE": "true" }, - - // Set *default* container specific settings.json values on container create. - "settings": { - "terminal.integrated.defaultProfile.linux": "zsh", - "python.pythonPath": "/usr/local/bin/python", - "python.languageServer": "Pylance", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", - "python.formatting.blackPath": "/usr/local/py-utils/bin/black", - "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", - "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", - "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", - "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", - "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", - "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", - "python.testing.pytestArgs": [ - "ietf" - ], - "python.testing.unittestEnabled": true, - "python.testing.pytestEnabled": false, - "python.testing.unittestArgs": [ - "-v", - "-s", - "./ietf", - "-p", - "test*.py" - ], - "sqltools.connections": [ - // Default connection to dev DB container - { - "name": "Local Dev", - "server": "db", - "port": 3306, - "database": "ietf_utf8", - "username": "django", - "password": "RkTkDPFnKpko", - "driver": "MySQL", - "askForPassword": false, - "connectionTimeout": 60 + + "features": { + }, + + "customizations": { + "vscode": { + "extensions": [ + "vue.volar", + "batisteo.vscode-django", + "eamodio.gitlens", + "editorconfig.editorconfig", + "mrmlnc.vscode-duplicate", + "ms-azuretools.vscode-docker", + "ms-playwright.playwright", + "ms-python.python", + "ms-python.pylint", + "ms-python.vscode-pylance", + "mutantdino.resourcemonitor", + "oderwat.indent-rainbow", + "oxc.oxc-vscode", + "redhat.vscode-yaml", + "charliermarsh.ruff" + ], + "settings": { + "terminal.integrated.defaultProfile.linux": "zsh", + "python.pythonPath": "/usr/local/bin/python", + "python.languageServer": "Default", + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.testing.pytestArgs": [ + "ietf" + ], + "python.testing.unittestEnabled": true, + "python.testing.pytestEnabled": false, + "python.testing.unittestArgs": [ + "-v", + "-s", + "./ietf", + "-p", + "test*.py" + ] } - ] - // "python.envFile": "${workspaceFolder}/.devcontainer/dev.env" + } }, - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "ms-azuretools.vscode-docker", - "editorconfig.editorconfig", - "redhat.vscode-yaml", - "visualstudioexptteam.vscodeintellicode", - "batisteo.vscode-django", - "mutantdino.resourcemonitor", - "spmeesseman.vscode-taskexplorer", - "mtxr.sqltools", - "mtxr.sqltools-driver-mysql", - "mrmlnc.vscode-duplicate", - "eamodio.gitlens", - "oderwat.indent-rainbow", - "johnsoncodehk.volar" - ], - // Use 'forwardPorts' to make a list of ports inside the container available locally. - "forwardPorts": [8000, 3306], + "forwardPorts": [3000, 5432, 8000], "portsAttributes": { + "3000": { + "label": "Vite", + "onAutoForward": "silent" + }, + "5432": { + "label": "PostgreSQL", + "onAutoForward": "silent" + }, "8000": { - "label": "Datatracker", + "label": "NGINX", "onAutoForward": "notify" }, - "3306": { - "label": "MariaDB", - "onAutoForward": "silent" + "8001": { + "label": "Datatracker", + "onAutoForward": "ignore" } - } + }, // Use 'postCreateCommand' to run commands after the container is created. // "postCreateCommand": "sh /docker-init.sh", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - // "remoteUser": "django" + "remoteUser": "dev" } diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml index 310872634e5..ce1ce259fdb 100644 --- a/.devcontainer/docker-compose.extend.yml +++ b/.devcontainer/docker-compose.extend.yml @@ -1,11 +1,21 @@ -version: '3.8' - services: app: environment: EDITOR_VSCODE: 1 volumes: - .:/workspace - - /workspace/node_modules + - /workspace/.parcel-cache + - /workspace/__pycache__ + - /workspace/.vite + - /workspace/.yarn/unplugged + - app-assets:/assets + # - datatracker-vscode-ext:/root/.vscode-server/extensions # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db + blobstore: + ports: + - '9000:9000' + - '9001:9001' + +volumes: + datatracker-vscode-ext: diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..85d195d3807 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +# Files to be excluded from the docker build context + +/.git +/.github +/svn-history +/data +/tmp-* diff --git a/.editorconfig b/.editorconfig index 95dc121e163..7d5f138b355 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,59 +4,42 @@ root = true -# Settings for IETF datatracker +# Global settings for IETF datatracker # --------------------------------------------------------- -# PEP8 Style [*] +indent_size = 2 indent_style = space -indent_size = 4 -end_of_line = lf charset = utf-8 -# to avoid tripping Henrik's commit hook: -trim_trailing_whitespace = false -insert_final_newline = false +trim_trailing_whitespace = true +insert_final_newline = true +end_of_line = lf + +# Specializations below *add to* the above defaults -# Settings for .github folder +# Settings for Python files # --------------------------------------------------------- -# GitHub Markdown Style +# PEP8 Style -[.github/**] -indent_style = space -indent_size = 2 -end_of_line = lf -charset = utf-8 +[*.{py,py.bak}] +indent_size = 4 trim_trailing_whitespace = false -insert_final_newline = true -# Settings for client-side JS / Vue files +# Settings for HTML templates # --------------------------------------------------------- -# StandardJS Style - -[client/**] -indent_style = space -indent_size = 2 -end_of_line = lf -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true +# HTML should not have final newlines, because that causes visible whitespace when includes are rendered -[package.json] -indent_style = space -indent_size = 2 -end_of_line = lf -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true +[ietf/**.html] +insert_final_newline = false -# Settings for cypress tests +# Settings for Markdown # --------------------------------------------------------- -# StandardJS Style +# Avoid trimming trailing whitespace +[*.md] +trim_trailing_whitespace = false -[cypress/**] -indent_style = space -indent_size = 2 -end_of_line = lf -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true \ No newline at end of file +# Unset for special / output files +# --------------------------------------------------------- +[{patch,vzic,bin}/**] +indent_style = unset +indent_size = unset diff --git a/.eslintrc.js b/.eslintrc.js deleted file mode 100644 index 349bc61b8df..00000000000 --- a/.eslintrc.js +++ /dev/null @@ -1,19 +0,0 @@ -module.exports = { - rules: { - indent: [2, 4], - camelcase: 0, - "require-jsdoc": 0, - quotes: [2, "double"], - "no-multiple-empty-lines": [2, {max: 2}], - "quote-props": [2, "as-needed"], - "brace-style": [2, "1tbs", {allowSingleLine: true}] - }, - env: { - browser: true, - jquery: true - }, - globals: { - d3: true - }, - extends: "google" -}; diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..62f4aae4323 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,280 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# --------------------------------------------------- +# Python Projects +# --------------------------------------------------- + +# Source files +*.pxd text diff=python +*.py text diff=python +*.py3 text diff=python +*.pyw text diff=python +*.pyx text diff=python +*.pyz text diff=python +*.pyi text diff=python + +# Binary files +*.db binary +*.p binary +*.pkl binary +*.pickle binary +*.pyc binary export-ignore +*.pyo binary export-ignore +*.pyd binary + +# Jupyter notebook +*.ipynb text eol=lf + +# --------------------------------------------------- +# Web Projects +# --------------------------------------------------- + +# Source code +*.bash text eol=lf +*.bat text eol=crlf +*.cmd text eol=crlf +*.coffee text +*.css text diff=css +*.htm text diff=html +*.html text diff=html +*.inc text +*.ini text +*.js text +*.mjs text +*.cjs text +*.json text +*.jsx text +*.less text +*.ls text +*.map text -diff +*.od text +*.onlydata text +*.php text diff=php +*.pl text +*.ps1 text eol=crlf +*.py text diff=python +*.rb text diff=ruby +*.sass text +*.scm text +*.scss text diff=css +*.sh text eol=lf +.husky/* text eol=lf +*.sql text +*.styl text +*.tag text +*.ts text +*.tsx text +*.xml text +*.xhtml text diff=html + +# Docker +Dockerfile text + +# Documentation +*.ipynb text eol=lf +*.markdown text diff=markdown +*.md text diff=markdown +*.mdwn text diff=markdown +*.mdown text diff=markdown +*.mkd text diff=markdown +*.mkdn text diff=markdown +*.mdtxt text +*.mdtext text +*.txt text +AUTHORS text +CHANGELOG text +CHANGES text +CONTRIBUTING text +COPYING text +copyright text +*COPYRIGHT* text +INSTALL text +license text +LICENSE text +NEWS text +readme text +*README* text +TODO text + +# Templates +*.dot text +*.ejs text +*.erb text +*.haml text +*.handlebars text +*.hbs text +*.hbt text +*.jade text +*.latte text +*.mustache text +*.njk text +*.phtml text +*.pug text +*.svelte text +*.tmpl text +*.tpl text +*.twig text +*.vue text + +# Configs +*.cnf text +*.conf text +*.config text +.editorconfig text +.env text +.gitattributes text +.gitconfig text +.htaccess text +*.lock text -diff +package.json text eol=lf +package-lock.json text eol=lf -diff +pnpm-lock.yaml text eol=lf -diff +.prettierrc text +yarn.lock text -diff +*.toml text +*.yaml text +*.yml text +browserslist text +Makefile text +makefile text +# Fixes syntax highlighting on GitHub to allow comments +tsconfig.json linguist-language=JSON-with-Comments + +# Heroku +Procfile text + +# Graphics +*.ai binary +*.bmp binary +*.eps binary +*.gif binary +*.gifv binary +*.ico binary +*.jng binary +*.jp2 binary +*.jpg binary +*.jpeg binary +*.jpx binary +*.jxr binary +*.pdf binary +*.png binary +*.psb binary +*.psd binary +*.svg text +*.svgz binary +*.tif binary +*.tiff binary +*.wbmp binary +*.webp binary + +# Audio +*.kar binary +*.m4a binary +*.mid binary +*.midi binary +*.mp3 binary +*.ogg binary +*.ra binary + +# Video +*.3gpp binary +*.3gp binary +*.as binary +*.asf binary +*.asx binary +*.avi binary +*.fla binary +*.flv binary +*.m4v binary +*.mng binary +*.mov binary +*.mp4 binary +*.mpeg binary +*.mpg binary +*.ogv binary +*.swc binary +*.swf binary +*.webm binary + +# Archives +*.7z binary +*.gz binary +*.jar binary +*.rar binary +*.tar binary +*.zip binary + +# Fonts +*.ttf binary +*.eot binary +*.otf binary +*.woff binary +*.woff2 binary + +# Executables +*.exe binary +*.pyc binary +# Prevents massive diffs caused by vendored, minified files +**/.yarn/releases/** binary +**/.yarn/plugins/** binary + +# RC files (like .babelrc or .eslintrc) +*.*rc text + +# Ignore files (like .npmignore or .gitignore) +*.*ignore text + +# Prevents massive diffs from built files +dist/* binary + +# --------------------------------------------------- +# Common +# --------------------------------------------------- + +# Documents +*.bibtex text diff=bibtex +*.doc diff=astextplain +*.DOC diff=astextplain +*.docx diff=astextplain +*.DOCX diff=astextplain +*.dot diff=astextplain +*.DOT diff=astextplain +*.pdf diff=astextplain +*.PDF diff=astextplain +*.rtf diff=astextplain +*.RTF diff=astextplain +*.md text diff=markdown +*.mdx text diff=markdown +*.tex text diff=tex +*.adoc text +*.textile text +*.mustache text +*.csv text eol=crlf +*.tab text +*.tsv text +*.txt text +*.sql text +*.epub diff=astextplain + +# Text files where line endings should be preserved +*.patch -text + +# --------------------------------------------------- +# Vzic specific +# --------------------------------------------------- + +*.pl text diff=perl +*.pm text diff=perl + +# C/C++ +*.c text diff=cpp +*.cc text diff=cpp +*.cxx text diff=cpp +*.cpp text diff=cpp +*.cpi text diff=cpp +*.c++ text diff=cpp +*.hpp text diff=cpp +*.h text diff=cpp +*.h++ text diff=cpp +*.hh text diff=cpp \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index c0f7ffdca66..320614b17ec 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,8 +1,8 @@ -blank_issues_enabled: false -contact_links: - - name: Help / Questions - url: https://github.com/ietf-tools/datatracker/discussions/categories/help-questions - about: Need help? Have a question on setting up the project or its usage? - - name: New Feature / Enhancement Request - url: https://github.com/ietf-tools/datatracker/discussions/categories/ideas - about: Submit ideas for new features or improvements. +blank_issues_enabled: false +contact_links: + - name: Help and questions + url: https://github.com/ietf-tools/datatracker/discussions/categories/help-questions + about: Need help? Have a question on setting up the project or its usage? + - name: Discuss new ideas + url: https://github.com/ietf-tools/datatracker/discussions/categories/ideas + about: Submit ideas for new features or improvements to be discussed. diff --git a/.github/ISSUE_TEMPLATE/new-feature.yml b/.github/ISSUE_TEMPLATE/new-feature.yml new file mode 100644 index 00000000000..285081e1c8b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new-feature.yml @@ -0,0 +1,25 @@ +name: Suggest new feature or enhancement +description: Propose a new idea to be implemented. +labels: ["enhancement"] +type: Feature +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to propose a new feature or enhancement idea. + - type: textarea + id: description + attributes: + label: Description + description: Include as much info as possible, including mockups or screenshots if available. + placeholder: Description + validations: + required: true + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this request, you agree to follow our [Code of Conduct](https://github.com/ietf-tools/.github/blob/main/CODE_OF_CONDUCT.md). + options: + - label: I agree to follow the [IETF's Code of Conduct](https://github.com/ietf-tools/.github/blob/main/CODE_OF_CONDUCT.md) + required: true diff --git a/.github/ISSUE_TEMPLATE/report-a-bug.yml b/.github/ISSUE_TEMPLATE/report-a-bug.yml index d911d8e2eda..47fa1185b40 100644 --- a/.github/ISSUE_TEMPLATE/report-a-bug.yml +++ b/.github/ISSUE_TEMPLATE/report-a-bug.yml @@ -1,34 +1,33 @@ -name: Report a Bug -description: Something isn't right? File a bug report +name: Report a Datatracker bug +description: Something in the datatracker's behavior isn't right? File a bug report. Don't use this to report RFC errata or issues with the content of Internet-Drafts. labels: ["bug"] +type: Bug body: - type: markdown attributes: value: | - Thanks for taking the time to fill out this bug report! + Thanks for taking the time to fill out this bug report. - type: textarea - id: what-happened + id: description attributes: - label: What happened? - description: Also tell us, what did you expect to happen? - placeholder: Tell us what you see! + label: Describe the issue + description: Include as much info as possible, including the current behavior, expected behavior, screenshots, etc. If this is a display or user interface issue, make sure to list the browser(s) you're experiencing the issue on. + placeholder: Description validations: required: true - - type: dropdown - id: browsers - attributes: - label: What browser(s) are you seeing the problem on? - multiple: true - options: - - Chrome / Edge - - Firefox - - Safari - - Not Applicable - type: checkboxes id: terms attributes: label: Code of Conduct description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/ietf-tools/.github/blob/main/CODE_OF_CONDUCT.md) options: - - label: I agree to follow the IETF's Code of Conduct + - label: I agree to follow the [IETF's Code of Conduct](https://github.com/ietf-tools/.github/blob/main/CODE_OF_CONDUCT.md) required: true + - type: markdown + attributes: + value: | + If you are having trouble logging into the datatracker, please do not open an issue here. Instead, please send email to support@ietf.org providing your name and username. + - type: markdown + attributes: + value: | + **Please do not report issues with the content of Internet-Drafts or RFCs using this repository. Send email to the relevant group instead. Some Internet-Drafts have their own github repositories where issues can be reported. See the datatracker's page for the I-D for links and email addresses. Errata for published RFCs are submitted at https://www.rfc-editor.org/errata.php#reportnew** diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..820887a07e6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,53 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "docker" + directory: "/docker" + schedule: + interval: "weekly" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + reviewers: + - "rjsparks" + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "weekly" + groups: + yarn: + patterns: + - "*" + - package-ecosystem: "npm" + directory: "/playwright" + schedule: + interval: "weekly" + groups: + npm: + patterns: + - "*" + - package-ecosystem: "npm" + directory: "/dev/coverage-action" + schedule: + interval: "weekly" + groups: + npm: + patterns: + - "*" + - package-ecosystem: "npm" + directory: "/dev/diff" + schedule: + interval: "weekly" + groups: + npm: + patterns: + - "*" diff --git a/.github/semantic.yml b/.github/semantic.yml new file mode 100644 index 00000000000..be3439f6b97 --- /dev/null +++ b/.github/semantic.yml @@ -0,0 +1,3 @@ +enabled: true +titleOnly: true +targetUrl: "https://www.conventionalcommits.org/en/v1.0.0/#summary" diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml new file mode 100644 index 00000000000..a2abe089ce6 --- /dev/null +++ b/.github/workflows/build-base-app.yml @@ -0,0 +1,67 @@ +name: Build Base App Docker Image + +on: + push: + branches: + - 'main' + paths: + - 'docker/base.Dockerfile' + - 'requirements.txt' + + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + + steps: + - uses: actions/checkout@v7 + with: + token: ${{ secrets.GH_COMMON_TOKEN }} + + - name: Set Version + run: | + printf -v CURDATE '%(%Y%m%dT%H%M)T' -1 + echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker Build & Push + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: docker/base.Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }} + ghcr.io/ietf-tools/datatracker-app-base:py312 + ${{ github.ref == 'refs/heads/main' && 'ghcr.io/ietf-tools/datatracker-app-base:latest' || '' }} + + - name: Update version references + run: | + sed -i "1s/.*/FROM ghcr.io\/ietf-tools\/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile + echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE + + - name: Commit CHANGELOG.md + uses: stefanzweifel/git-auto-commit-action@v7 + with: + branch: ${{ github.ref_name }} + commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}' + file_pattern: dev/build/Dockerfile dev/build/TARGET_BASE diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml new file mode 100644 index 00000000000..429aa08b006 --- /dev/null +++ b/.github/workflows/build-devblobstore.yml @@ -0,0 +1,47 @@ +name: Build Dev/Test Blobstore Docker Image + +on: + push: + branches: + - 'main' + paths: + - '.github/workflows/build-devblobstore.yml' + + workflow_dispatch: + +env: + MINIO_VERSION: latest + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v7 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker Build & Push + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: docker/devblobstore.Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + build-args: MINIO_VERSION=${{ env.MINIO_VERSION }} + tags: | + ghcr.io/ietf-tools/datatracker-devblobstore:${{ env.MINIO_VERSION }} + ghcr.io/ietf-tools/datatracker-devblobstore:latest diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml new file mode 100644 index 00000000000..7832e65a3a3 --- /dev/null +++ b/.github/workflows/build-mq-broker.yml @@ -0,0 +1,63 @@ +name: Build MQ Broker Docker Image + +on: + push: + branches: + - 'main' + paths: + - 'dev/mq/**' + - '.github/workflows/build-mq-broker.yml' + + workflow_dispatch: + inputs: + rabbitmq_version: + description: 'RabbitMQ Version' + default: '3.13-alpine' + required: true + type: string + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v7 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set rabbitmq version + id: rabbitmq-version + run: | + if [[ "${{ inputs.rabbitmq_version }}" == "" ]]; then + echo "RABBITMQ_VERSION=3.13-alpine" >> $GITHUB_OUTPUT + else + echo "RABBITMQ_VERSION=${{ inputs.rabbitmq_version }}" >> $GITHUB_OUTPUT + fi + + - name: Docker Build & Push + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: dev/mq/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + build-args: RABBITMQ_VERSION=${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} + tags: | + ghcr.io/ietf-tools/datatracker-mq:${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} + ghcr.io/ietf-tools/datatracker-mq:latest diff --git a/.github/workflows/build-test-base.yml b/.github/workflows/build-test-base.yml deleted file mode 100644 index 4feb6911e94..00000000000 --- a/.github/workflows/build-test-base.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Build Base Test Docker Image - -on: - push: - branches: - # TEMPORARY: Replace with main once bs5 is merged - - 'feat/bs5' - paths: - - 'package.json' - - 'requirements.txt' - - workflow_dispatch: - -env: - REGISTRY: ghcr.io - IMAGE_NAME: datatracker-test-base - -jobs: - publish: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - uses: actions/checkout@v2 - - - name: Docker Build & Push Action - uses: mr-smithers-excellent/docker-build-push@v5.6 - with: - image: ${{ env.IMAGE_NAME }} - tags: latest - registry: ${{ env.REGISTRY }} - dockerfile: dev/docker-test-base/Dockerfile - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 885d33701a3..e7f0f8abcd3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,124 +1,244 @@ -name: Build and Release +name: Build Dev and Deploy +run-name: Dev Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} on: - push: - tags: - - 'v*' - workflow_dispatch: inputs: - publish: - description: 'Create Production Release' + deploy: + description: 'Deploy to K8S' + default: 'Skip' + required: true + type: choice + options: + - Skip + - Dev + - Staging + deployStrategy: + description: 'Deploy Strategy (Staging Only)' + default: 'Recreate' + required: true + type: choice + options: + - Recreate + - RollingUpdate + - Current + devNoDbRefresh: + description: 'Disable Daily DB Refresh (Dev Only)' + default: false + required: true + type: boolean + skipTests: + description: 'Skip Tests' + default: false + required: true + type: boolean + skipArmBuild: + description: 'Skip ARM64 Build' + default: false + required: true + type: boolean + ignoreLowerCoverage: + description: 'Ignore Lower Coverage' + default: false + required: true + type: boolean + updateCoverage: + description: 'Update Baseline Coverage' + default: false required: true type: boolean +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - publish: + # ----------------------------------------------------------------- + # PREPARE + # ----------------------------------------------------------------- + prepare: + name: Prepare runs-on: ubuntu-latest + outputs: + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} - steps: - - uses: actions/checkout@v2 + steps: + - uses: actions/checkout@v7 with: - fetch-depth: 0 - - - name: Get Next Version - if: ${{ github.event.inputs.publish == 'true' }} + fetch-depth: 1 + fetch-tags: false + + - name: Get Dev Version id: semver uses: ietf-tools/semver-action@v1 with: token: ${{ github.token }} - branch: main - - - name: Set Next Version Env Var - if: ${{ github.event.inputs.publish == 'true' }} - env: - NEXT_VERSION: ${{ steps.semver.outputs.next }} - run: | - echo "NEXT_VERSION=$next" >> $GITHUB_ENV - - - name: Create Draft Release - uses: ncipollo/release-action@v1 - if: ${{ github.event.inputs.publish == 'true' }} - with: - prerelease: true - draft: false - commit: ${{ github.sha }} - tag: ${{ env.NEXT_VERSION }} - name: ${{ env.NEXT_VERSION }} - body: '*pending*' - token: ${{ secrets.GITHUB_TOKEN }} + branch: release + skipInvalidTags: true + noVersionBumpBehavior: 'current' + noNewCommitBehavior: 'current' - name: Set Build Variables + id: buildvars run: | - if [[ $NEXT_VERSION ]]; then - echo "Using AUTO SEMVER mode: $NEXT_VERSION" - echo "SHOULD_DEPLOY=true" >> $GITHUB_ENV - echo "PKG_VERSION=$NEXT_VERSION" >> $GITHUB_ENV - echo "PKG_VERSION_STRICT=${NEXT_VERSION#?}" >> $GITHUB_ENV - echo "::notice::Release created using branch $GITHUB_REF_NAME" - elif [[ "$GITHUB_REF" =~ ^refs/tags/v* ]]; then - echo "Using TAG mode: $GITHUB_REF_NAME" - echo "SHOULD_DEPLOY=true" >> $GITHUB_ENV - echo "PKG_VERSION=$GITHUB_REF_NAME" >> $GITHUB_ENV - echo "PKG_VERSION_STRICT=${GITHUB_REF_NAME#?}" >> $GITHUB_ENV - echo "::notice::Release created using tag $GITHUB_REF_NAME" - else - echo "Using TEST mode: v7.0.0-dev.$GITHUB_RUN_NUMBER" - echo "SHOULD_DEPLOY=false" >> $GITHUB_ENV - echo "PKG_VERSION=v3.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV - echo "PKG_VERSION_STRICT=3.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_ENV - echo "::notice::Non-production build created using branch $GITHUB_REF_NAME" - fi - - - name: Setup Node.js - uses: actions/setup-node@v3.0.0 - with: - node-version: 16.x - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Make Release Build - env: - DEBIAN_FRONTEND: noninteractive + SHORT_SHA="${GITHUB_SHA:0:7}" + echo "Will set build version to: ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" + echo "buildVersion=${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" >> $GITHUB_OUTPUT + echo "::notice::Non-production build ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA created using branch $GITHUB_REF_NAME" + + - name: Get Base Image Target Version + id: baseimgversion run: | - echo "Running build script..." - chmod +x ./dev/deploy/build.sh - sh ./dev/deploy/build.sh - echo "Setting version $PKG_VERSION_STRICT..." - sed -i -r -e "s/^__version__ += '.*'$/__version__ = '$PKG_VERSION_STRICT'/" ietf/__init__.py - sed -i -r -e "s/^__release_hash__ += '.*'$/__release_hash__ = '$GITHUB_SHA'/" ietf/__init__.py - sed -i -r -e "s/^__release_branch__ += '.*'$/__release_branch__ = '$GITHUB_REF_NAME'/" ietf/__init__.py - echo "Build release tarball..." - mkdir -p /home/runner/work/release - tar -czf /home/runner/work/release/release.tar.gz -X dev/deploy/exclude-patterns.txt . + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + + # ----------------------------------------------------------------- + # TESTS + # ----------------------------------------------------------------- + tests: + name: Run Tests + uses: ./.github/workflows/reusable-tests.yml + if: ${{ !inputs.skipTests }} + needs: [prepare] + secrets: inherit + with: + ignoreLowerCoverage: ${{ inputs.ignoreLowerCoverage == 'true' }} + skipSelenium: true + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # BUILD IMAGE + # ----------------------------------------------------------------- + build: + name: Build Image + uses: ./.github/workflows/reusable-build.yml + if: ${{ !failure() && !cancelled() }} + needs: [tests, prepare] + permissions: + contents: write + packages: write + secrets: inherit + with: + buildVersion: ${{ needs.prepare.outputs.buildVersion }} + isReleaseBuild: false + previousVersion: ${{ needs.prepare.outputs.previousVersion }} + handleCoverageResults: ${{ inputs.skipTests == 'false' }} + updateCoverageBaseline: ${{ inputs.updateCoverage }} + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + skipArmBuild: ${{ inputs.skipArmBuild }} - - name: Update CHANGELOG - id: changelog - uses: Requarks/changelog-action@v1 - if: env.SHOULD_DEPLOY == 'true' - with: - token: ${{ github.token }} - tag: ${{ env.PKG_VERSION }} - writeToFile: false + # ----------------------------------------------------------------- + # NOTIFY + # ----------------------------------------------------------------- + notify: + name: Notify + if: ${{ always() }} + needs: [prepare, tests, build] + runs-on: ubuntu-latest + env: + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} - - name: Create Release - uses: ncipollo/release-action@v1 - if: env.SHOULD_DEPLOY == 'true' + steps: + - name: Notify on Slack (Success) + if: ${{ !contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 with: - allowUpdates: true - draft: false - tag: ${{ env.PKG_VERSION }} - name: ${{ env.PKG_VERSION }} - body: ${{ steps.changelog.outputs.changes }} - artifacts: "/home/runner/work/release/release.tar.gz" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload Build Artifacts - uses: actions/upload-artifact@v2.3.1 - if: env.SHOULD_DEPLOY == 'false' + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" + attachments: + - color: "28a745" + fields: + - title: "Status" + short: true + value: "Completed" + - name: Notify on Slack (Failure) + if: ${{ contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 with: - name: release - path: /home/runner/work/release/release.tar.gz + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" + attachments: + - color: "a82929" + fields: + - title: "Status" + short: true + value: "Failed 😱" + + # ----------------------------------------------------------------- + # DEV + # ----------------------------------------------------------------- + dev: + name: Deploy to Dev + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Dev' }} + needs: [prepare, build] + runs-on: ubuntu-latest + environment: + name: dev + env: + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} + + steps: + - uses: actions/checkout@v7 + with: + ref: main + + - name: Get Deploy Name + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Install Get Deploy Name CLI dependencies..." + cd dev/k8s-get-deploy-name + npm ci + echo "Get Deploy Name..." + echo "DEPLOY_NAMESPACE=$(node cli.js --branch ${{ github.ref_name }})" >> "$GITHUB_ENV" + + - name: Deploy to dev + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy-dev.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' + waitForCompletionTimeout: 60m + + # ----------------------------------------------------------------- + # STAGING + # ----------------------------------------------------------------- + staging: + name: Deploy to Staging + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Staging' }} + needs: [prepare, build] + runs-on: ubuntu-latest + environment: + name: staging + env: + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} + + steps: + - name: Refresh Staging DB + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy-db.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' + waitForCompletionTimeout: 120m + + - name: Deploy to staging + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml new file mode 100644 index 00000000000..50ec4c1943b --- /dev/null +++ b/.github/workflows/ci-run-tests.yml @@ -0,0 +1,46 @@ +name: PR - Run All Tests + +on: + pull_request: + branches: + - 'main' + - 'feat/rfc' + paths: + - 'client/**' + - 'ietf/**' + - 'playwright/**' + - 'requirements.txt' + - 'package.json' + +jobs: + # ----------------------------------------------------------------- + # PREPARE + # ----------------------------------------------------------------- + prepare: + name: Prepare + runs-on: ubuntu-latest + outputs: + base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Get Base Image Target Version + id: baseimgversion + run: | + echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + + # ----------------------------------------------------------------- + # TESTS + # ----------------------------------------------------------------- + tests: + name: Run Tests + uses: ./.github/workflows/reusable-tests.yml + needs: [prepare] + with: + ignoreLowerCoverage: false + skipSelenium: true + targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2ed7034d6ca..15b2231ce39 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,12 +26,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v7 - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 00000000000..5bb26e4f6fa --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,22 @@ +# Dependency Review Action +# +# This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging. +# +# Source repository: https://github.com/actions/dependency-review-action +# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement +name: 'Dependency Review' +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@v7 + - name: 'Dependency Review' + uses: actions/dependency-review-action@v5 + with: + vulnerability-check: false diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml new file mode 100644 index 00000000000..83121354e64 --- /dev/null +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -0,0 +1,49 @@ +# GITHUB ACTIONS - WORKFLOW + +# RSync the assets in the shared assets volume + +name: Nightly Dev Shared Assets Sync + +# Controls when the workflow will run +on: + # Run every night + schedule: + - cron: '0 1 * * *' + + # Run on app-rsync-extras.sh changes + push: + branches: + - main + paths: + - 'docker/scripts/app-rsync-extras.sh' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + build: + name: Build Docker Image + runs-on: ubuntu-latest + if: ${{ github.event_name != 'schedule' }} + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v7 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker Build & Push + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: dev/shared-assets-sync/Dockerfile + push: true + tags: ghcr.io/ietf-tools/datatracker-rsync-assets:latest diff --git a/.github/workflows/dev-db-nightly.yml b/.github/workflows/dev-db-nightly.yml deleted file mode 100644 index e6e348f3733..00000000000 --- a/.github/workflows/dev-db-nightly.yml +++ /dev/null @@ -1,51 +0,0 @@ -# GITHUB ACTIONS - WORKFLOW - -# Build the database dev docker image with the latest database dump every night -# so that developers don't have to manually build it themselves. - -name: Nightly Dev DB Image - -# Controls when the workflow will run -on: - # Run every night - schedule: - - cron: '0 0 * * *' - - # Run on db.Dockerfile changes - push: - branches: - - main - paths: - - 'docker/db.Dockerfile' - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -env: - REGISTRY: ghcr.io - IMAGE_NAME: datatracker-db - -jobs: - build: - runs-on: ubuntu-latest - if: ${{ github.ref == 'refs/heads/main' }} - permissions: - contents: read - packages: write - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - - name: Get Current Date as Tag - id: date - run: echo "::set-output name=date::$(date +'%Y%m%d')" - - - name: Docker Build & Push Action - uses: mr-smithers-excellent/docker-build-push@v5.6 - with: - image: ${{ env.IMAGE_NAME }} - tags: nightly-${{ steps.date.outputs.date }}, latest - registry: ${{ env.REGISTRY }} - dockerfile: docker/db.Dockerfile - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml new file mode 100644 index 00000000000..b90300c09b2 --- /dev/null +++ b/.github/workflows/lock-threads.yml @@ -0,0 +1,25 @@ +name: Lock Threads + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + +concurrency: + group: lock + +jobs: + action: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v6 + with: + github-token: ${{ github.token }} + process-only: 'issues, prs' + issue-inactive-days: 7 + pr-inactive-days: 3 + log-output: true diff --git a/.github/workflows/publish-bs3.yml b/.github/workflows/publish-bs3.yml deleted file mode 100644 index 18ed241607d..00000000000 --- a/.github/workflows/publish-bs3.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Publish BS3 - -on: - push: - branches: [ main ] - paths: [ bootstrap/** ] - - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Setup Node.js 16.x - uses: actions/setup-node@v3.0.0 - with: - node-version: 16.x - registry-url: https://registry.npmjs.org/ - - - name: Set package.json version - uses: HarmvZ/set-package-json-version-action@v0.1.2 - with: - version: 1.0.${{ github.run_number }} - path: ./bootstrap - - - name: Build Release - working-directory: bootstrap - run: | - npm install -g grunt-cli - npm ci - grunt dist - - - name: Publish to NPM - run: npm publish --access public - working-directory: bootstrap - env: - NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} - - - name: Upload a Build Artifact - uses: actions/upload-artifact@v3.0.0 - with: - name: dist - path: bootstrap/dist diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..2de02eb2852 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,209 @@ +name: Make Release and Deploy +run-name: RELEASE ${{ github.run_number }} by @${{ github.actor }} + +on: + workflow_dispatch: + inputs: + deployStrategy: + description: 'Deploy Strategy' + default: 'Recreate' + required: true + type: choice + options: + - Recreate + - RollingUpdate + - Current + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + # ----------------------------------------------------------------- + # PREPARE + # ----------------------------------------------------------------- + prepare: + name: Prepare Release + runs-on: ubuntu-latest + outputs: + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} + + steps: + - name: Ensure release branch + if: github.ref != 'refs/heads/release' + run: | + echo "This workflow can only run on the 'release' branch (current: ${{ github.ref_name }})" + exit 1 + + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Get Next Version + id: semver + uses: ietf-tools/semver-action@v1 + with: + token: ${{ github.token }} + branch: release + skipInvalidTags: true + patchList: fix, bugfix, perf, refactor, test, tests, chore + + - name: Create Draft Release + uses: ncipollo/release-action@v1.21.0 + with: + prerelease: true + draft: false + commit: ${{ github.sha }} + tag: ${{ steps.semver.outputs.nextStrict }} + name: ${{ steps.semver.outputs.nextStrict }} + body: '*pending*' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Build Variables + id: buildvars + run: | + echo "Will set build version to: ${{ steps.semver.outputs.nextStrict }}" + echo "buildVersion=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT + echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" + + - name: Get Base Image Target Version + id: baseimgversion + run: | + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + + # ----------------------------------------------------------------- + # TESTS + # ----------------------------------------------------------------- + tests: + name: Run Tests + uses: ./.github/workflows/reusable-tests.yml + needs: [prepare] + secrets: inherit + with: + ignoreLowerCoverage: false + skipSelenium: true + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # BUILD RELEASE + # ----------------------------------------------------------------- + build: + name: Build Release Image + uses: ./.github/workflows/reusable-build.yml + if: ${{ !failure() && !cancelled() }} + needs: [tests, prepare] + permissions: + contents: write + packages: write + secrets: inherit + with: + buildVersion: ${{needs.prepare.outputs.buildVersion}} + isReleaseBuild: true + previousVersion: ${{needs.prepare.outputs.previousVersion}} + handleCoverageResults: true + updateCoverageBaseline: true + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # NOTIFY + # ----------------------------------------------------------------- + notify: + name: Notify + if: ${{ always() }} + needs: [prepare, tests, build] + runs-on: ubuntu-latest + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Notify on Slack (Success) + if: ${{ !contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "28a745" + fields: + - title: "Status" + short: true + value: "Completed" + - name: Notify on Slack (Failure) + if: ${{ contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "a82929" + fields: + - title: "Status" + short: true + value: "Failed 😱" + + # ----------------------------------------------------------------- + # STAGING + # ----------------------------------------------------------------- + staging: + name: Deploy to Staging + if: ${{ !failure() && !cancelled() }} + needs: [prepare, build] + runs-on: ubuntu-latest + environment: + name: staging + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Refresh Staging DB + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy-db.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' + waitForCompletionTimeout: 120m + + - name: Deploy to staging + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m + + # ----------------------------------------------------------------- + # PROD + # ----------------------------------------------------------------- + prod: + name: Deploy to Production + if: ${{ !failure() && !cancelled() }} + needs: [prepare, staging] + runs-on: ubuntu-latest + environment: + name: production + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Deploy to production + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml new file mode 100644 index 00000000000..b3add89834a --- /dev/null +++ b/.github/workflows/reusable-build.yml @@ -0,0 +1,207 @@ +name: Reusable Build Workflow + +on: + workflow_call: + inputs: + buildVersion: + required: true + type: string + isReleaseBuild: + default: false + required: false + type: boolean + previousVersion: + required: false + type: string + handleCoverageResults: + default: true + required: false + type: boolean + updateCoverageBaseline: + default: false + required: false + type: boolean + baseImageVersion: + default: latest + required: false + type: string + skipArmBuild: + default: false + required: false + type: boolean + +jobs: + # ----------------------------------------------------------------- + # BUILD IMAGE + # ----------------------------------------------------------------- + build: + name: Build Image + runs-on: + group: hperf-8c32r + permissions: + contents: write + packages: write + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Setup Node.js environment + uses: actions/setup-node@v6 + with: + node-version: 18.x + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.x" + + - name: Setup AWS CLI + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2.22.35 + + - name: Download a Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Make Build + env: + DEBIAN_FRONTEND: noninteractive + BROWSERSLIST_IGNORE_OLD_DATA: 1 + run: | + echo "BUILD_VERSION: ${{ inputs.buildVersion }}" + echo "GITHUB_SHA: $GITHUB_SHA" + echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" + echo "Running frontend build script..." + echo "Installing node packages..." + npm ci + echo "Packaging static assets..." + npm run build -- --base=https://static.ietf.org/dt/${{ inputs.buildVersion }}/ + npm run legacy:build + echo "Setting version ${{ inputs.buildVersion }}..." + sed -i -r -e "s|^__version__ += '.*'$|__version__ = '${{ inputs.buildVersion }}'|" ietf/__init__.py + sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py + sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py + + - name: Set Production Flags + if: ${{ inputs.isReleaseBuild }} + run: | + echo "Setting production flags in settings.py..." + sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py + + - name: Make Tarball + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Build release tarball..." + mkdir -p /home/runner/work/release + tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . + + - name: Collect + Push Statics + env: + DEBIAN_FRONTEND: noninteractive + AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} + AWS_DEFAULT_REGION: auto + AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} + run: | + echo "Collecting statics..." + echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }}" + docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }} sh dev/build/collectstatics.sh + echo "Pushing statics..." + cd static + aws s3 sync . s3://static/dt/${{ inputs.buildVersion }} --only-show-errors + + - name: Augment dockerignore for docker image build + env: + DEBIAN_FRONTEND: noninteractive + run: | + cat >> .dockerignore <> $GITHUB_ENV + + - name: Build Images + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: dev/build/Dockerfile + platforms: ${{ inputs.skipArmBuild && 'linux/amd64' || 'linux/amd64,linux/arm64' }} + push: true + tags: | + ghcr.io/ietf-tools/datatracker:${{ inputs.buildVersion }} + ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} + + - name: Generate CHANGELOG + id: changelog + uses: Requarks/changelog-action@v1 + if: ${{ inputs.isReleaseBuild }} + with: + token: ${{ github.token }} + fromTag: ${{ inputs.buildVersion }} + toTag: ${{ inputs.previousVersion }} + writeToFile: false + + - name: Download Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8 + with: + name: coverage + + - name: Create Release + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.isReleaseBuild }} + with: + allowUpdates: true + makeLatest: true + draft: false + tag: ${{ inputs.buildVersion }} + name: ${{ inputs.buildVersion }} + body: ${{ steps.changelog.outputs.changes }} + artifacts: "/home/runner/work/release/release.tar.gz,coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update Baseline Coverage + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.updateCoverageBaseline }} + with: + allowUpdates: true + tag: baseline + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + omitPrereleaseDuringUpdate: true + replacesArtifacts: true + artifacts: "coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload Build Artifacts + uses: actions/upload-artifact@v7 + with: + name: release-${{ inputs.buildVersion }} + path: /home/runner/work/release/release.tar.gz diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml new file mode 100644 index 00000000000..d60dc9857ce --- /dev/null +++ b/.github/workflows/reusable-tests.yml @@ -0,0 +1,190 @@ +name: Reusable Tests Workflow + +on: + workflow_call: + inputs: + ignoreLowerCoverage: + description: 'Ignore Lower Coverage' + default: false + required: true + type: boolean + skipSelenium: + description: 'Skip Selenium Tests' + default: false + required: false + type: boolean + targetBaseVersion: + description: 'Target Base Image Version' + default: latest + required: false + type: string + +jobs: + tests-python: + name: Python Tests + runs-on: ubuntu-latest + container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} + + services: + db: + image: ghcr.io/ietf-tools/datatracker-db:latest + blobstore: + image: ghcr.io/ietf-tools/datatracker-devblobstore:latest + + steps: + - uses: actions/checkout@v7 + + - name: Prepare for tests + run: | + chmod +x ./dev/tests/prepare.sh + sh ./dev/tests/prepare.sh + + - name: Ensure DB is ready + run: | + /usr/local/bin/wait-for db:5432 -- echo "DB ready" + + - name: Run all tests + shell: bash + run: | + echo "Running checks..." + ./ietf/manage.py check + ./ietf/manage.py migrate --fake-initial + echo "Validating migrations..." + if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then + echo "Model changes without migrations found." + exit 1 + fi + if [[ "x${{ inputs.skipSelenium }}" == "xtrue" ]]; then + echo "Disable selenium tests..." + rm /usr/bin/geckodriver + fi + echo "Running tests..." + if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then + echo "Lower coverage failures will be ignored." + HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage + else + HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test + fi + coverage xml + + - name: Upload geckodriver.log + uses: actions/upload-artifact@v7 + if: ${{ failure() }} + with: + name: geckodriverlog + path: geckodriver.log + + - name: Upload Coverage Results to Codecov + uses: codecov/codecov-action@v7 + with: + disable_search: true + files: coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Convert Coverage Results + if: ${{ always() }} + run: | + mv latest-coverage.json coverage.json + + - name: Upload Coverage Results as Build Artifact + uses: actions/upload-artifact@v7 + if: ${{ always() }} + with: + name: coverage + path: coverage.json + + tests-playwright: + name: Playwright Tests + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + project: [chromium, firefox] + + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v6 + with: + node-version: '26' + + - name: Run all tests + run: | + echo "Installing dependencies..." + npm ci + echo "Installing Playwright..." + cd playwright + mkdir test-results + npm ci + npx playwright install --with-deps ${{ matrix.project }} + echo "Running tests..." + npx playwright test --project=${{ matrix.project }} + + - name: Upload Report + uses: actions/upload-artifact@v7 + if: ${{ always() }} + continue-on-error: true + with: + name: playwright-results-${{ matrix.project }} + path: playwright/test-results/ + if-no-files-found: ignore + + tests-playwright-legacy: + if: ${{ false }} # disable until we sort out suspected test runner issue + name: Playwright Legacy Tests + runs-on: ubuntu-latest + container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} + strategy: + fail-fast: false + matrix: + project: [chromium, firefox] + + services: + db: + image: ghcr.io/ietf-tools/datatracker-db:latest + + steps: + - uses: actions/checkout@v7 + + - name: Prepare for tests + run: | + chmod +x ./dev/tests/prepare.sh + sh ./dev/tests/prepare.sh + + - name: Ensure DB is ready + run: | + /usr/local/bin/wait-for db:5432 -- echo "DB ready" + + - name: Start Datatracker + run: | + echo "Running checks..." + ./ietf/manage.py check + ./ietf/manage.py migrate --fake-initial + echo "Starting datatracker..." + ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local & + echo "Waiting for datatracker to be ready..." + /usr/local/bin/wait-for localhost:8000 -- echo "Datatracker ready" + + - name: Run all tests + env: + # Required to get firefox to run as root: + HOME: "" + run: | + echo "Installing dependencies..." + npm ci + echo "Installing Playwright..." + cd playwright + mkdir test-results + npm ci + npx playwright install --with-deps ${{ matrix.project }} + echo "Running tests..." + npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js + + - name: Upload Report + uses: actions/upload-artifact@v7 + if: ${{ always() }} + continue-on-error: true + with: + name: playwright-legacy-results-${{ matrix.project }} + path: playwright/test-results/ + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 7de6c658248..ccc7a46b08d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,57 +1,40 @@ -# A simulation of Subversion default ignores, generated by reposurgeon. -*.o -*.lo -*.la -*.al -*.libs -*.so -*.so.[0-9]* -*.a -*.pyc -*.pyo -*.rej -*~ -*.#* -.*.swp .DS_store -# Simulated Subversion default ignores end here +datatracker.sublime-project +datatracker.sublime-workspace +/.claude /.coverage +/.factoryboy_random_state /.mypy_cache +/.parcel-cache /.project /.pydevproject /.Python /.settings /.tmp -/.factoryboy_random_state -/attic -/bin -/etc +/.vite +/client/dist +/data +/dist +/docker/docker-compose.extend-custom.yml /env /ghostdriver.log +/geckodriver.log /htmlcov -/include -/INSTALL.pdf +/ietf/static/dist-neue /latest-coverage.json -/lib -/local /media -/mergelog -/mergequeue -/minimum-requirements.txt -/pip-selfcheck.json -/selenium -/settings_local.py -/share +/node_modules +/release-coverage.json /static -/testresult -/tmp -/tmp-nomcom-public-keys-dir -/trunk27 -/trunk36 -/trunk37 -/unix.tag +/tmp-* +/.testresult +*.swp *.pyc __pycache__ -node_modules -ietf/static/ietf/bootstrap -/docker/docker-compose.extend-custom.yml \ No newline at end of file +.yarn/* +!.yarn/cache +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions diff --git a/.npmrc b/.npmrc index 20daf462178..fcbbb678280 100644 --- a/.npmrc +++ b/.npmrc @@ -1,4 +1,5 @@ +audit = false +fund = false save-exact = true save-prefix = "" -fund = false -loglevel = "warn" \ No newline at end of file +update-notifier = false diff --git a/.oxfmtrc.json b/.oxfmtrc.json new file mode 100644 index 00000000000..518e8c95e1d --- /dev/null +++ b/.oxfmtrc.json @@ -0,0 +1,9 @@ +{ + "$schema": "./node_modules/oxfmt/configuration_schema.json", + "semi": false, + "singleQuote": true, + "trailingComma": "none", + "bracketSameLine": true, + "endOfLine": "lf", + "insertFinalNewline": true +} diff --git a/.oxlintrc.json b/.oxlintrc.json new file mode 100644 index 00000000000..d5648b966b8 --- /dev/null +++ b/.oxlintrc.json @@ -0,0 +1,10 @@ +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "plugins": ["eslint", "typescript", "unicorn", "oxc", "vue"], + "env": { + "browser": true + }, + "categories": { + "correctness": "error" + } +} diff --git a/.pylintrc b/.pylintrc index c9a33fcec2c..008f89e4547 100644 --- a/.pylintrc +++ b/.pylintrc @@ -405,4 +405,4 @@ analyse-fallback-blocks=no # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=builtins.Exception diff --git a/.sassrc.json b/.sassrc.json new file mode 100644 index 00000000000..0dea8892cc4 --- /dev/null +++ b/.sassrc.json @@ -0,0 +1,3 @@ +{ + "silenceDeprecations": ["import", "if-function", "global-builtin", "color-functions"] +} diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000000..1612a87fe84 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": [] +} diff --git a/.vscode/launch.json b/.vscode/launch.json index 8176b439588..227eb8f6155 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,10 +12,33 @@ "program": "${workspaceFolder}/ietf/manage.py", "args": [ "runserver", - "0.0.0.0:8000", + "0.0.0.0:8001", "--settings=settings_local" ], - "django": true + "django": true, + "presentation": { + "hidden": false, + "group": "main", + "order": 1 + } + }, + { + "name": "Run Server in Vite Dev Mode", + "type": "python", + "request": "launch", + "python": "/usr/local/bin/python", + "program": "${workspaceFolder}/ietf/manage.py", + "args": [ + "runserver", + "0.0.0.0:8001", + "--settings=settings_local_vite" + ], + "django": true, + "presentation": { + "hidden": false, + "group": "main", + "order": 1 + } }, { "name": "Run Server with Debug Toolbar", @@ -25,10 +48,15 @@ "program": "${workspaceFolder}/ietf/manage.py", "args": [ "runserver", - "0.0.0.0:8000", + "0.0.0.0:8001", "--settings=settings_local_debug" ], - "django": true + "django": true, + "presentation": { + "hidden": false, + "group": "main", + "order": 1 + } } ] -} \ No newline at end of file +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 7dcffaa3f20..f15519dae81 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,45 +1,49 @@ { - "taskExplorer.exclude": [ - "**/.vscode-test/**", - "**/bin/**", - "**/build/**", - "**/CompiledOutput/**", - "**/dist/**", - "**/doc/**", - "**/ext/**", - "**/out/**", - "**/output/**", - "**/packages/**", - "**/release/**", - "**/releases/**", - "**/samples/**", - "**/sdks/**", - "**/static/**", - "**/target/**", - "**/test/**", - "**/third_party/**", - "**/vendor/**", - "**/work/**", - "/workspace/bootstrap/nuget/MyGet.ps1" - ], - "taskExplorer.enableAnt": false, - "taskExplorer.enableAppPublisher": false, - "taskExplorer.enablePipenv": false, - "taskExplorer.enableBash": false, - "taskExplorer.enableBatch": false, - "taskExplorer.enableGradle": false, - "taskExplorer.enableGrunt": false, - "taskExplorer.enableGulp": false, - "taskExplorer.enablePerl": false, - "taskExplorer.enableMake": false, - "taskExplorer.enableMaven": false, - "taskExplorer.enableNsis": false, - "taskExplorer.enableNpm": false, - "taskExplorer.enablePowershell": false, - "taskExplorer.enablePython": false, - "taskExplorer.enableRuby": false, - "taskExplorer.enableTsc": false, - "taskExplorer.enableWorkspace": true, - "taskExplorer.enableExplorerView": false, - "taskExplorer.enableSideBar": true -} \ No newline at end of file + "search.exclude": { + "**/.yarn": true, + "**/.pnp.*": true + }, + "python.linting.pylintArgs": ["--load-plugins", "pylint_django"], + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": false, + "python.linting.enabled": true, + "python.terminal.shellIntegration.enabled": false, + "vs-kubernetes": { + "disable-linters": ["resource-limits"] + }, + "editor.codeActionsOnSave": { + "source.fixAll.oxc": "explicit" + }, + "[javascript]": { + "editor.defaultFormatter": "oxc.oxc-vscode", + "editor.formatOnSave": true + }, + "[vue]": { + "editor.defaultFormatter": "oxc.oxc-vscode", + "editor.formatOnSave": true + }, + "[scss]": { + "editor.defaultFormatter": "oxc.oxc-vscode", + "editor.formatOnSave": true + }, + "[css]": { + "editor.defaultFormatter": "oxc.oxc-vscode", + "editor.formatOnSave": true + }, + "[postcss]": { + "editor.defaultFormatter": "oxc.oxc-vscode", + "editor.formatOnSave": true + }, + "[json]": { + "editor.defaultFormatter": "oxc.oxc-vscode", + "editor.formatOnSave": true + }, + "[jsonc]": { + "editor.defaultFormatter": "oxc.oxc-vscode", + "editor.formatOnSave": true + }, + "[html]": { + "editor.defaultFormatter": "oxc.oxc-vscode", + "editor.formatOnSave": true + } +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 7680b79cbe6..8b36b0e6ac9 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -22,6 +22,25 @@ }, "problemMatcher": [] }, + { + "label": "Run Migrations", + "type": "shell", + "command": "/usr/local/bin/python", + "args": [ + "${workspaceFolder}/ietf/manage.py", + "migrate", + "--settings=settings_local" + ], + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": true, + "clear": false + }, + "problemMatcher": [] + }, { "label": "Run All Tests", "type": "shell", @@ -29,7 +48,7 @@ "args": [ "${workspaceFolder}/ietf/manage.py", "test", - "--settings=settings_local_sqlitetest" + "--settings=settings_test" ], "group": "test", "presentation": { @@ -49,8 +68,8 @@ "args": [ "${workspaceFolder}/ietf/manage.py", "test", - "--settings=settings_local_sqlitetest", - "--pattern=tests_js.py", + "--settings=settings_test", + "--pattern=tests_js.py" ], "group": "test", "presentation": { @@ -84,7 +103,15 @@ "label": "Run SMTP Debugging Server", "type": "shell", "command": "/usr/local/bin/python", - "args": ["-m", "smtpd", "-n", "-c", "DebuggingServer", "localhost:2025"], + "args": [ + "-m", + "aiosmtpd", + "-n", + "-c", + "ietf.utils.aiosmtpd.DevDebuggingHandler", + "-l", + "localhost:2025" + ], "presentation": { "echo": true, "reveal": "always", @@ -146,6 +173,28 @@ "clear": false }, "problemMatcher": [] + }, + { + "label": "Run Vite Dev Server", + "type": "shell", + "command": "yarn", + "args": [ + "dev" + ], + "isBackground": true, + "presentation": { + "group": "main", + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": false, + "clear": false + }, + "runOptions": { + "instanceLimit": 1 + }, + "problemMatcher": [] } ] -} \ No newline at end of file +} diff --git a/LICENSE b/LICENSE index 6ffb15e4bae..dc6e0c56635 100644 --- a/LICENSE +++ b/LICENSE @@ -1,19 +1,21 @@ -Copyright (c) 2008,2018, The IETF Trust +BSD 3-Clause License + +Copyright (c) 2008-2024, The IETF Trust All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE diff --git a/README.md b/README.md index 6c61fd9af83..7e2ee131f02 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ [![Release](https://img.shields.io/github/release/ietf-tools/datatracker.svg?style=flat&maxAge=300)](https://github.com/ietf-tools/datatracker/releases) [![License](https://img.shields.io/github/license/ietf-tools/datatracker)](https://github.com/ietf-tools/datatracker/blob/main/LICENSE) -[![Nightly Dev DB Image](https://github.com/ietf-tools/datatracker/actions/workflows/dev-db-nightly.yml/badge.svg)](https://github.com/ietf-tools/datatracker/pkgs/container/datatracker-db) -[![Python Version](https://img.shields.io/badge/python-3.6-blue?logo=python&logoColor=white)](#prerequisites) -[![Django Version](https://img.shields.io/badge/django-2.x-51be95?logo=django&logoColor=white)](#prerequisites) -[![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites) -[![MariaDB Version](https://img.shields.io/badge/mariadb-10-blue?logo=mariadb&logoColor=white)](#prerequisites) +[![Code Coverage](https://codecov.io/gh/ietf-tools/datatracker/branch/feat/bs5/graph/badge.svg?token=V4DXB0Q28C)](https://codecov.io/gh/ietf-tools/datatracker) +[![Python Version](https://img.shields.io/badge/python-3.12-blue?logo=python&logoColor=white)](#prerequisites) +[![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites) +[![Node Version](https://img.shields.io/badge/node.js-26.x-green?logo=node.js&logoColor=white)](#prerequisites) +[![MariaDB Version](https://img.shields.io/badge/postgres-17-blue?logo=postgresql&logoColor=white)](#prerequisites) ##### The day-to-day front-end to the IETF database for people who work on IETF standards. @@ -17,11 +17,26 @@ - [**Production Website**](https://datatracker.ietf.org) - [Changelog](https://github.com/ietf-tools/datatracker/releases) - [Contributing](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md) -- [Getting Started](#getting-started) +- [Getting Started](#getting-started) - *[ tl;dr ](#the-tldr-to-get-going)* + - [Creating a Fork](#creating-a-fork) - [Git Cloning Tips](#git-cloning-tips) - [Docker Dev Environment](docker/README.md) - [Database & Assets](#database--assets) - [Old Datatracker Branches](https://github.com/ietf-tools/old-datatracker-branches/branches/all) +- [Frontend Development](#frontend-development) + - [Intro](#intro) + - [Vite](#vite-vue-3) + - [Parcel](#parcel-legacyjquery) + - [Bootstrap](#bootstrap) + - [Serving Static Files via CDN](#serving-static-files-via-cdn) + - [Handling of External Javascript and CSS Components](#handling-of-external-javascript-and-css-components) + - [Handling of Internal Static Files](#handling-of-internal-static-files) + - [Changes to Template Files](#changes-to-template-files) + - [Deployment](#deployment) +- [Running Tests](#running-tests) + - [Python](#python-tests) + - [Frontend](#frontend-tests) + - [Diff Tool](#diff-tool) --- @@ -29,14 +44,24 @@ This project is following the standard **Git Feature Workflow** development model. Learn about all the various steps of the development workflow, from creating a fork to submitting a pull request, in the [Contributing](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md) guide. +> [!TIP] > Make sure to read the [Styleguides](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md#styleguides) section to ensure a cohesive code format across the project. You can submit bug reports, enhancement and new feature requests in the [discussions](https://github.com/ietf-tools/datatracker/discussions) area. Accepted tickets will be converted to issues. +#### Creating a Fork + +Click the Fork button in the top-right corner of the repository to create a personal copy that you can work on. + +> [!NOTE] +> Some GitHub Actions might be enabled by default in your fork. You should disable them by going to **Settings** > **Actions** > **General** and selecting **Disable actions** (then Save). + #### Git Cloning Tips As outlined in the [Contributing](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md) guide, you will first want to create a fork of the datatracker project in your personal GitHub account before cloning it. +Windows developers: [Start with WSL2 from the beginning](https://github.com/ietf-tools/.github/blob/main/docs/windows-dev.md). + Because of the extensive history of this project, cloning the datatracker project locally can take a long time / disk space. You can speed up the cloning process by limiting the history depth, for example *(replace `USERNAME` with your GitHub username)*: - To fetch only up to the 10 latest commits: @@ -48,6 +73,25 @@ Because of the extensive history of this project, cloning the datatracker projec git clone --shallow-since=DATE https://github.com/USERNAME/datatracker.git ``` +#### The tl;dr to get going + +Note that you will have to have cloned the datatracker code locally - please read the above sections. + +Datatracker development is performed using Docker containers. You will need to be able to run docker (and docker-compose) on your machine to effectively develop. It is possible to get a purely native install working, but it is _very complicated_ and typically takes a first time datatracker developer a full day of setup, where the docker setup completes in a small number of minutes. + +Many developers are using [VS Code](https://code.visualstudio.com/) and taking advantage of VS Code's ability to start a project in a set of containers. If you are using VS Code, simply start VS Code in your clone and inside VS Code choose `Restart in container`. + +If VS Code is not available to you, in your clone, type `cd docker; ./run` + +Once the containers are started, run the tests to make sure your checkout is a good place to start from (all tests should pass - if any fail, ask for help at tools-help@). Inside the app container's shell type: +```sh +ietf/manage.py test --settings=settings_test +``` + +Note that we recently moved the datatracker onto PostgreSQL - you may still find older documentation that suggests testing with settings_sqlitetest. That will no longer work. + +For a more detailed description of getting going, see [docker/README.md](docker/README.md). + #### Overview of the datatracker models A beginning of a [walkthrough of the datatracker models](https://notes.ietf.org/iab-aid-datatracker-database-overview) was prepared for the IAB AID workshop. @@ -60,8 +104,215 @@ Read the [Docker Dev Environment](docker/README.md) guide to get started. ### Database & Assets -Nightly database dumps of the datatracker are available at -https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz +Nightly database dumps of the datatracker are available as Docker images: `ghcr.io/ietf-tools/datatracker-db:latest` + +> [!TIP] +> In order to update the database in your dev environment to the latest version, you should run the `docker/cleandb` script. + +### Blob storage for dev/test + +The dev and test environments use [minio](https://github.com/minio/minio) to provide local blob storage. See the settings files for how the app container communicates with the blobstore container. If you need to work with minio directly from outside the containers (to interact with its api or console), use `docker compose` from the top level directory of your clone to expose it at an ephemeral port. + +``` +$ docker compose port blobstore 9001 +0.0.0.0: + +$ curl -I http://localhost: +HTTP/1.1 200 OK +... +``` + + +The minio container exposes the minio api at port 9000 and the minio console at port 9001 + + +### Frontend Development + +#### Intro + +Assets are built + bundled using `vite` / `parcel`. + +The datatracker uses 2 different build systems, depending on the use case: +- [**Vite**](https://vitejs.dev/) for Vue 3 pages / components +- [**Parcel**](https://parceljs.org/) for legacy pages / jQuery + +#### Vite *(Vue 3)* + +Pages will gradually be updated to Vue 3 components. These components are located under the `/client` directory. + +Each Vue 3 app has its own sub-directory. For example, the agenda app is located under `/client/agenda`. + +The datatracker makes use of the Django-Vite plugin to point to either the Vite.js server or the precompiled production files. The `DJANGO_VITE["default"]["dev_mode"]` flag, found in the `ietf/settings_local.py` file determines whether the Vite.js server is used or not. + +In development mode, you must start the Vite.js development server, in addition to the usual Datatracker server: + +```sh +npm run dev +``` + +Any changes made to the files under `/client` will automatically trigger a hot-reload of the modified components. + +To generate production assets, run the build command: + +```sh +npm run build +``` + +This will create packages under `ietf/static/dist-neue`, which are then served by the Django development server, and which must be uploaded to the CDN. + +#### Parcel *(Legacy/jQuery)* + +The Datatracker includes these packages from the various Javascript and CSS files in `ietf/static/js` and `ietf/static/css` respectively, bundled using Parcel. +Static images are likewise in `ietf/static/images`. + +Whenever changes are made to the files under `ietf/static`, you must re-run the build command to package them: + +``` shell +npm run legacy:build +``` + +This will create packages under `ietf/static/dist/ietf`, which are then served by the Django development server, and which must be uploaded to the CDN. + +#### Bootstrap + +The "new" datatracker uses Twitter Bootstrap for the UI. + +Get familiar with and use those UI elements, CSS classes, etc. instead of cooking up your own. + +Some ground rules: + +- Think hard before tweaking the bootstrap CSS, it will make it harder to upgrade to future releases. +- No `').appendTo('head') - - $('