diff --git a/.github/workflows/build-celery-worker.yml b/.github/workflows/build-celery-worker.yml deleted file mode 100644 index d14e4f2c8b..0000000000 --- a/.github/workflows/build-celery-worker.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Build Celery Worker Docker Image - -on: - push: - branches: - - 'main' - paths: - - 'requirements.txt' - - 'dev/celery/**' - - '.github/workflows/build-celery-worker.yml' - - workflow_dispatch: - -jobs: - publish: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker Build & Push - uses: docker/build-push-action@v6 - env: - DOCKER_BUILD_NO_SUMMARY: true - with: - context: . - file: dev/celery/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ghcr.io/ietf-tools/datatracker-celery:latest - diff --git a/dev/celery/Dockerfile b/dev/celery/Dockerfile deleted file mode 100644 index 12eb15eb81..0000000000 --- a/dev/celery/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# Dockerfile for celery worker -# -FROM ghcr.io/ietf-tools/datatracker-app-base:latest -LABEL maintainer="IETF Tools Team " - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get purge -y imagemagick imagemagick-6-common - -# Copy the startup file -COPY dev/celery/docker-init.sh /docker-init.sh -RUN sed -i 's/\r$//' /docker-init.sh && \ - chmod +x /docker-init.sh - -# Install current datatracker python dependencies -COPY requirements.txt /tmp/pip-tmp/ -RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt -RUN rm -rf /tmp/pip-tmp - -# Add watchmedo utility for dev containers -RUN pip3 --disable-pip-version-check --no-cache-dir install watchdog[watchmedo] - -ENTRYPOINT [ "/docker-init.sh" ] diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js index 4aee7ba1ed..1a2d993ac4 100644 --- a/dev/deploy-to-container/cli.js +++ b/dev/deploy-to-container/cli.js @@ -70,6 +70,7 @@ async function main () { .replace('__HOSTNAME__', hostname) ) await fs.copy(path.join(basePath, 'docker/scripts/app-create-dirs.sh'), path.join(releasePath, 'app-create-dirs.sh')) + await fs.copy(path.join(basePath, 'docker/scripts/app-init-celery.sh'), path.join(releasePath, 'app-init-celery.sh')) await fs.copy(path.join(basePath, 'dev/deploy-to-container/start.sh'), path.join(releasePath, 'start.sh')) await fs.copy(path.join(basePath, 'test/data'), path.join(releasePath, 'test/data')) console.info('Updated configuration files.') @@ -98,14 +99,6 @@ async function main () { }) console.info('Pulled latest MQ docker image.') - // Pull latest Celery image - console.info('Pulling latest Celery docker image...') - const celeryImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-celery:latest') - await new Promise((resolve, reject) => { - dock.modem.followProgress(celeryImagePullStream, (err, res) => err ? reject(err) : resolve(res)) - }) - console.info('Pulled latest Celery docker image.') - // Terminate existing containers console.info('Ensuring existing containers with same name are terminated...') const containers = await dock.listContainers({ all: true }) @@ -221,7 +214,7 @@ async function main () { const celeryContainers = {} for (const conConf of conConfs) { celeryContainers[conConf.name] = await dock.createContainer({ - Image: 'ghcr.io/ietf-tools/datatracker-celery:latest', + Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest', name: `dt-${conConf.name}-${branch}`, Hostname: `dt-${conConf.name}-${branch}`, Env: [ @@ -243,7 +236,7 @@ async function main () { Name: 'unless-stopped' } }, - Cmd: ['--loglevel=INFO'] + Entrypoint: ['bash', '-c', 'chmod +x ./app-init-celery.sh && ./app-init-celery.sh'] }) } console.info('Created Celery docker containers successfully.') diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile index e44200398c..e7c7b9cc3f 100644 --- a/docker/celery.Dockerfile +++ b/docker/celery.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-celery:latest +FROM ghcr.io/ietf-tools/datatracker-app-base:latest LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive @@ -34,7 +34,7 @@ RUN bash /tmp/library-scripts/docker-setup-python.sh "none" "/usr/local" "${PIPX RUN rm -rf /tmp/library-scripts # Copy the startup file -COPY dev/celery/docker-init.sh /docker-init.sh +COPY docker/scripts/app-init-celery.sh /docker-init.sh RUN sed -i 's/\r$//' /docker-init.sh && \ chmod +x /docker-init.sh diff --git a/docker/docker-compose.celery.yml b/docker/docker-compose.celery.yml deleted file mode 100644 index b6cc3d09e8..0000000000 --- a/docker/docker-compose.celery.yml +++ /dev/null @@ -1,47 +0,0 @@ -services: - mq: - image: rabbitmq:3-alpine - user: '${RABBITMQ_UID:-499:499}' - hostname: datatracker-mq -# deploy: -# resources: -# limits: -# memory: 1gb # coordinate with settings in rabbitmq.conf -# reservations: -# memory: 512mb - mem_limit: 1gb # coordinate with settings in rabbitmq.conf - ports: - - '${MQ_PORT:-5672}:5672' - volumes: - - ./lib.rabbitmq:/var/lib/rabbitmq - - ./rabbitmq.conf:/etc/rabbitmq/conf.d/90-ietf.conf - - ./definitions.json:/ietf-conf/definitions.json - restart: unless-stopped - logging: - driver: "syslog" - options: - syslog-address: 'unixgram:///dev/log' - tag: 'docker/{{.Name}}' -# syslog-address: "tcp://ietfa.amsl.com:514" - - celery: - image: ghcr.io/ietf-tools/datatracker-celery:latest - environment: - CELERY_APP: ietf - # UPDATE_REQUIREMENTS: 1 # uncomment to update Python requirements on startup - command: - - '--loglevel=INFO' - user: '${CELERY_UID:-499:499}' - volumes: - - '${DATATRACKER_PATH:-..}:/workspace' - - '${MYSQL_SOCKET_PATH:-/run/mysql}:/run/mysql' - depends_on: - - mq - network_mode: 'service:mq' - restart: unless-stopped - logging: - driver: "syslog" - options: - syslog-address: 'unixgram:///dev/log' - tag: 'docker/{{.Name}}' -# syslog-address: "tcp://ietfa.amsl.com:514" diff --git a/dev/celery/docker-init.sh b/docker/scripts/app-init-celery.sh similarity index 90% rename from dev/celery/docker-init.sh rename to docker/scripts/app-init-celery.sh index 6a36cb6d78..5788b943da 100755 --- a/dev/celery/docker-init.sh +++ b/docker/scripts/app-init-celery.sh @@ -90,19 +90,22 @@ if [[ "${CELERY_ROLE}" == "worker" ]]; then run_as_celery_uid /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check fi +USER_BIN_PATH="/home/dev/.local/bin" +WATCHMEDO="$USER_BIN_PATH/watchmedo" +CELERY="$USER_BIN_PATH/celery" trap 'trap "" TERM; cleanup' TERM # start celery in the background so we can trap the TERM signal -if [[ -n "${DEV_MODE}" ]]; then - watchmedo auto-restart \ +if [[ -n "${DEV_MODE}" && -x "${WATCHMEDO}" ]]; then + $WATCHMEDO auto-restart \ --patterns '*.py' \ --directory 'ietf' \ --recursive \ --debounce-interval 5 \ -- \ - celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" $@ & + $CELERY --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" $@ & celery_pid=$! else - celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" & + $CELERY --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" & celery_pid=$! fi