Version of the new docker setup emailed out for testing on 2021-10-12

 - Legacy-Id: 19417

Author: larseggert

docker/Dockerfile

@@ -1,185 +1,85 @@
-# -*- shell-mode -*-
-# This file is a docker (https://www.docker.com/what-docker) recipe, which can be used to build
-# a docker image which is ready to run a datatracker in development mode.
+# This is a Dockerfile with everything in it to run the IETF datatracker.
 #
-# It is used to build an image (once you've installed docker) using a command like this (assuming
-# suitable replacement of $variables:
+# If you make changes to the datatracker that add new dependencies (python
+# packages or otherwise), you need to rebuild this image to make them
+# available. Do this in the top-level directory of your datatracker source
+# tree:
 #
-#	$ docker build -t $yourdockerid/datatracker:$version
+# docker build --tag dt:latest .
 #
-# To use a pre-built image, assuming we're on OS X and have a checked-out datatracker repository
-# at /Users/$login/src/6.8.1.dev0, you would start (again assuming you've installed docker)
-# a container from an image, as follows:
-# 
-#	$ docker run -ti --name=$containername -v /Users/$login:/home/$login levkowetz/datatracker:$version /bin/bash
-# 
-# This maps your home directory to /home/$login in the container, and starts it running /bin/bash.
-# 
-# In this first version, the docker environment is set up so that tests will run successfully,
-# but the database has *not* been loaded with a dump, and supporting files (drafts, charters, etc.)
-# have *not* been downloaded.
+# You can then execute the datatracker like this (also from the top-level
+# datatracker source directory):
+#
+# docker run -ti -v $(pwd):/root/src dt:latest
 
-FROM dyne/devuan:beowulf
-LABEL maintainer="Henrik Levkowetz <henrik@levkowetz.com>"
+FROM ubuntu:hirsute
+LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 # Default django runserver port
-EXPOSE	8000
-
-# Run apt-get noninteractive
-ENV DEBIAN_FRONTEND=noninteractive
-ENV DEVUAN_FRONTEND=noninteractive
-
-# Uncomment this to be able to install and run apt-show-versions:
-RUN rm -v /etc/apt/apt.conf.d/docker-compress
-RUN rm -v /var/lib/apt/lists/*lz4
-
-RUN apt-get update --allow-releaseinfo-change
-RUN apt-get install -qy apt-transport-https
-
-# Use backports, updates, and security updates
-RUN echo "deb http://deb.devuan.org/merged beowulf main contrib non-free"  > /etc/apt/sources.list
-RUN echo "deb http://deb.devuan.org/merged beowulf-security main contrib non-free" >> /etc/apt/sources.list
-RUN echo "deb http://deb.devuan.org/merged beowulf-updates main contrib non-free" >> /etc/apt/sources.list
-RUN echo "deb http://deb.devuan.org/merged beowulf-backports main contrib non-free" >> /etc/apt/sources.list
+EXPOSE 8000
 
-# Remove some excludes for the docker image
-RUN sed -i -e '/^path-exclude=.*\/groff/d' \
-           -e '/^path-exclude=.*\/locale/d' \
-           -e '/^path-exclude=.*\/man/d' /etc/dpkg/dpkg.cfg.d/docker-excludes
+# Default mysqld/mariadb port
+EXPOSE 3306
 
-# Install needed packages
-#
-# We're not including graphviz and ghostscript, needed for the 3 document
-# dependency graph tests; they would increase the size of the image by about
-# 15%, about 100MB.
-
-# Fetch apt package information, and upgrade to latest package versions
-
-RUN apt-get update
-RUN apt-get -qy upgrade
-
-# Install the packages we need
-RUN apt-get install -qy \
-	build-essential \
-	bzip2 \
-	ca-certificates \
-	colordiff \
-	gawk \
-	gcc \
-	ipython \
-	jq \
-	less \
-	libbz2-dev \
-	libdb5.3-dev \
-	libexpat1-dev \
-	libffi-dev \
-	libgdbm-dev \
-	libjpeg62-turbo-dev \
-	liblzma-dev \
-	libmagic1 \
-	libmariadbclient-dev \
-	libncurses5-dev \
-	libncursesw5-dev \
-	libreadline-dev \
-	libsqlite3-dev \
-	libssl-dev \
-	libsvn1 \
-	libxml2-dev \
-	libxslt-dev \
-	libz-dev \
-	libffi-dev \
-	locales \
-	make \
-	man \
-	mariadb-client \
-	mariadb-server \
-	openssh-client \
-	patch \
-	procps \
-	pv \
-	rsync \
+# Install all dependencies that are available as packages
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get -y update && \
+    apt-get -y install --no-install-recommends \
+        apache2-utils \
+        apt-file \
+        apt-utils \
+        chromium-driver \
+        curl \
+        enscript \
+        gcc \
+        ghostscript \
+        git \
+        graphviz \
+        libmagic-dev \
+        libmariadb-dev \
+        mariadb-server \
+        npm \
+        pigz \
+        pv \
+        python-is-python3 \
+        python3-dev \
+        python3-pip \
         rsyslog \
-	subversion \
-	sudo \
-	uuid-dev  \
-	vim \
-	wget \
-	xz-utils\
-	zile \
-	zlib1g-dev
+        unzip \
+        yang-tools && \
+    apt-get -y clean && \
+    rm -rf /var/lib/apt/lists/*
 
-# Postgresql packages
-RUN apt-get install -qy \
-        postgresql-11 \
-        postgresql-server-dev-11 
+# Install bower
+RUN npm install -g bower
 
-# Get the key used to sign the libyang repo
-RUN wget -nv http://download.opensuse.org/repositories/home:liberouter/Debian_9.0/Release.key
-RUN apt-key add - < Release.key
-RUN rm Release.key
-
-# Add apt source entry for libyang
-RUN echo "deb http://download.opensuse.org/repositories/home:/liberouter/Debian_9.0/ /" >> /etc/apt/sources.list.d/libyang.list
-
-# Update the package defs, and install the desired mysql from the mysql repo
-RUN apt-get update
-RUN apt-get install -qy libyang1
-
-# This is expected to exist by the mysql startup scripts:
-#RUN touch /etc/mysql/debian.cnf
-# ------------------------------------------------------------------------------
-
-# Get rid of installation files we don't need in the image, to reduce size
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Enable some common locales
-RUN sed -i -e 's/^. en_US/en_US/' -e 's/^. en_GB/en_GB/' -e 's/^. en_IE/en_IE/' /etc/locale.gen
-RUN locale-gen
-
-# Remove an rsyslog module that we don't need, which also requires extra permissions
-RUN sed -i -e '/load="imklog"/d' /etc/rsyslog.conf
-
-# Set up root password
-RUN echo "root:root" | chpasswd
-
-# MySQL
-VOLUME /var/lib/mysql
-
-# idnits and dependencies
+# Install idnits
 ADD https://tools.ietf.org/tools/idnits/idnits /usr/local/bin/
 RUN chmod +rx /usr/local/bin/idnits
 
-# Directory for Mac certs
-RUN mkdir /etc/certificates
-
-# # Python 3
-# # Comment in if OS does not provide python3.6, which is the current
-# # production version
-ENV PYVER=3.6.10
-ENV PYREV=3.6
+# Install current datatracker python dependencies
+COPY requirements.txt /
+RUN pip install -r /requirements.txt
 
-WORKDIR /usr/src
-RUN wget -q https://www.python.org/ftp/python/$PYVER/Python-$PYVER.tar.xz
-RUN tar xJf Python-$PYVER.tar.xz
-RUN rm Python-$PYVER.tar.xz
-WORKDIR /usr/src/Python-$PYVER/
-RUN ./configure
-RUN make
-RUN make altinstall
-WORKDIR /usr/src
-RUN rm -rf /usr/src/Python-$PYVER/
+# Turn off rsyslog kernel logging (doesn't work in Docker)
+RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf
 
-ENV HOSTNAME="datatracker"
+# Allow access to mariadb over the network
+RUN sed -i 's/127.0.0.1/0.0.0.0/' /etc/mysql/mariadb.conf.d/50-server.cnf
 
-ENV DDIR="/usr/local/share/datatracker"
-RUN mkdir -p $DDIR
-WORKDIR $DDIR
+# Turn on mariadb performance_schema
+RUN sed -i 's/\[mysqld\]/\[mysqld\]\nperformance_schema=ON/' /etc/mysql/mariadb.conf.d/50-server.cnf
 
-COPY requirements.txt ./
-COPY setprompt ./
+# Make the mariadb sys schema available for possible installation
+# We would normally use the next line, but that has a bug:
+# ADD https://github.com/FromDual/mariadb-sys/archive/master.zip /
+# This is the repo that has the PR:
+ADD https://github.com/grooverdan/mariadb-sys/archive/refs/heads/master.zip /
+RUN unzip /master.zip
 
+# Copy the startup file
 COPY docker-init.sh /docker-init.sh
 RUN chmod +x /docker-init.sh
-ENTRYPOINT ["/docker-init.sh"]
 
-CMD	/bin/bash
+WORKDIR /root/src
+ENTRYPOINT ["/docker-init.sh"]

docker/README.md

@@ -0,0 +1,37 @@
+# Datatracker Development in Docker
+
+1. [Set up Docker](https://docs.docker.com/get-started/) on your preferred
+   platform.
+
+2. If you have a copy of the datatracker code checked out already, simply `cd`
+   to the top-level directory.
+
+   If not, check out a datatracker branch as usual. We'll check out `trunk`
+   below, but you can use any branch:
+
+       svn co https://svn.ietf.org/svn/tools/ietfdb/trunk
+       cd trunk
+
+3. **TEMPORARY:** Replace the contents of the `docker` directory with Lars'
+   files.
+
+4. **TEMPORARY:** Until Lars' changes have been merged and a docker image is
+   available for download, you will need to build it locally:
+
+       docker/build -l
+
+    This will take a while, but only needs to be done once.
+
+5. Use the `docker/run` script to start the datatracker container. You will be
+   dropped into a shell from which you can start the datatracker and execute
+   related commands as usual, for example:
+
+       ietf/manage.py runserver 0.0.0.0:8000
+
+   If you do not already have a copy of the IETF database available in the
+   `data` directory, one will be downloaded and imported the first time you run
+   `docker/run`. This will take some time.
+
+   Once the datatracker has started, you should be able to open
+   [http://localhost:8000](http://localhost:8000) in a browser and see the
+   landing page.