Skip to content

Commit b6186f1

Browse files
NGPixelNGPixel
authored
chore(dev): fix uid/gid mapping for non-vscode devs (ietf-tools#3981)
* chore: fix uid/gid mapping for non-vscode devs * chore: fix volume ownership in devcontainer * chore: use zsh in init script + remove temp mounted folders * chore: ignore group mapping error during devcontainer build * chore: fix ownership when running as root on host * chore: use new docker compose command format + fix root check * chore: improve uid/gid mapping message output * chore: include chromedriver permission fix * chore: skip parent dev image and install node directly instead of nvm * chore: reapply gitignore changes + better dev help message
1 parent fa17ddd commit b6186f1

7 files changed

Lines changed: 116 additions & 39 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,8 @@ datatracker.sublime-workspace
1919
/media
2020
/node_modules
2121
/release-coverage.json
22-
/tmp
23-
/tmp-nomcom-public-keys-dir
24-
/tmp-storage-meetinghosttests-dir
22+
/tmp-*
23+
/.testresult
2524
*.pyc
2625
__pycache__
2726
.yarn/*

‎.vscode/extensions.json‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
11
{
2-
"recommendations": [
3-
"arcanis.vscode-zipfs",
4-
"dbaeumer.vscode-eslint"
5-
]
2+
"recommendations": []
63
}

‎docker/app.Dockerfile‎

Lines changed: 49 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,55 @@
1-
# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.202.5/containers/python-3/.devcontainer/base.Dockerfile
1+
# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/python-3/.devcontainer/base.Dockerfile
22

33
# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
4-
ARG VARIANT="3.10-bullseye"
5-
FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
4+
ARG VARIANT=3-bullseye
5+
FROM python:${VARIANT}
66
LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
77

8-
# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
9-
ARG NODE_VERSION="none"
10-
RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
11-
RUN npm install -g yarn
8+
ENV DEBIAN_FRONTEND=noninteractive
129

10+
# Copy library scripts to execute
11+
ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/common-debian.sh /tmp/library-scripts/
12+
ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/python-debian.sh /tmp/library-scripts/
13+
ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/meta.env /tmp/library-scripts/
14+
15+
# [Option] Install zsh
16+
ARG INSTALL_ZSH="true"
17+
# [Option] Upgrade OS packages to their latest versions
18+
ARG UPGRADE_PACKAGES="true"
19+
# Install needed packages and setup non-root user. Use a separate RUN statement to add your own dependencies.
20+
ARG USERNAME=vscode
21+
ARG USER_UID=1000
22+
ARG USER_GID=$USER_UID
23+
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
24+
# Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
25+
&& apt-get purge -y imagemagick imagemagick-6-common \
26+
# Install common packages, non-root user
27+
&& bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true"
28+
29+
# Setup default python tools in a venv via pipx to avoid conflicts
30+
ENV PIPX_HOME=/usr/local/py-utils \
31+
PIPX_BIN_DIR=/usr/local/py-utils/bin
32+
ENV PATH=${PATH}:${PIPX_BIN_DIR}
33+
RUN bash /tmp/library-scripts/python-debian.sh "none" "/usr/local" "${PIPX_HOME}" "${USERNAME}"
34+
35+
# [Choice] Node.js version: lts, 18, 16, 14, 12, 10
36+
ARG NODE_VERSION="16"
37+
RUN curl -fsSL "https://deb.nodesource.com/setup_${NODE_VERSION}.x" | bash -
38+
RUN apt-get install -y nodejs make gcc g++ && npm install -g yarn
39+
40+
# Remove library scripts for final image
41+
RUN rm -rf /tmp/library-scripts
42+
43+
# Expose port 8000
1344
EXPOSE 8000
1445

15-
ENV DEBIAN_FRONTEND=noninteractive
16-
1746
# Add Docker Source
1847
RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
1948
RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
2049
$(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
2150

22-
RUN apt-get update --fix-missing
23-
24-
# apt-get upgrade is normally not a good idea, but this is a dev container
25-
RUN apt-get -qy upgrade
26-
2751
# Install the packages we need
28-
RUN apt-get install -qy \
52+
RUN apt-get update --fix-missing && apt-get install -qy \
2953
apache2-utils \
3054
apt-file \
3155
apt-utils \
@@ -87,8 +111,11 @@ RUN sed -i 's/\r$//' /tmp/app-install-chromedriver.sh && \
87111
chmod +x /tmp/app-install-chromedriver.sh
88112
RUN /tmp/app-install-chromedriver.sh
89113

114+
# Fix /dev/shm permissions for chromedriver
115+
RUN chmod 1777 /dev/shm
116+
90117
# Get rid of installation files we don't need in the image, to reduce size
91-
RUN apt-get clean && rm -rf /var/lib/apt/lists/*
118+
RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
92119

93120
# "fake" dbus address to prevent errors
94121
# https://github.com/SeleniumHQ/docker-selenium/issues/87
@@ -132,6 +159,12 @@ RUN sed -i 's/\r$//' /docker-init.sh && \
132159
RUN mkdir -p /workspace
133160
WORKDIR /workspace
134161

162+
# Fix user UID / GID to match host
163+
RUN groupmod --gid $USER_GID $USERNAME \
164+
&& usermod --uid $USER_UID --gid $USER_GID $USERNAME \
165+
&& chown -R $USER_UID:$USER_GID /home/$USERNAME \
166+
|| exit 0
167+
135168
USER vscode:vscode
136169

137170
# Install current datatracker python dependencies

‎docker/cleanall‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
cd ..
44
echo "Shutting down any instance still running and purge images..."
5-
docker-compose down -v --rmi all
5+
docker compose down -v --rmi all
66
echo "Purging dangling images..."
77
docker image prune
88
cd docker

‎docker/cleandb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33
echo "Shutting down any instance still running..."
44
cd ..
5-
docker-compose down -v
5+
docker compose down -v
66
echo "Rebuilding the DB image..."
7-
docker-compose pull db
8-
docker-compose build --no-cache db
7+
docker compose pull db
8+
docker compose build --no-cache db
99
cd docker
10-
echo "Done!"
10+
echo "Done!"

‎docker/run‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,20 @@
33
# Usage info
44
show_help() {
55
cat << EOF
6-
Usage: ${0##*/} [-h] [-p PORT]
6+
Usage: ${0##*/} [-h] [-p PORT] [-r]
77
Run datatracker in dev containers using docker-compose.
88
99
-h display this help and exit
1010
-p PORT use custom HTTP port for datatracker
11+
-r force rebuild the app container
1112
1213
EOF
1314
}
1415

1516
CUSTOM_PORT=8000
17+
FORCE_REBUILD=0
1618

17-
while getopts hp: opt; do
19+
while getopts "hp:r" opt; do
1820
case $opt in
1921
h)
2022
show_help
@@ -24,19 +26,57 @@ while getopts hp: opt; do
2426
CUSTOM_PORT=$OPTARG
2527
echo "Using custom port $CUSTOM_PORT..."
2628
;;
29+
r)
30+
FORCE_REBUILD=1
31+
echo "Will force rebuild the app container..."
32+
;;
2733
*)
2834
CUSTOM_PORT=8000
2935
echo "Using port 8000..."
3036
;;
3137
esac
3238
done
3339

40+
# Remove mounted temp directories
41+
rm -rf .parcel-cache __pycache__
42+
43+
# Create extended docker-compose definition
3444
cp docker-compose.extend.yml docker-compose.extend-custom.yml
3545
sed -i -r -e "s/CUSTOM_PORT/$CUSTOM_PORT/" docker-compose.extend-custom.yml
3646
cd ..
37-
docker-compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml up -d
38-
docker-compose port db 3306
39-
docker-compose exec app /bin/sh /docker-init.sh
40-
docker-compose stop
47+
48+
# Set UID/GID mappings
49+
NEW_UID=$(id -u)
50+
NEW_GID=$(id -g)
51+
if [ $NEW_UID -gt 0 ]; then
52+
echo "Will use the following user/group mapping:"
53+
echo "USER ID: $NEW_UID"
54+
echo "GROUP ID: $NEW_GID"
55+
else
56+
echo "Running as root, will use default user/group mapping..."
57+
NEW_UID=1000
58+
NEW_GID=1000
59+
fi
60+
61+
# Build / Rebuild Containers
62+
if [ $FORCE_REBUILD == 1 ]; then
63+
docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml down
64+
docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml rm -f
65+
docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml build --no-cache --pull --build-arg USER_UID=$NEW_UID --build-arg USER_GID=$NEW_GID
66+
docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml up -d --force-recreate
67+
else
68+
docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml build --build-arg USER_UID=$NEW_UID --build-arg USER_GID=$NEW_GID
69+
docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml up -d
70+
fi
71+
72+
# Output database port
73+
echo "Database exposed on port:"
74+
docker compose port db 3306
75+
76+
# Start init script
77+
docker compose exec app /bin/zsh /docker-init.sh
78+
79+
# Exit scripts
80+
docker compose stop
4181
cd docker
4282
rm -f docker-compose.extend-custom.yml

‎docker/scripts/app-init.sh‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ WORKSPACEDIR="/workspace"
44

55
sudo service rsyslog start
66

7-
# fix permissions for npm-related paths
8-
WORKSPACE_UID_GID=$(stat --format="%u:%g" "$WORKSPACEDIR")
9-
sudo chown -R "$WORKSPACE_UID_GID" "$WORKSPACEDIR/.parcel-cache"
7+
# Fix ownership of volumes
8+
echo "Fixing volumes ownership..."
9+
sudo chown -R vscode:vscode "$WORKSPACEDIR/.parcel-cache"
10+
sudo chown -R vscode:vscode "$WORKSPACEDIR/__pycache__"
11+
sudo chown -R vscode:vscode "$WORKSPACEDIR"
1012

1113
# Build node packages that requrie native compilation
1214
echo "Compiling native node packages..."
@@ -80,7 +82,9 @@ echo "Running initial checks..."
8082
/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local
8183
# /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
8284

85+
echo "-----------------------------------------------------------------"
8386
echo "Done!"
87+
echo "-----------------------------------------------------------------"
8488

8589
if [ -z "$EDITOR_VSCODE" ]; then
8690
CODE=0
@@ -93,11 +97,15 @@ if [ -z "$EDITOR_VSCODE" ]; then
9397
echo
9498
echo "to start a development instance of the Datatracker."
9599
echo
96-
bash
100+
echo " ietf/manage.py test --settings=settings_sqlitetest"
101+
echo
102+
echo "to run all the tests."
103+
echo
104+
zsh
97105
else
98106
echo "Executing \"$*\" and stopping container."
99107
echo
100-
bash -c "$*"
108+
zsh -c "$*"
101109
CODE=$?
102110
fi
103111
sudo service rsyslog stop

0 commit comments

Comments
 (0)