Skip to content

Commit 27fccc6

Browse files
rjsparksjennifer-richards
rjsparks
and
jennifer-richards
authored
feat: move to postgresql (ietf-tools#4744)
* feat: move to postgresql * fix: repair fractional replace statement * fix: use pathlib to manipulate settings_local Co-authored-by: Jennifer Richards <jennifer@painless-security.com> * fix: do two string replacements, not one followed by another that throws away the first. Co-authored-by: Jennifer Richards <jennifer@painless-security.com> * fix: use pathlib again to manipulate settings_local Co-authored-by: Jennifer Richards <jennifer@painless-security.com> * fix: properly use assert (1/2) Co-authored-by: Jennifer Richards <jennifer@painless-security.com> * fix: properly use assert (2/2) Co-authored-by: Jennifer Richards <jennifer@painless-security.com> Co-authored-by: Jennifer Richards <jennifer@painless-security.com>
1 parent 131cdf9 commit 27fccc6

13 files changed

Lines changed: 210 additions & 2 deletions

‎docker-compose.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,16 @@ services:
5656
- '--innodb-write-io-threads=8'
5757
- '--innodb-flush-log-at-trx-commit=0'
5858
- '--performance-schema=1'
59+
60+
pgdb:
61+
build:
62+
context: .
63+
dockerfile: docker/pgdb.Dockerfile
64+
restart: unless-stopped
65+
environment:
66+
POSTGRES_PASSWORD: hk2j22sfiv
67+
volumes:
68+
- postgresdb-data:/var/lib/postgresql/data
5969

6070
# Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally.
6171
# (Adding the "ports" property to this file will not forward from a Codespace.)
@@ -101,5 +111,6 @@ services:
101111
# - .:/workspace
102112

103113
volumes:
114+
postgresdb-data:
104115
mariadb-data:
105116
app-assets:

‎docker/app.Dockerfile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,13 @@ ARG USER_UID=1000
99
ARG USER_GID=$USER_UID
1010
COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh
1111
RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh
12+
13+
# Add Postgresql Apt Repository to get 14
14+
RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
15+
RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
16+
1217
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
18+
&& apt-get install -y --no-install-recommends postgresql-client-14 pgloader \
1319
# Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
1420
&& apt-get purge -y imagemagick imagemagick-6-common \
1521
# Install common packages, non-root user

‎docker/configs/settings_mysqldb.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
DATABASES = {
2+
'default': {
3+
'HOST': 'db',
4+
'PORT': 3306,
5+
'NAME': 'ietf_utf8',
6+
'ENGINE': 'django.db.backends.mysql',
7+
'USER': 'django',
8+
'PASSWORD': 'RkTkDPFnKpko',
9+
'OPTIONS': {
10+
'sql_mode': 'STRICT_TRANS_TABLES',
11+
'init_command': 'SET storage_engine=InnoDB; SET names "utf8"',
12+
},
13+
},
14+
}
15+
16+
DATABASE_TEST_OPTIONS = {
17+
'init_command': 'SET storage_engine=InnoDB',
18+
}

‎docker/configs/settings_postgresqldb.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
DATABASES = {
2+
'default': {
3+
'HOST': 'pgdb',
4+
'PORT': 5432,
5+
'NAME': 'ietf',
6+
'ENGINE': 'django.db.backends.postgresql_psycopg2',
7+
'USER': 'django',
8+
'PASSWORD': 'RkTkDPFnKpko',
9+
},
10+
}

‎docker/pgdb.Dockerfile‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
FROM postgres:14.5
2+
LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
3+
4+
#RUN apt-get update \
5+
# && apt-get install -y --no-install-recommends \
6+
# postgresql-14-pg-catcheck \
7+
# postgresql-14-powa \
8+
# postgresql-14-pg-qualstats \
9+
# postgresql-14-pg-stat-kcache \
10+
# postgresql-14-pg-stat-monitor \
11+
# postgresql-14-pg-top \
12+
# postgresql-14-pg-track_settings \
13+
# postgresql-14-pg-wait_sampling \
14+
# pgsql_tweaks
15+
16+
ENV POSTGRES_PASSWORD=hk2j22sfiv
17+
ENV POSTGRES_HOST_AUTH_METHOD=trust
18+
19+
COPY docker/scripts/pgdb-ietf-init.sh /docker-entrypoint-initdb.d/

‎docker/scripts/app-init.sh‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,17 @@ yarn build
2828
yarn legacy:build
2929

3030
# Copy config files if needed
31+
if [ ! -f "$WORKSPACEDIR/ietf/settings_mysqldb.py" ]; then
32+
cp $WORKSPACEDIR/docker/configs/settings_mysqldb.py $WORKSPACEDIR/ietf/settings_mysqldb.py
33+
fi
34+
if [ ! -f "$WORKSPACEDIR/ietf/settings_postgresqldb.py" ]; then
35+
cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/settings_postgresqldb.py
36+
fi
3137

3238
if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then
3339
echo "Setting up a default settings_local.py ..."
3440
cp $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
41+
3542
else
3643
echo "Using existing ietf/settings_local.py file"
3744
if ! cmp -s $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py; then
@@ -73,6 +80,10 @@ else
7380
fi
7481
fi
7582

83+
# Recondition settings to make changing databases easier. (Remember that we may have developers starting with settings_local from earlier work)
84+
python docker/scripts/db-include-fix.py
85+
cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_postgresqldb import DATABASES/from ietf.settings_mysqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
86+
7687
# Create data directories
7788

7889
echo "Creating data directories..."
@@ -88,7 +99,8 @@ curl -fsSL https://github.com/ietf-tools/datatracker/releases/download/baseline/
8899

89100
if [ -n "$EDITOR_VSCODE" ]; then
90101
echo "Waiting for DB container to come online ..."
91-
/usr/local/bin/wait-for localhost:3306 -- echo "DB ready"
102+
/usr/local/bin/wait-for db:3306 -- echo "MariaDB ready"
103+
/usr/local/bin/wait-for pgdb:5432 -- echo "Postgresql ready"
92104
fi
93105

94106
# Run memcached
@@ -112,7 +124,11 @@ if ietf/manage.py showmigrations | grep "\[ \] 0003_pause_to_change_use_tz"; the
112124
# This is expected to exit non-zero at the pause
113125
/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true
114126
cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
115-
/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
127+
# This is also expected to exit non-zero at the 2nd pause
128+
echo "DEBUGGING pt 1 - this should say mysqldb"
129+
grep "DATA" $WORKSPACEDIR/ietf/settings_local.py
130+
/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true
131+
# More migrations after the move to postgres below.
116132

117133
else
118134
if grep "USE_TZ" $WORKSPACEDIR/ietf/settings_local.py; then
@@ -123,6 +139,25 @@ else
123139
fi
124140
fi
125141

142+
echo "DEBUGGING pt 2 - this should say mysqldb"
143+
grep "DATA" $WORKSPACEDIR/ietf/settings_local.py
144+
cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
145+
echo "DEBUGGING pt 3 - this should say postgresdb"
146+
grep "DATA" $WORKSPACEDIR/ietf/settings_local.py
147+
148+
# Now transfer the migrated database from mysql to postgres unless that's already happened.
149+
if psql -U django -h pgdb -d ietf -c "\dt" 2>&1 | grep -q "Did not find any relations."; then
150+
cat << EOF > cast.load
151+
LOAD DATABASE
152+
FROM mysql://django:RkTkDPFnKpko@db/ietf_utf8
153+
INTO postgresql://django:RkTkDPFnKpko@pgdb/ietf
154+
CAST type varchar to text drop typemod;
155+
EOF
156+
time pgloader --verbose --logfile=ietf_pgloader.run --summary=ietf_pgloader.summary cast.load
157+
rm cast.load
158+
/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
159+
fi
160+
126161
echo "-----------------------------------------------------------------"
127162
echo "Done!"
128163
echo "-----------------------------------------------------------------"

‎docker/scripts/db-include-fix.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
from pathlib import Path
2+
content = Path('/workspace/ietf/settings_local.py').read_text()
3+
newcontent = content.replace(
4+
"""DATABASES = {
5+
'default': {
6+
'HOST': 'db',
7+
'PORT': 3306,
8+
'NAME': 'ietf_utf8',
9+
'ENGINE': 'django.db.backends.mysql',
10+
'USER': 'django',
11+
'PASSWORD': 'RkTkDPFnKpko',
12+
'OPTIONS': {
13+
'sql_mode': 'STRICT_TRANS_TABLES',
14+
'init_command': 'SET storage_engine=InnoDB; SET names "utf8"',
15+
},
16+
},
17+
}""",
18+
"from ietf.settings_mysqldb import DATABASES",
19+
).replace(
20+
"""DATABASES = {
21+
'default': {
22+
'HOST': 'pgdb',
23+
'PORT': 5432,
24+
'NAME': 'ietf',
25+
'ENGINE': 'django.db.backends.postgresql_psycopg2',
26+
'USER': 'django',
27+
'PASSWORD': 'RkTkDPFnKpko',
28+
},
29+
}""",
30+
"from ietf.settings_postgresqldb import DATABASES",
31+
)
32+
with Path('/workspace/ietf/settings_local.py').open('w') as replacementfile:
33+
replacementfile.write(newcontent)

‎docker/scripts/pgdb-ietf-init.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
set -e
3+
4+
psql -v ON_ERROR_STOP=1 --username postgres --dbname postgres <<-EOSQL
5+
CREATE USER django PASSWORD 'RkTkDPFnKpko';
6+
CREATE DATABASE ietf;
7+
GRANT ALL PRIVILEGES ON DATABASE ietf TO django;
8+
ALTER USER django set search_path=ietf_utf8,django,public;
9+
EOSQL

‎ietf/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,6 @@
44
/settings_local_debug.py
55
/settings_local_sqlitetest.py
66
/settings_local_vite.py
7+
/settings_mysqldb.py
8+
/settings_postgresqldb.py
79
/ietfdb.sql.gz

‎ietf/community/migrations/0010_doc_ids_are_ints.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Generated by Django 2.2.28 on 2022-11-05 11:07
2+
3+
from django.db import migrations
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
('community', '0009_add_group_exp_rule_to_groups'),
10+
]
11+
12+
operations = [
13+
migrations.RunSQL(
14+
sql="alter table community_communitylist_added_docs modify document_id int unsigned not null;",
15+
reverse_sql="alter table community_communitylist_added_docs modify document_id varchar(255) not null;"
16+
),
17+
migrations.RunSQL(
18+
sql="alter table community_searchrule_name_contains_index modify document_id int unsigned not null;",
19+
reverse_sql="alter table community_searchrule_name_contains_index modify document_id varchar(255) not null;"
20+
),
21+
]

0 commit comments

Comments
 (0)