From 80d51e29bcdf13115cbaafc5bd380f61418f69bf Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 10 Nov 2022 12:12:47 +0000 Subject: [PATCH 1/7] feat: move to postgresql --- docker-compose.yml | 11 ++++++ docker/app.Dockerfile | 6 +++ docker/configs/settings_mysqldb.py | 18 +++++++++ docker/configs/settings_postgresqldb.py | 10 +++++ docker/pgdb.Dockerfile | 19 +++++++++ docker/scripts/app-init.sh | 39 ++++++++++++++++++- docker/scripts/db-include-fix.py | 28 +++++++++++++ docker/scripts/pgdb-ietf-init.sh | 9 +++++ ietf/.gitignore | 2 + .../migrations/0010_doc_ids_are_ints.py | 21 ++++++++++ .../0059_durationfield_are_intervals.py | 23 +++++++++++ .../0004_pause_to_change_database_engines.py | 20 ++++++++++ requirements.txt | 1 + 13 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 docker/configs/settings_mysqldb.py create mode 100644 docker/configs/settings_postgresqldb.py create mode 100644 docker/pgdb.Dockerfile create mode 100644 docker/scripts/db-include-fix.py create mode 100755 docker/scripts/pgdb-ietf-init.sh create mode 100644 ietf/community/migrations/0010_doc_ids_are_ints.py create mode 100644 ietf/meeting/migrations/0059_durationfield_are_intervals.py create mode 100644 ietf/utils/migrations/0004_pause_to_change_database_engines.py diff --git a/docker-compose.yml b/docker-compose.yml index db8ea9152a2..b25a64ef97e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,6 +56,16 @@ services: - '--innodb-write-io-threads=8' - '--innodb-flush-log-at-trx-commit=0' - '--performance-schema=1' + + pgdb: + build: + context: . + dockerfile: docker/pgdb.Dockerfile + restart: unless-stopped + environment: + POSTGRES_PASSWORD: hk2j22sfiv + volumes: + - postgresdb-data:/var/lib/postgresql/data # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) @@ -101,5 +111,6 @@ services: # - .:/workspace volumes: + postgresdb-data: mariadb-data: app-assets: diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index 6d539b1a561..af475a4e6be 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -9,7 +9,13 @@ ARG USER_UID=1000 ARG USER_GID=$USER_UID COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh + +# Add Postgresql Apt Repository to get 14 +RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user diff --git a/docker/configs/settings_mysqldb.py b/docker/configs/settings_mysqldb.py new file mode 100644 index 00000000000..445f01e70df --- /dev/null +++ b/docker/configs/settings_mysqldb.py @@ -0,0 +1,18 @@ +DATABASES = { + 'default': { + 'HOST': 'db', + 'PORT': 3306, + 'NAME': 'ietf_utf8', + 'ENGINE': 'django.db.backends.mysql', + 'USER': 'django', + 'PASSWORD': 'RkTkDPFnKpko', + 'OPTIONS': { + 'sql_mode': 'STRICT_TRANS_TABLES', + 'init_command': 'SET storage_engine=InnoDB; SET names "utf8"', + }, + }, +} + +DATABASE_TEST_OPTIONS = { + 'init_command': 'SET storage_engine=InnoDB', +} diff --git a/docker/configs/settings_postgresqldb.py b/docker/configs/settings_postgresqldb.py new file mode 100644 index 00000000000..e400420b5b5 --- /dev/null +++ b/docker/configs/settings_postgresqldb.py @@ -0,0 +1,10 @@ +DATABASES = { + 'default': { + 'HOST': 'pgdb', + 'PORT': 5432, + 'NAME': 'ietf', + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'USER': 'django', + 'PASSWORD': 'RkTkDPFnKpko', + }, +} diff --git a/docker/pgdb.Dockerfile b/docker/pgdb.Dockerfile new file mode 100644 index 00000000000..e566ab6459e --- /dev/null +++ b/docker/pgdb.Dockerfile @@ -0,0 +1,19 @@ +FROM postgres:14.5 +LABEL maintainer="IETF Tools Team " + +#RUN apt-get update \ +# && apt-get install -y --no-install-recommends \ +# postgresql-14-pg-catcheck \ +# postgresql-14-powa \ +# postgresql-14-pg-qualstats \ +# postgresql-14-pg-stat-kcache \ +# postgresql-14-pg-stat-monitor \ +# postgresql-14-pg-top \ +# postgresql-14-pg-track_settings \ +# postgresql-14-pg-wait_sampling \ +# pgsql_tweaks + +ENV POSTGRES_PASSWORD=hk2j22sfiv +ENV POSTGRES_HOST_AUTH_METHOD=trust + +COPY docker/scripts/pgdb-ietf-init.sh /docker-entrypoint-initdb.d/ diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index 7557e8f45fc..94a1232361b 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -28,10 +28,17 @@ yarn build yarn legacy:build # Copy config files if needed +if [ ! -f "$WORKSPACEDIR/ietf/settings_mysqldb.py" ]; then + cp $WORKSPACEDIR/docker/configs/settings_mysqldb.py $WORKSPACEDIR/ietf/settings_mysqldb.py +fi +if [ ! -f "$WORKSPACEDIR/ietf/settings_postgresqldb.py" ]; then + cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/settings_postgresqldb.py +fi if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then echo "Setting up a default settings_local.py ..." cp $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py + else echo "Using existing ietf/settings_local.py file" if ! cmp -s $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py; then @@ -73,6 +80,10 @@ else fi fi +# Recondition settings to make changing databases easier. (Remember that we may have developers starting with settings_local from earlier work) + python docker/scripts/db-include-fix.py + cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_postgresqldb import DATABASES/from ietf.settings_mysqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py + # Create data directories echo "Creating data directories..." @@ -88,7 +99,8 @@ curl -fsSL https://github.com/ietf-tools/datatracker/releases/download/baseline/ if [ -n "$EDITOR_VSCODE" ]; then echo "Waiting for DB container to come online ..." - /usr/local/bin/wait-for localhost:3306 -- echo "DB ready" + /usr/local/bin/wait-for db:3306 -- echo "MariaDB ready" + /usr/local/bin/wait-for pgdb:5432 -- echo "Postgresql ready" fi # Run memcached @@ -112,7 +124,11 @@ if ietf/manage.py showmigrations | grep "\[ \] 0003_pause_to_change_use_tz"; the # This is expected to exit non-zero at the pause /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py - /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local + # This is also expected to exit non-zero at the 2nd pause + echo "DEBUGGING pt 1 - this should say mysqldb" + grep "DATA" $WORKSPACEDIR/ietf/settings_local.py + /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true + # More migrations after the move to postgres below. else if grep "USE_TZ" $WORKSPACEDIR/ietf/settings_local.py; then @@ -123,6 +139,25 @@ else fi fi +echo "DEBUGGING pt 2 - this should say mysqldb" +grep "DATA" $WORKSPACEDIR/ietf/settings_local.py +cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py +echo "DEBUGGING pt 3 - this should say postgresdb" +grep "DATA" $WORKSPACEDIR/ietf/settings_local.py + +# Now transfer the migrated database from mysql to postgres unless that's already happened. +if psql -U django -h pgdb -d ietf -c "\dt" 2>&1 | grep -q "Did not find any relations."; then + cat << EOF > cast.load +LOAD DATABASE +FROM mysql://django:RkTkDPFnKpko@db/ietf_utf8 +INTO postgresql://django:RkTkDPFnKpko@pgdb/ietf +CAST type varchar to text drop typemod; +EOF + time pgloader --verbose --logfile=ietf_pgloader.run --summary=ietf_pgloader.summary cast.load + rm cast.load + /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local +fi + echo "-----------------------------------------------------------------" echo "Done!" echo "-----------------------------------------------------------------" diff --git a/docker/scripts/db-include-fix.py b/docker/scripts/db-include-fix.py new file mode 100644 index 00000000000..ffad90e8721 --- /dev/null +++ b/docker/scripts/db-include-fix.py @@ -0,0 +1,28 @@ +with open('/workspace/ietf/settings_local.py','r') as contentfile: + content = contentfile.readlines() +newcontent = content.replace("""DATABASES = { + 'default': { + 'HOST': 'db', + 'PORT': 3306, + 'NAME': 'ietf_utf8', + 'ENGINE': 'django.db.backends.mysql', + 'USER': 'django', + 'PASSWORD': 'RkTkDPFnKpko', + 'OPTIONS': { + 'sql_mode': 'STRICT_TRANS_TABLES', + 'init_command': 'SET storage_engine=InnoDB; SET names "utf8"', + }, + }, +} +newcontent = content.replace("""DATABASES = { + 'default': { + 'HOST': 'pgdb', + 'PORT': 5432, + 'NAME': 'ietf', + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'USER': 'django', + 'PASSWORD': 'RkTkDPFnKpko', + }, +}""", "from ietf.settings_postgresqldb import DATABASES") +with open('/workspace/ietf/settings_local.py','w') as replacementfile: + replacementfile.write(newcontent) diff --git a/docker/scripts/pgdb-ietf-init.sh b/docker/scripts/pgdb-ietf-init.sh new file mode 100755 index 00000000000..53308405653 --- /dev/null +++ b/docker/scripts/pgdb-ietf-init.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +psql -v ON_ERROR_STOP=1 --username postgres --dbname postgres <<-EOSQL + CREATE USER django PASSWORD 'RkTkDPFnKpko'; + CREATE DATABASE ietf; + GRANT ALL PRIVILEGES ON DATABASE ietf TO django; + ALTER USER django set search_path=ietf_utf8,django,public; +EOSQL diff --git a/ietf/.gitignore b/ietf/.gitignore index 322cc214d08..9df45d76ab7 100644 --- a/ietf/.gitignore +++ b/ietf/.gitignore @@ -4,4 +4,6 @@ /settings_local_debug.py /settings_local_sqlitetest.py /settings_local_vite.py +/settings_mysqldb.py +/settings_postgresqldb.py /ietfdb.sql.gz diff --git a/ietf/community/migrations/0010_doc_ids_are_ints.py b/ietf/community/migrations/0010_doc_ids_are_ints.py new file mode 100644 index 00000000000..a76eda92a7b --- /dev/null +++ b/ietf/community/migrations/0010_doc_ids_are_ints.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.28 on 2022-11-05 11:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('community', '0009_add_group_exp_rule_to_groups'), + ] + + operations = [ + migrations.RunSQL( + sql="alter table community_communitylist_added_docs modify document_id int unsigned not null;", + reverse_sql="alter table community_communitylist_added_docs modify document_id varchar(255) not null;" + ), + migrations.RunSQL( + sql="alter table community_searchrule_name_contains_index modify document_id int unsigned not null;", + reverse_sql="alter table community_searchrule_name_contains_index modify document_id varchar(255) not null;" + ), + ] diff --git a/ietf/meeting/migrations/0059_durationfield_are_intervals.py b/ietf/meeting/migrations/0059_durationfield_are_intervals.py new file mode 100644 index 00000000000..b30b5acea44 --- /dev/null +++ b/ietf/meeting/migrations/0059_durationfield_are_intervals.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.28 on 2022-11-10 02:35 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('meeting', '0058_meeting_time_zone_not_blank'), + ('utils', '0004_pause_to_change_database_engines'), + ] + + operations = [ + # reversesql is intentionally not provided. There is no looking back... + migrations.RunSQL(sql="update meeting_meeting set idsubmit_cutoff_time_utc=idsubmit_cutoff_time_utc/1000"), + migrations.RunSQL(sql="alter table meeting_meeting alter column idsubmit_cutoff_time_utc type interval using (idsubmit_cutoff_time_utc::text||' milliseconds')::interval;"), + migrations.RunSQL(sql="update meeting_meeting set idsubmit_cutoff_warning_days=idsubmit_cutoff_warning_days/1000"), + migrations.RunSQL(sql="alter table meeting_meeting alter column idsubmit_cutoff_warning_days type interval using (idsubmit_cutoff_warning_days::text||' milliseconds')::interval;"), + migrations.RunSQL(sql="update meeting_timeslot set duration=duration/1000"), + migrations.RunSQL(sql="alter table meeting_timeslot alter column duration type interval using (duration::text||' milliseconds')::interval;"), + migrations.RunSQL(sql="update meeting_session set requested_duration=requested_duration/1000"), + migrations.RunSQL(sql="alter table meeting_session alter column requested_duration type interval using (requested_duration::text||' milliseconds')::interval;"), + ] diff --git a/ietf/utils/migrations/0004_pause_to_change_database_engines.py b/ietf/utils/migrations/0004_pause_to_change_database_engines.py new file mode 100644 index 00000000000..5833249be0b --- /dev/null +++ b/ietf/utils/migrations/0004_pause_to_change_database_engines.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.28 on 2022-11-10 09:27 + +from django.conf import settings +from django.db import migrations + +def forward(apps, schema_editor): + assert(settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2', 'Please change to use postgres before continuing') + +def reverse(apps, schema_editor): + assert(settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql', 'Please change to use mariadb before continuing') + +class Migration(migrations.Migration): + + dependencies = [ + ('utils', '0003_pause_to_change_use_tz'), + ] + + operations = [ + migrations.RunPython(forward, reverse), + ] diff --git a/requirements.txt b/requirements.txt index 155e5c67669..dd5349a7ab4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,6 +45,7 @@ mypy>=0.782,<0.790 # Version requirements determined by django-stubs. mysqlclient>=2.1.0 oic>=1.3 # Used only by tests Pillow>=9.1.0 +psycopg2<2.9 pyang>=2.5.3 pyflakes>=2.4.0 pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency From 4001a63a7454514b03c1e6ad1973a79c2f58ef8f Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 10 Nov 2022 16:49:15 +0000 Subject: [PATCH 2/7] fix: repair fractional replace statement --- docker/scripts/db-include-fix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/scripts/db-include-fix.py b/docker/scripts/db-include-fix.py index ffad90e8721..9f5902cb46c 100644 --- a/docker/scripts/db-include-fix.py +++ b/docker/scripts/db-include-fix.py @@ -13,7 +13,7 @@ 'init_command': 'SET storage_engine=InnoDB; SET names "utf8"', }, }, -} +}""", "from ietf.settings_mysqldb import DATABASES") newcontent = content.replace("""DATABASES = { 'default': { 'HOST': 'pgdb', From c3cd0e86d65b989d256b28f97cb7938db9161aae Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 11 Nov 2022 10:04:26 +0000 Subject: [PATCH 3/7] fix: use pathlib to manipulate settings_local Co-authored-by: Jennifer Richards --- docker/scripts/db-include-fix.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/scripts/db-include-fix.py b/docker/scripts/db-include-fix.py index 9f5902cb46c..7c3e4d5b187 100644 --- a/docker/scripts/db-include-fix.py +++ b/docker/scripts/db-include-fix.py @@ -1,6 +1,7 @@ -with open('/workspace/ietf/settings_local.py','r') as contentfile: - content = contentfile.readlines() -newcontent = content.replace("""DATABASES = { +from pathlib import Path +content = Path('/workspace/ietf/settings_local.py').read_text() +newcontent = content.replace( + """DATABASES = { 'default': { 'HOST': 'db', 'PORT': 3306, From 4f3464588a84e9e78e36ff49e3f1f12da16ae691 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 11 Nov 2022 10:06:06 +0000 Subject: [PATCH 4/7] fix: do two string replacements, not one followed by another that throws away the first. Co-authored-by: Jennifer Richards --- docker/scripts/db-include-fix.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/scripts/db-include-fix.py b/docker/scripts/db-include-fix.py index 7c3e4d5b187..cf1b7072ed8 100644 --- a/docker/scripts/db-include-fix.py +++ b/docker/scripts/db-include-fix.py @@ -14,8 +14,10 @@ 'init_command': 'SET storage_engine=InnoDB; SET names "utf8"', }, }, -}""", "from ietf.settings_mysqldb import DATABASES") -newcontent = content.replace("""DATABASES = { +}""", + "from ietf.settings_mysqldb import DATABASES", +).replace( + """DATABASES = { 'default': { 'HOST': 'pgdb', 'PORT': 5432, From c285dba97bdf90351e074bb0e800ac5c0f83fa50 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 11 Nov 2022 10:07:02 +0000 Subject: [PATCH 5/7] fix: use pathlib again to manipulate settings_local Co-authored-by: Jennifer Richards --- docker/scripts/db-include-fix.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/scripts/db-include-fix.py b/docker/scripts/db-include-fix.py index cf1b7072ed8..369867061cd 100644 --- a/docker/scripts/db-include-fix.py +++ b/docker/scripts/db-include-fix.py @@ -26,6 +26,8 @@ 'USER': 'django', 'PASSWORD': 'RkTkDPFnKpko', }, -}""", "from ietf.settings_postgresqldb import DATABASES") -with open('/workspace/ietf/settings_local.py','w') as replacementfile: +}""", + "from ietf.settings_postgresqldb import DATABASES", +) +with Path('/workspace/ietf/settings_local.py').open('w') as replacementfile: replacementfile.write(newcontent) From d8d363ee27b4f069186d30aa07363229e35f4b3b Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 11 Nov 2022 10:08:28 +0000 Subject: [PATCH 6/7] fix: properly use assert (1/2) Co-authored-by: Jennifer Richards --- ietf/utils/migrations/0004_pause_to_change_database_engines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/utils/migrations/0004_pause_to_change_database_engines.py b/ietf/utils/migrations/0004_pause_to_change_database_engines.py index 5833249be0b..853c2f8c756 100644 --- a/ietf/utils/migrations/0004_pause_to_change_database_engines.py +++ b/ietf/utils/migrations/0004_pause_to_change_database_engines.py @@ -4,7 +4,7 @@ from django.db import migrations def forward(apps, schema_editor): - assert(settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2', 'Please change to use postgres before continuing') + assert settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2', 'Please change to use postgres before continuing' def reverse(apps, schema_editor): assert(settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql', 'Please change to use mariadb before continuing') From d135cbea18569183116a93065014f4926cc0ac7a Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 11 Nov 2022 10:08:46 +0000 Subject: [PATCH 7/7] fix: properly use assert (2/2) Co-authored-by: Jennifer Richards --- ietf/utils/migrations/0004_pause_to_change_database_engines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/utils/migrations/0004_pause_to_change_database_engines.py b/ietf/utils/migrations/0004_pause_to_change_database_engines.py index 853c2f8c756..58a502b2542 100644 --- a/ietf/utils/migrations/0004_pause_to_change_database_engines.py +++ b/ietf/utils/migrations/0004_pause_to_change_database_engines.py @@ -7,7 +7,7 @@ def forward(apps, schema_editor): assert settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2', 'Please change to use postgres before continuing' def reverse(apps, schema_editor): - assert(settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql', 'Please change to use mariadb before continuing') + assert settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql', 'Please change to use mariadb before continuing' class Migration(migrations.Migration):