From ce08a57d35c4464214c66aad1f1e20d9f217b4db Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 25 Oct 2023 07:58:16 -0700 Subject: [PATCH 01/17] Switch to using geckodriver --- docker/app.Dockerfile | 2 +- docker/base.Dockerfile | 303 ++++++++++++++++++++----------------- docker/scripts/app-init.sh | 3 - ietf/settings.py | 2 - ietf/utils/jstest.py | 23 ++- 5 files changed, 176 insertions(+), 157 deletions(-) diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index c8e0fe7ad35..b7dd44b6f1b 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -11,7 +11,7 @@ COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh # Add Postgresql Apt Repository to get 14 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 499040e4335..f7aff849fbc 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -1,138 +1,165 @@ -FROM python:3.9-bullseye -LABEL maintainer="IETF Tools Team " - -ENV DEBIAN_FRONTEND=noninteractive - -# Update system packages -RUN apt-get update \ - && apt-get -qy upgrade \ - && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 - -# Add Node.js Source -RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - - -# Add Docker Source -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg -RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ - $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null - -# Add PostgreSQL Source -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - -# Install the packages we need -RUN apt-get update --fix-missing && apt-get install -qy \ - apache2-utils \ - apt-file \ - bash \ - build-essential \ - curl \ - default-jdk \ - docker-ce-cli \ - enscript \ - gawk \ - g++ \ - gcc \ - ghostscript \ - git \ - gnupg \ - jq \ - less \ - libcairo2-dev \ - libgtk2.0-0 \ - libgtk-3-0 \ - libnotify-dev \ - libgconf-2-4 \ - libgbm-dev \ - libnss3 \ - libxss1 \ - libasound2 \ - libxtst6 \ - libmagic-dev \ - libmariadb-dev \ - libmemcached-tools \ - locales \ - make \ - mariadb-client \ - memcached \ - nano \ - netcat \ - nodejs \ - pgloader \ - pigz \ - postgresql-client-14 \ - pv \ - python3-ipython \ - ripgrep \ - rsync \ - rsyslog \ - ruby \ - ruby-rubygems \ - unzip \ - wget \ - xauth \ - xvfb \ - yang-tools \ - zsh - -# Install kramdown-rfc2629 (ruby) -RUN gem install kramdown-rfc2629 - -# Install chromedriver -COPY docker/scripts/app-install-chromedriver.sh /tmp/app-install-chromedriver.sh -RUN sed -i 's/\r$//' /tmp/app-install-chromedriver.sh && \ - chmod +x /tmp/app-install-chromedriver.sh -RUN /tmp/app-install-chromedriver.sh - -# Fix /dev/shm permissions for chromedriver -RUN chmod 1777 /dev/shm - -# Activate Yarn -RUN corepack enable - -# Get rid of installation files we don't need in the image, to reduce size -RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - -# "fake" dbus address to prevent errors -# https://github.com/SeleniumHQ/docker-selenium/issues/87 -ENV DBUS_SESSION_BUS_ADDRESS=/dev/null - -# avoid million NPM install messages -ENV npm_config_loglevel warn -# allow installing when the main user is root -ENV npm_config_unsafe_perm true -# disable NPM funding messages -ENV npm_config_fund false - -# Set locale to en_US.UTF-8 -RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \ - echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ - echo "LANG=en_US.UTF-8" > /etc/locale.conf && \ - dpkg-reconfigure locales && \ - locale-gen en_US.UTF-8 && \ - update-locale LC_ALL en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 - -# Install idnits -ADD https://raw.githubusercontent.com/ietf-tools/idnits-mirror/main/idnits /usr/local/bin/ -RUN chmod +rx /usr/local/bin/idnits - -# Turn off rsyslog kernel logging (doesn't work in Docker) -RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf - -# Colorize the bash shell -RUN sed -i 's/#force_color_prompt=/force_color_prompt=/' /root/.bashrc - -# Turn off rsyslog kernel logging (doesn't work in Docker) -RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf - -# Fetch wait-for utility -ADD https://raw.githubusercontent.com/eficode/wait-for/v2.1.3/wait-for /usr/local/bin/ -RUN chmod +rx /usr/local/bin/wait-for - -# Create assets directory -RUN mkdir -p /assets - -# Create workspace -RUN mkdir -p /workspace -WORKDIR /workspace +FROM python:3.9-bullseye +LABEL maintainer="IETF Tools Team " + +ENV DEBIAN_FRONTEND=noninteractive + +# Update system packages +RUN apt-get update \ + && apt-get -qy upgrade \ + && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 + +# Add Node.js Source +RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \ + && mkdir -p /etc/apt/keyrings\ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg +ENV NODE_MAJOR=16 +RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + +# Add Docker Source +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null + +# Add PostgreSQL Source +RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + +RUN cat /etc/apt/sources.list.d/docker.list + +# Install the packages we need +RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends \ + apache2-utils \ + apt-file \ + bash \ + build-essential \ + curl \ + default-jdk \ + docker-ce-cli \ + enscript \ + firefox-esr \ + gawk \ + g++ \ + gcc \ + ghostscript \ + git \ + gnupg \ + jq \ + less \ + libcairo2-dev \ + libgtk2.0-0 \ + libgtk-3-0 \ + libnotify-dev \ + libgconf-2-4 \ + libgbm-dev \ + libnss3 \ + libxss1 \ + libasound2 \ + libxtst6 \ + libmagic-dev \ + libmariadb-dev \ + libmemcached-tools \ + locales \ + make \ + mariadb-client \ + memcached \ + nano \ + netcat \ + nodejs \ + pgloader \ + pigz \ + postgresql-client-14 \ + pv \ + python3-ipython \ + ripgrep \ + rsync \ + rsyslog \ + ruby \ + ruby-rubygems \ + unzip \ + wget \ + xauth \ + xvfb \ + yang-tools \ + zsh + +# Install kramdown-rfc2629 (ruby) +RUN gem install kramdown-rfc2629 + +# Firefox +# ARG FIREFOX_VERSION=latest +# RUN FIREFOX_DOWNLOAD_URL=$(if [ $FIREFOX_VERSION = "latest" ] || [ $FIREFOX_VERSION = "beta-latest" ] || [ $FIREFOX_VERSION = "nightly-latest" ] || [ $FIREFOX_VERSION = "devedition-latest" ] || [ $FIREFOX_VERSION = "esr-latest" ]; then echo "https://download.mozilla.org/?product=firefox-$FIREFOX_VERSION-ssl&os=linux64&lang=en-US"; else echo "https://download-installer.cdn.mozilla.net/pub/firefox/releases/$FIREFOX_VERSION/linux-x86_64/en-US/firefox-$FIREFOX_VERSION.tar.bz2"; fi) \ +# && apt-get update -qqy \ +# && apt-get -qqy --no-install-recommends install firefox-esr libavcodec-extra \ +# && rm -rf /var/lib/apt/lists/* /var/cache/apt/* \ +# && wget --no-verbose -O /tmp/firefox.tar.bz2 $FIREFOX_DOWNLOAD_URL \ +# && apt-get -y purge firefox-esr \ +# && rm -rf /opt/firefox \ +# && tar -C /opt -xjf /tmp/firefox.tar.bz2 \ +# && rm /tmp/firefox.tar.bz2 \ +# && mv /opt/firefox /opt/firefox-$FIREFOX_VERSION \ +# && ln -fs /opt/firefox-$FIREFOX_VERSION/firefox /usr/bin/firefox + +# GeckoDriver +ARG GECKODRIVER_VERSION=latest +RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0.33.0"; else echo $GECKODRIVER_VERSION; fi) \ + && echo "Using GeckoDriver version: "$GK_VERSION \ + && wget --no-verbose -O /tmp/geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v$GK_VERSION/geckodriver-v$GK_VERSION-linux64.tar.gz \ + && rm -rf /opt/geckodriver \ + && tar -C /opt -zxf /tmp/geckodriver.tar.gz \ + && rm /tmp/geckodriver.tar.gz \ + && mv /opt/geckodriver /opt/geckodriver-$GK_VERSION \ + && chmod 755 /opt/geckodriver-$GK_VERSION \ + && ln -fs /opt/geckodriver-$GK_VERSION /usr/bin/geckodriver + +# # Fix /dev/shm permissions for chromedriver +# RUN chmod 1777 /dev/shm + +# Activate Yarn +RUN corepack enable + +# Get rid of installation files we don't need in the image, to reduce size +RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /var/cache/apt/* + +# "fake" dbus address to prevent errors +# https://github.com/SeleniumHQ/docker-selenium/issues/87 +ENV DBUS_SESSION_BUS_ADDRESS=/dev/null + +# avoid million NPM install messages +ENV npm_config_loglevel warn +# allow installing when the main user is root +ENV npm_config_unsafe_perm true +# disable NPM funding messages +ENV npm_config_fund false + +# Set locale to en_US.UTF-8 +RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \ + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ + echo "LANG=en_US.UTF-8" > /etc/locale.conf && \ + dpkg-reconfigure locales && \ + locale-gen en_US.UTF-8 && \ + update-locale LC_ALL en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +# Install idnits +ADD https://raw.githubusercontent.com/ietf-tools/idnits-mirror/main/idnits /usr/local/bin/ +RUN chmod +rx /usr/local/bin/idnits + +# Turn off rsyslog kernel logging (doesn't work in Docker) +RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf + +# Colorize the bash shell +RUN sed -i 's/#force_color_prompt=/force_color_prompt=/' /root/.bashrc + +# Turn off rsyslog kernel logging (doesn't work in Docker) +RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf + +# Fetch wait-for utility +ADD https://raw.githubusercontent.com/eficode/wait-for/v2.1.3/wait-for /usr/local/bin/ +RUN chmod +rx /usr/local/bin/wait-for + +# Create assets directory +RUN mkdir -p /assets + +# Create workspace +RUN mkdir -p /workspace +WORKDIR /workspace diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index 7e58e797ca3..c8286b24290 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -18,9 +18,6 @@ sudo chown -R dev:dev "$WORKSPACEDIR/.vite" sudo chown -R dev:dev "$WORKSPACEDIR/.yarn/unplugged" sudo chown dev:dev "/assets" -echo "Fix chromedriver /dev/shm permissions..." -sudo chmod 1777 /dev/shm - # Run nginx echo "Starting nginx..." sudo nginx diff --git a/ietf/settings.py b/ietf/settings.py index 34076f3299f..d0cb6b8527d 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -572,8 +572,6 @@ def skip_unreadable_post(record): TEST_DIFF_FAILURE_DIR = "/tmp/test/failure/" -TEST_GHOSTDRIVER_LOG_PATH = "ghostdriver.log" - # These are regexes TEST_URL_COVERAGE_EXCLUDE = [ r"^\^admin/", diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py index a901df66f5c..081f11507a9 100644 --- a/ietf/utils/jstest.py +++ b/ietf/utils/jstest.py @@ -9,8 +9,8 @@ skip_message = "" try: from selenium import webdriver - from selenium.webdriver.chrome.service import Service - from selenium.webdriver.chrome.options import Options + from selenium.webdriver.firefox.service import Service + from selenium.webdriver.firefox.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.common.desired_capabilities import DesiredCapabilities except ImportError as e: @@ -21,7 +21,7 @@ from ietf.utils.pipe import pipe from ietf.utils.test_runner import IetfLiveServerTestCase -executable_name = 'chromedriver' +executable_name = 'geckodriver' code, out, err = pipe('{} --version'.format(executable_name)) if code != 0: skip_selenium = True @@ -30,20 +30,17 @@ print(" "+skip_message) def start_web_driver(): - service = Service(executable_path="chromedriver", - log_path=settings.TEST_GHOSTDRIVER_LOG_PATH) + service = Service(executable_path=executable_name) service.start() options = Options() - options.add_argument("headless") - options.add_argument("disable-extensions") - options.add_argument("disable-gpu") # headless needs this - options.add_argument("no-sandbox") # docker needs this - dc = DesiredCapabilities.CHROME - dc["goog:loggingPrefs"] = {"browser": "ALL"} + # options.log.level = "TRACE" + options.add_argument("--headless") + options.add_argument("--window-size=2048,1536") + dc = DesiredCapabilities.FIREFOX # For selenium 3: - return webdriver.Chrome("chromedriver", options=options, desired_capabilities=dc) + return webdriver.Firefox(options=options, desired_capabilities=dc) # For selenium 4: - # return webdriver.Chrome(service=service, options=options, desired_capabilities=dc) + # return webdriver.Firefox(service=service, options=options, desired_capabilities=dc) def selenium_enabled(): From 00765eb608926e288e6489561ff4dc35e03535d3 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 25 Oct 2023 10:18:54 -0700 Subject: [PATCH 02/17] Switch to selenium 4 --- docker/configs/settings_local.py | 2 ++ docker/scripts/app-install-chromedriver.sh | 18 ------------------ ietf/utils/jstest.py | 11 ++--------- requirements.txt | 4 ++-- 4 files changed, 6 insertions(+), 29 deletions(-) delete mode 100755 docker/scripts/app-install-chromedriver.sh diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 07c16c2e9a1..a9647958d4f 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -7,6 +7,8 @@ from ietf.settings_postgresqldb import DATABASES # pyflakes:ignore +IDTRACKER_BASE_URL="http://localhost:8000" + IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" IDSUBMIT_STAGING_PATH = "test/staging/" diff --git a/docker/scripts/app-install-chromedriver.sh b/docker/scripts/app-install-chromedriver.sh deleted file mode 100755 index 43532a1cf68..00000000000 --- a/docker/scripts/app-install-chromedriver.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -HOSTARCH=$(arch) -if [ $HOSTARCH == "x86_64" ]; then - echo "Installing chrome driver..." - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list - apt-get update -y - apt-get install -y google-chrome-stable - CHROMEVER=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") - DRIVERVER=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROMEVER") - wget -q --continue -P /chromedriver "http://chromedriver.storage.googleapis.com/$DRIVERVER/chromedriver_linux64.zip" - unzip /chromedriver/chromedriver* -d /chromedriver - ln -s /chromedriver/chromedriver /usr/local/bin/chromedriver - ln -s /chromedriver/chromedriver /usr/bin/chromedriver -else - echo "This architecture doesn't support chromedriver. Skipping installation..." -fi \ No newline at end of file diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py index 081f11507a9..f206021e701 100644 --- a/ietf/utils/jstest.py +++ b/ietf/utils/jstest.py @@ -12,7 +12,6 @@ from selenium.webdriver.firefox.service import Service from selenium.webdriver.firefox.options import Options from selenium.webdriver.common.by import By - from selenium.webdriver.common.desired_capabilities import DesiredCapabilities except ImportError as e: skip_selenium = True skip_message = "Skipping selenium tests: %s" % e @@ -30,17 +29,11 @@ print(" "+skip_message) def start_web_driver(): - service = Service(executable_path=executable_name) - service.start() + service = Service(service_args=['--log-no-truncate']) options = Options() # options.log.level = "TRACE" options.add_argument("--headless") - options.add_argument("--window-size=2048,1536") - dc = DesiredCapabilities.FIREFOX - # For selenium 3: - return webdriver.Firefox(options=options, desired_capabilities=dc) - # For selenium 4: - # return webdriver.Firefox(service=service, options=options, desired_capabilities=dc) + return webdriver.Firefox(service=service, options=options) def selenium_enabled(): diff --git a/requirements.txt b/requirements.txt index bbd2ee5f70d..d714d4da7f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,12 +65,12 @@ types-requests>=2.27.1 requests-mock>=1.9.3 rfc2html>=2.0.3 scout-apm>=2.24.2 -selenium>=3.141.0,<4.0 +selenium>=4.0 tblib>=1.7.0 # So that the django test runner provides tracebacks tlds>=2022042700 # Used to teach bleach about which TLDs currently exist tqdm>=4.64.0 Unidecode>=1.3.4 -urllib3<2 # v2 causes selenium tests to fail with "Timeout value was =2 weasyprint>=59 xml2rfc>=3.12.4 xym>=0.6,<1.0 From 36e82f6e42a022d132cff5ed0dbb13ac5c4b7f73 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 25 Oct 2023 10:19:30 -0700 Subject: [PATCH 03/17] Undo --- docker/configs/settings_local.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index a9647958d4f..07c16c2e9a1 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -7,8 +7,6 @@ from ietf.settings_postgresqldb import DATABASES # pyflakes:ignore -IDTRACKER_BASE_URL="http://localhost:8000" - IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" IDSUBMIT_STAGING_PATH = "test/staging/" From 53a071a08e922a9e329672609bfaa9c34f121ec9 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 25 Oct 2023 10:24:10 -0700 Subject: [PATCH 04/17] Remove comment --- ietf/utils/jstest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py index f206021e701..722cf57b4a3 100644 --- a/ietf/utils/jstest.py +++ b/ietf/utils/jstest.py @@ -31,7 +31,6 @@ def start_web_driver(): service = Service(service_args=['--log-no-truncate']) options = Options() - # options.log.level = "TRACE" options.add_argument("--headless") return webdriver.Firefox(service=service, options=options) From 75c9868cced64f33299b51677a1d30dac0ad6646 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 25 Oct 2023 11:25:47 -0700 Subject: [PATCH 05/17] Fixes --- ietf/meeting/tests_js.py | 2 +- ietf/utils/jstest.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index e69afe5ca4a..226156372be 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -499,7 +499,7 @@ def test_past_swap_days_buttons(self): clicked_index = 1 # scroll so the button we want to click is just below the navbar, otherwise it may # fall beneath the sessions panel - navbar = self.driver.find_element_by_class_name('navbar') + navbar = self.driver.find_element(By.CSS_SELECTOR, '.navbar') self.driver.execute_script( 'window.scrollBy({top: %s, behavior: "instant"})' % ( future_swap_days_buttons[1].location['y'] - navbar.size['height'] diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py index 722cf57b4a3..07d6ed9dd00 100644 --- a/ietf/utils/jstest.py +++ b/ietf/utils/jstest.py @@ -1,7 +1,8 @@ # Copyright The IETF Trust 2014-2021, All Rights Reserved # -*- coding: utf-8 -*- -from django.conf import settings +import os + from django.urls import reverse as urlreverse from unittest import skipIf @@ -29,9 +30,10 @@ print(" "+skip_message) def start_web_driver(): - service = Service(service_args=['--log-no-truncate']) + service = Service(log_output=f"{executable_name}.log", service_args=['--log-no-truncate']) options = Options() options.add_argument("--headless") + os.environ["MOZ_REMOTE_SETTINGS_DEVTOOLS"] = "1" return webdriver.Firefox(service=service, options=options) From 371571b4148b8f63ad7743261bc4b64d26699773 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 25 Oct 2023 11:27:43 -0700 Subject: [PATCH 06/17] Restore non-standard line endings to minimize diff --- docker/base.Dockerfile | 316 +++++++++++++++---------------- docker/configs/settings_local.py | 2 + docker/docker-compose.extend.yml | 2 +- 3 files changed, 154 insertions(+), 166 deletions(-) diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index f7aff849fbc..36eb41a9a9f 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -1,165 +1,151 @@ -FROM python:3.9-bullseye -LABEL maintainer="IETF Tools Team " - -ENV DEBIAN_FRONTEND=noninteractive - -# Update system packages -RUN apt-get update \ - && apt-get -qy upgrade \ - && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 - -# Add Node.js Source -RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \ - && mkdir -p /etc/apt/keyrings\ - && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg -ENV NODE_MAJOR=16 -RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list - -# Add Docker Source -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg -RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null - -# Add PostgreSQL Source -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - -RUN cat /etc/apt/sources.list.d/docker.list - -# Install the packages we need -RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends \ - apache2-utils \ - apt-file \ - bash \ - build-essential \ - curl \ - default-jdk \ - docker-ce-cli \ - enscript \ - firefox-esr \ - gawk \ - g++ \ - gcc \ - ghostscript \ - git \ - gnupg \ - jq \ - less \ - libcairo2-dev \ - libgtk2.0-0 \ - libgtk-3-0 \ - libnotify-dev \ - libgconf-2-4 \ - libgbm-dev \ - libnss3 \ - libxss1 \ - libasound2 \ - libxtst6 \ - libmagic-dev \ - libmariadb-dev \ - libmemcached-tools \ - locales \ - make \ - mariadb-client \ - memcached \ - nano \ - netcat \ - nodejs \ - pgloader \ - pigz \ - postgresql-client-14 \ - pv \ - python3-ipython \ - ripgrep \ - rsync \ - rsyslog \ - ruby \ - ruby-rubygems \ - unzip \ - wget \ - xauth \ - xvfb \ - yang-tools \ - zsh - -# Install kramdown-rfc2629 (ruby) -RUN gem install kramdown-rfc2629 - -# Firefox -# ARG FIREFOX_VERSION=latest -# RUN FIREFOX_DOWNLOAD_URL=$(if [ $FIREFOX_VERSION = "latest" ] || [ $FIREFOX_VERSION = "beta-latest" ] || [ $FIREFOX_VERSION = "nightly-latest" ] || [ $FIREFOX_VERSION = "devedition-latest" ] || [ $FIREFOX_VERSION = "esr-latest" ]; then echo "https://download.mozilla.org/?product=firefox-$FIREFOX_VERSION-ssl&os=linux64&lang=en-US"; else echo "https://download-installer.cdn.mozilla.net/pub/firefox/releases/$FIREFOX_VERSION/linux-x86_64/en-US/firefox-$FIREFOX_VERSION.tar.bz2"; fi) \ -# && apt-get update -qqy \ -# && apt-get -qqy --no-install-recommends install firefox-esr libavcodec-extra \ -# && rm -rf /var/lib/apt/lists/* /var/cache/apt/* \ -# && wget --no-verbose -O /tmp/firefox.tar.bz2 $FIREFOX_DOWNLOAD_URL \ -# && apt-get -y purge firefox-esr \ -# && rm -rf /opt/firefox \ -# && tar -C /opt -xjf /tmp/firefox.tar.bz2 \ -# && rm /tmp/firefox.tar.bz2 \ -# && mv /opt/firefox /opt/firefox-$FIREFOX_VERSION \ -# && ln -fs /opt/firefox-$FIREFOX_VERSION/firefox /usr/bin/firefox - -# GeckoDriver -ARG GECKODRIVER_VERSION=latest -RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0.33.0"; else echo $GECKODRIVER_VERSION; fi) \ - && echo "Using GeckoDriver version: "$GK_VERSION \ - && wget --no-verbose -O /tmp/geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v$GK_VERSION/geckodriver-v$GK_VERSION-linux64.tar.gz \ - && rm -rf /opt/geckodriver \ - && tar -C /opt -zxf /tmp/geckodriver.tar.gz \ - && rm /tmp/geckodriver.tar.gz \ - && mv /opt/geckodriver /opt/geckodriver-$GK_VERSION \ - && chmod 755 /opt/geckodriver-$GK_VERSION \ - && ln -fs /opt/geckodriver-$GK_VERSION /usr/bin/geckodriver - -# # Fix /dev/shm permissions for chromedriver -# RUN chmod 1777 /dev/shm - -# Activate Yarn -RUN corepack enable - -# Get rid of installation files we don't need in the image, to reduce size -RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /var/cache/apt/* - -# "fake" dbus address to prevent errors -# https://github.com/SeleniumHQ/docker-selenium/issues/87 -ENV DBUS_SESSION_BUS_ADDRESS=/dev/null - -# avoid million NPM install messages -ENV npm_config_loglevel warn -# allow installing when the main user is root -ENV npm_config_unsafe_perm true -# disable NPM funding messages -ENV npm_config_fund false - -# Set locale to en_US.UTF-8 -RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \ - echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ - echo "LANG=en_US.UTF-8" > /etc/locale.conf && \ - dpkg-reconfigure locales && \ - locale-gen en_US.UTF-8 && \ - update-locale LC_ALL en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 - -# Install idnits -ADD https://raw.githubusercontent.com/ietf-tools/idnits-mirror/main/idnits /usr/local/bin/ -RUN chmod +rx /usr/local/bin/idnits - -# Turn off rsyslog kernel logging (doesn't work in Docker) -RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf - -# Colorize the bash shell -RUN sed -i 's/#force_color_prompt=/force_color_prompt=/' /root/.bashrc - -# Turn off rsyslog kernel logging (doesn't work in Docker) -RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf - -# Fetch wait-for utility -ADD https://raw.githubusercontent.com/eficode/wait-for/v2.1.3/wait-for /usr/local/bin/ -RUN chmod +rx /usr/local/bin/wait-for - -# Create assets directory -RUN mkdir -p /assets - -# Create workspace -RUN mkdir -p /workspace -WORKDIR /workspace +FROM python:3.9-bullseye +LABEL maintainer="IETF Tools Team " + +ENV DEBIAN_FRONTEND=noninteractive + +# Update system packages +RUN apt-get update \ + && apt-get -qy upgrade \ + && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 + +# Add Node.js Source +RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \ + && mkdir -p /etc/apt/keyrings\ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg +ENV NODE_MAJOR=16 +RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + +# Add Docker Source +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null + +# Add PostgreSQL Source +RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + +RUN cat /etc/apt/sources.list.d/docker.list + +# Install the packages we need +RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends \ + apache2-utils \ + apt-file \ + bash \ + build-essential \ + curl \ + default-jdk \ + docker-ce-cli \ + enscript \ + firefox-esr \ + gawk \ + g++ \ + gcc \ + ghostscript \ + git \ + gnupg \ + jq \ + less \ + libcairo2-dev \ + libgtk2.0-0 \ + libgtk-3-0 \ + libnotify-dev \ + libgconf-2-4 \ + libgbm-dev \ + libnss3 \ + libxss1 \ + libasound2 \ + libxtst6 \ + libmagic-dev \ + libmariadb-dev \ + libmemcached-tools \ + locales \ + make \ + mariadb-client \ + memcached \ + nano \ + netcat \ + nodejs \ + pgloader \ + pigz \ + postgresql-client-14 \ + pv \ + python3-ipython \ + ripgrep \ + rsync \ + rsyslog \ + ruby \ + ruby-rubygems \ + unzip \ + wget \ + xauth \ + xvfb \ + yang-tools \ + zsh + +# Install kramdown-rfc2629 (ruby) +RUN gem install kramdown-rfc2629 + +# GeckoDriver +ARG GECKODRIVER_VERSION=latest +RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0.33.0"; else echo $GECKODRIVER_VERSION; fi) \ + && echo "Using GeckoDriver version: "$GK_VERSION \ + && wget --no-verbose -O /tmp/geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v$GK_VERSION/geckodriver-v$GK_VERSION-linux64.tar.gz \ + && rm -rf /opt/geckodriver \ + && tar -C /opt -zxf /tmp/geckodriver.tar.gz \ + && rm /tmp/geckodriver.tar.gz \ + && mv /opt/geckodriver /opt/geckodriver-$GK_VERSION \ + && chmod 755 /opt/geckodriver-$GK_VERSION \ + && ln -fs /opt/geckodriver-$GK_VERSION /usr/bin/geckodriver + +# # Fix /dev/shm permissions for chromedriver +# RUN chmod 1777 /dev/shm + +# Activate Yarn +RUN corepack enable + +# Get rid of installation files we don't need in the image, to reduce size +RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /var/cache/apt/* + +# "fake" dbus address to prevent errors +# https://github.com/SeleniumHQ/docker-selenium/issues/87 +ENV DBUS_SESSION_BUS_ADDRESS=/dev/null + +# avoid million NPM install messages +ENV npm_config_loglevel warn +# allow installing when the main user is root +ENV npm_config_unsafe_perm true +# disable NPM funding messages +ENV npm_config_fund false + +# Set locale to en_US.UTF-8 +RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \ + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ + echo "LANG=en_US.UTF-8" > /etc/locale.conf && \ + dpkg-reconfigure locales && \ + locale-gen en_US.UTF-8 && \ + update-locale LC_ALL en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +# Install idnits +ADD https://raw.githubusercontent.com/ietf-tools/idnits-mirror/main/idnits /usr/local/bin/ +RUN chmod +rx /usr/local/bin/idnits + +# Turn off rsyslog kernel logging (doesn't work in Docker) +RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf + +# Colorize the bash shell +RUN sed -i 's/#force_color_prompt=/force_color_prompt=/' /root/.bashrc + +# Turn off rsyslog kernel logging (doesn't work in Docker) +RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf + +# Fetch wait-for utility +ADD https://raw.githubusercontent.com/eficode/wait-for/v2.1.3/wait-for /usr/local/bin/ +RUN chmod +rx /usr/local/bin/wait-for + +# Create assets directory +RUN mkdir -p /assets + +# Create workspace +RUN mkdir -p /workspace +WORKDIR /workspace diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 07c16c2e9a1..a9647958d4f 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -7,6 +7,8 @@ from ietf.settings_postgresqldb import DATABASES # pyflakes:ignore +IDTRACKER_BASE_URL="http://localhost:8000" + IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" IDSUBMIT_STAGING_PATH = "test/staging/" diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index d055c976f49..8dc1bbac05b 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -11,7 +11,7 @@ services: - /workspace/__pycache__ - /workspace/.vite - /workspace/.yarn/unplugged - - app-assets:/assets + - /Users/lars/Documents/IETF/assets:/assets db: ports: - '5432' From b9f16520a07642b456d6aefb911ddf47a20bb19a Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 25 Oct 2023 11:30:46 -0700 Subject: [PATCH 07/17] Undo --- docker/configs/settings_local.py | 2 -- docker/docker-compose.extend.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index a9647958d4f..07c16c2e9a1 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -7,8 +7,6 @@ from ietf.settings_postgresqldb import DATABASES # pyflakes:ignore -IDTRACKER_BASE_URL="http://localhost:8000" - IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" IDSUBMIT_STAGING_PATH = "test/staging/" diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index 8dc1bbac05b..d055c976f49 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -11,7 +11,7 @@ services: - /workspace/__pycache__ - /workspace/.vite - /workspace/.yarn/unplugged - - /Users/lars/Documents/IETF/assets:/assets + - app-assets:/assets db: ports: - '5432' From c4df0bfb9737695eb58b4478a0658e5b5fb8c080 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 25 Oct 2023 15:25:50 -0700 Subject: [PATCH 08/17] Remove comment --- docker/base.Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 36eb41a9a9f..f2fea13bac2 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -97,9 +97,6 @@ RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0. && chmod 755 /opt/geckodriver-$GK_VERSION \ && ln -fs /opt/geckodriver-$GK_VERSION /usr/bin/geckodriver -# # Fix /dev/shm permissions for chromedriver -# RUN chmod 1777 /dev/shm - # Activate Yarn RUN corepack enable From 21e54c0f879251e94b5d1875bee699fb46867934 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 26 Oct 2023 17:31:14 -0300 Subject: [PATCH 09/17] test: Fix test_upcoming_view_time_zone_selection The inputs are hidden and managed by JS, so click the visible elements instead. --- ietf/meeting/tests_js.py | 43 ++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index 226156372be..6afae1f5a37 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -1232,10 +1232,13 @@ def _assert_ietf_tz_correct(meetings, tz): self.driver.get(self.absreverse('ietf.meeting.views.upcoming')) tz_select_input = self.driver.find_element(By.ID, 'timezone-select') tz_select_bottom_input = self.driver.find_element(By.ID, 'timezone-select-bottom') - local_tz_link = self.driver.find_element(By.ID, 'local-timezone') - utc_tz_link = self.driver.find_element(By.ID, 'utc-timezone') - local_tz_bottom_link = self.driver.find_element(By.ID, 'local-timezone-bottom') - utc_tz_bottom_link = self.driver.find_element(By.ID, 'utc-timezone-bottom') + + # For things we click, need to click the labels / actually visible items. The actual inputs are hidden + # and managed by the JS. + local_tz_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="local-timezone"]') + utc_tz_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="utc-timezone"]') + local_tz_bottom_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="local-timezone-bottom"]') + utc_tz_bottom_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="utc-timezone-bottom"]') # wait for the select box to be updated - look for an arbitrary time zone to be in # its options list to detect this @@ -1245,7 +1248,10 @@ def _assert_ietf_tz_correct(meetings, tz): (By.CSS_SELECTOR, '#timezone-select > option[value="%s"]' % arbitrary_tz) ) ) - + tz_selector_clickables = self.driver.find_elements(By.CSS_SELECTOR, ".tz-display .select2") + self.assertEqual(len(tz_selector_clickables), 2) + (tz_selector_top, tz_selector_bottom) = tz_selector_clickables + arbitrary_tz_bottom_opt = tz_select_bottom_input.find_element(By.CSS_SELECTOR, '#timezone-select-bottom > option[value="%s"]' % arbitrary_tz) @@ -1266,8 +1272,7 @@ def _assert_ietf_tz_correct(meetings, tz): _assert_ietf_tz_correct(ietf_meetings, local_tz) # click 'utc' button - self.driver.execute_script("arguments[0].click();", utc_tz_link) # FIXME-LARS: not working: - # utc_tz_link.click() + utc_tz_link.click() self.wait.until(expected_conditions.element_to_be_selected(utc_tz_opt)) self.assertFalse(local_tz_opt.is_selected()) self.assertFalse(local_tz_bottom_opt.is_selected()) @@ -1279,8 +1284,7 @@ def _assert_ietf_tz_correct(meetings, tz): _assert_ietf_tz_correct(ietf_meetings, 'UTC') # click back to 'local' - self.driver.execute_script("arguments[0].click();", local_tz_link) # FIXME-LARS: not working: - # local_tz_link.click() + local_tz_link.click() self.wait.until(expected_conditions.element_to_be_selected(local_tz_opt)) self.assertTrue(local_tz_opt.is_selected()) self.assertTrue(local_tz_bottom_opt.is_selected()) @@ -1292,7 +1296,12 @@ def _assert_ietf_tz_correct(meetings, tz): _assert_ietf_tz_correct(ietf_meetings, local_tz) # Now select a different item from the select input - arbitrary_tz_opt.click() + tz_selector_top.click() + self.wait.until( + expected_conditions.presence_of_element_located( + (By.CSS_SELECTOR, 'span.select2-container .select2-results li[id$="America/Halifax"]') + ) + ).click() self.wait.until(expected_conditions.element_to_be_selected(arbitrary_tz_opt)) self.assertFalse(local_tz_opt.is_selected()) self.assertFalse(local_tz_bottom_opt.is_selected()) @@ -1305,8 +1314,7 @@ def _assert_ietf_tz_correct(meetings, tz): # Now repeat those tests using the widgets at the bottom of the page # click 'utc' button - self.driver.execute_script("arguments[0].click();", utc_tz_bottom_link) # FIXME-LARS: not working: - # utc_tz_bottom_link.click() + utc_tz_bottom_link.click() self.wait.until(expected_conditions.element_to_be_selected(utc_tz_opt)) self.assertFalse(local_tz_opt.is_selected()) self.assertFalse(local_tz_bottom_opt.is_selected()) @@ -1318,8 +1326,7 @@ def _assert_ietf_tz_correct(meetings, tz): _assert_ietf_tz_correct(ietf_meetings, 'UTC') # click back to 'local' - self.driver.execute_script("arguments[0].click();", local_tz_bottom_link) # FIXME-LARS: not working: - # local_tz_bottom_link.click() + local_tz_bottom_link.click() self.wait.until(expected_conditions.element_to_be_selected(local_tz_opt)) self.assertTrue(local_tz_opt.is_selected()) self.assertTrue(local_tz_bottom_opt.is_selected()) @@ -1331,7 +1338,13 @@ def _assert_ietf_tz_correct(meetings, tz): _assert_ietf_tz_correct(ietf_meetings, local_tz) # Now select a different item from the select input - arbitrary_tz_bottom_opt.click() + self.scroll_to_element(tz_selector_bottom) + tz_selector_bottom.click() + self.wait.until( + expected_conditions.presence_of_element_located( + (By.CSS_SELECTOR, 'span.select2-container .select2-results li[id$="America/Halifax"]') + ) + ).click() self.wait.until(expected_conditions.element_to_be_selected(arbitrary_tz_opt)) self.assertFalse(local_tz_opt.is_selected()) self.assertFalse(local_tz_bottom_opt.is_selected()) From 228e32a4be0d38febb47a8ce37ea2c756dcd9a61 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 26 Oct 2023 18:08:33 -0300 Subject: [PATCH 10/17] test: Clumsy fix to test_upcoming_materials_modal Waiting for the button to be clickable does not work because the modal is still fading in, so does not actually close. Would be better to check for it responding, but I haven't found the right way to do that yet. --- ietf/meeting/tests_js.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index 6afae1f5a37..e299cfef6f5 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -1314,6 +1314,7 @@ def _assert_ietf_tz_correct(meetings, tz): # Now repeat those tests using the widgets at the bottom of the page # click 'utc' button + self.scroll_to_element(utc_tz_bottom_link) utc_tz_bottom_link.click() self.wait.until(expected_conditions.element_to_be_selected(utc_tz_opt)) self.assertFalse(local_tz_opt.is_selected()) @@ -1326,6 +1327,7 @@ def _assert_ietf_tz_correct(meetings, tz): _assert_ietf_tz_correct(ietf_meetings, 'UTC') # click back to 'local' + self.scroll_to_element(local_tz_bottom_link) local_tz_bottom_link.click() self.wait.until(expected_conditions.element_to_be_selected(local_tz_opt)) self.assertTrue(local_tz_opt.is_selected()) @@ -1395,6 +1397,7 @@ def test_upcoming_materials_modal(self): ), 'Modal close button not found or not clickable', ) + time.sleep(0.3) # gross, but the button is clickable while still fading in close_modal_button.click() self.wait.until( expected_conditions.invisibility_of_element(modal_div), From 70e2e2ce1b8b9ebf86b2faa2a90107523612d209 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 26 Oct 2023 22:28:59 -0300 Subject: [PATCH 11/17] test: Fix test_add_author_forms Sending '\n' does not seem to work as it did before, so click the option instead. Also reverted some fixme hacks that seem to be obe. --- ietf/doc/tests_js.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ietf/doc/tests_js.py b/ietf/doc/tests_js.py index ac63c0995ef..acd74c4a0b1 100644 --- a/ietf/doc/tests_js.py +++ b/ietf/doc/tests_js.py @@ -41,7 +41,7 @@ def _fill_in_author_form(form_elt, name, email, affiliation, country): (By.CSS_SELECTOR, result_selector), name )) - input.send_keys('\n') # select the object + self.driver.find_element(By.CSS_SELECTOR, result_selector).click() # After the author is selected, the email select options will be populated. # Wait for that, then click on the option corresponding to the requested email. @@ -94,10 +94,8 @@ def _read_author_form(form_elt): # get the "add author" button so we can add blank author forms add_author_button = self.driver.find_element(By.ID, 'add-author-button') for index, auth in enumerate(authors): - self.driver.execute_script("arguments[0].scrollIntoView();", add_author_button) # FIXME: no idea why this fails: - # self.scroll_to_element(add_author_button) # Can only click if it's in view! - self.driver.execute_script("arguments[0].click();", add_author_button) # FIXME: no idea why this fails: - # add_author_button.click() # Create a new form. Automatically scrolls to it. + self.scroll_to_element(add_author_button) # Can only click if it's in view! + add_author_button.click() # Create a new form. Automatically scrolls to it. author_forms = authors_list.find_elements(By.CLASS_NAME, 'author-panel') authors_added = index + 1 self.assertEqual(len(author_forms), authors_added + 1) # Started with 1 author, hence +1 @@ -119,9 +117,8 @@ def _read_author_form(form_elt): self.driver.find_element(By.ID, 'id_basis').send_keys('change testing') # Now click the 'submit' button and check that the update was accepted. submit_button = self.driver.find_element(By.CSS_SELECTOR, '#content button[type="submit"]') - self.driver.execute_script("arguments[0].click();", submit_button) # FIXME: no idea why this fails: - # self.scroll_to_element(submit_button) - # submit_button.click() + self.scroll_to_element(submit_button) + submit_button.click() # Wait for redirect to the document_main view self.wait.until( expected_conditions.url_to_be( From 06a576fd1db149b335cd448df498e18fb4d21563 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Tue, 7 Nov 2023 22:24:27 -0500 Subject: [PATCH 12/17] ci: Update base.Dockerfile --- docker/base.Dockerfile | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index d0b7ab47472..12d03d74d2d 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -24,8 +24,6 @@ RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/d RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN cat /etc/apt/sources.list.d/docker.list - # Install the packages we need RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends \ apache2-utils \ @@ -85,18 +83,6 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends # Install kramdown-rfc2629 (ruby) RUN gem install kramdown-rfc2629 -# GeckoDriver -ARG GECKODRIVER_VERSION=latest -RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0.33.0"; else echo $GECKODRIVER_VERSION; fi) \ - && echo "Using GeckoDriver version: "$GK_VERSION \ - && wget --no-verbose -O /tmp/geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v$GK_VERSION/geckodriver-v$GK_VERSION-linux64.tar.gz \ - && rm -rf /opt/geckodriver \ - && tar -C /opt -zxf /tmp/geckodriver.tar.gz \ - && rm /tmp/geckodriver.tar.gz \ - && mv /opt/geckodriver /opt/geckodriver-$GK_VERSION \ - && chmod 755 /opt/geckodriver-$GK_VERSION \ - && ln -fs /opt/geckodriver-$GK_VERSION /usr/bin/geckodriver - # GeckoDriver ARG GECKODRIVER_VERSION=latest RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0.33.0"; else echo $GECKODRIVER_VERSION; fi) \ From 2234a00e98073ee0efcbec4d06067ad88ef65cb3 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Sun, 12 Nov 2023 21:21:52 -0500 Subject: [PATCH 13/17] test: add resource limits to dev/tests/debug.sh env --- dev/tests/debug.sh | 2 +- dev/tests/docker-compose.debug.yml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dev/tests/debug.sh b/dev/tests/debug.sh index 405daae3774..8ed28371aec 100644 --- a/dev/tests/debug.sh +++ b/dev/tests/debug.sh @@ -12,7 +12,7 @@ echo "Fetching latest images..." docker pull ghcr.io/ietf-tools/datatracker-app-base:latest docker pull ghcr.io/ietf-tools/datatracker-db:latest echo "Starting containers..." -docker compose -f docker-compose.debug.yml -p dtdebug up -d +docker compose -f docker-compose.debug.yml -p dtdebug --compatibility up -d echo "Copying working directory into container..." docker compose -p dtdebug cp ../../. app:/__w/datatracker/datatracker/ echo "Run prepare script..." diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml index 74491a5b2e8..6362ef072e4 100644 --- a/dev/tests/docker-compose.debug.yml +++ b/dev/tests/docker-compose.debug.yml @@ -16,6 +16,12 @@ services: CI: 'true' GITHUB_ACTIONS: 'true' HOME: /github/home + deploy: + resources: + limits: + cpus: '2' + memory: '7GB' + db: image: ghcr.io/ietf-tools/datatracker-db:latest restart: unless-stopped From e24abdc117280e60a4107c50e09617f7fa15b8de Mon Sep 17 00:00:00 2001 From: NGPixel Date: Mon, 13 Nov 2023 04:20:31 -0500 Subject: [PATCH 14/17] ci: add upload of geckodriver.log on failure --- .github/workflows/tests.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5884b724fc8..df5d68fad7b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,6 +51,13 @@ jobs: fi coverage xml + - name: Upload geckodriver.log + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: geckodriverlog + path: geckodriver.log + - name: Upload Coverage Results to Codecov uses: codecov/codecov-action@v3.1.4 with: @@ -161,4 +168,4 @@ jobs: with: name: playwright-legacy-results-${{ matrix.project }} path: playwright/test-results/ - if-no-files-found: ignore \ No newline at end of file + if-no-files-found: ignore From 92ba7bc002611f758a072b5e2fb3a632642467bf Mon Sep 17 00:00:00 2001 From: NGPixel Date: Mon, 13 Nov 2023 05:14:26 -0500 Subject: [PATCH 15/17] ci: run tests as user 1001 --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index df5d68fad7b..8bbe373aaad 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,9 @@ jobs: tests-python: name: Python Tests runs-on: ubuntu-latest - container: ghcr.io/ietf-tools/datatracker-app-base:latest + container: + image: ghcr.io/ietf-tools/datatracker-app-base:latest + options: --user 1001 services: db: From 22f6673a705ee802ad4436b5e549210815ba42ce Mon Sep 17 00:00:00 2001 From: NGPixel Date: Mon, 13 Nov 2023 05:24:27 -0500 Subject: [PATCH 16/17] ci: run app-create-dirs as sudo --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8bbe373aaad..966fb1a931f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,6 +28,7 @@ jobs: run: | chmod +x ./dev/tests/prepare.sh sh ./dev/tests/prepare.sh + sudo sh ./docker/scripts/app-create-dirs.sh - name: Ensure DB is ready run: | From fd7b088a3376c7bf84b63f32033bed7226a280bc Mon Sep 17 00:00:00 2001 From: NGPixel Date: Mon, 13 Nov 2023 05:32:19 -0500 Subject: [PATCH 17/17] ci: set home folder to root to run tests --- .github/workflows/tests.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 966fb1a931f..31150ed99b0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,9 +13,7 @@ jobs: tests-python: name: Python Tests runs-on: ubuntu-latest - container: - image: ghcr.io/ietf-tools/datatracker-app-base:latest - options: --user 1001 + container: ghcr.io/ietf-tools/datatracker-app-base:latest services: db: @@ -28,7 +26,6 @@ jobs: run: | chmod +x ./dev/tests/prepare.sh sh ./dev/tests/prepare.sh - sudo sh ./docker/scripts/app-create-dirs.sh - name: Ensure DB is ready run: | @@ -48,9 +45,9 @@ jobs: echo "Running tests..." if [[ "x${{ github.event.inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then echo "Lower coverage failures will be ignored." - ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage + HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage else - ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test + HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test fi coverage xml