ci: Switch to using geckodriver

.github/workflows/tests.yml

@@ -45,12 +45,19 @@ jobs:
         echo "Running tests..."
         if [[ "x${{ github.event.inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then
           echo "Lower coverage failures will be ignored."
-          ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage
+          HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage
         else
-          ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test
+          HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test
         fi
         coverage xml
 
+    - name: Upload geckodriver.log
+      uses: actions/upload-artifact@v3
+      if: ${{ failure() }}
+      with:
+        name: geckodriverlog
+        path: geckodriver.log
+
     - name: Upload Coverage Results to Codecov
       uses: codecov/codecov-action@v3.1.4
       with:
@@ -161,4 +168,4 @@ jobs:
       with:
         name: playwright-legacy-results-${{ matrix.project }}
         path: playwright/test-results/
-        if-no-files-found: ignore
+        if-no-files-found: ignore

dev/tests/debug.sh

@@ -12,7 +12,7 @@ echo "Fetching latest images..."
 docker pull ghcr.io/ietf-tools/datatracker-app-base:latest
 docker pull ghcr.io/ietf-tools/datatracker-db:latest
 echo "Starting containers..."
-docker compose -f docker-compose.debug.yml -p dtdebug up -d
+docker compose -f docker-compose.debug.yml -p dtdebug --compatibility up -d
 echo "Copying working directory into container..."
 docker compose -p dtdebug cp ../../. app:/__w/datatracker/datatracker/
 echo "Run prepare script..."

dev/tests/docker-compose.debug.yml

@@ -16,6 +16,12 @@ services:
             CI: 'true'
             GITHUB_ACTIONS: 'true'
             HOME: /github/home
+        deploy:    
+            resources:
+                limits:
+                    cpus: '2'
+                    memory: '7GB'
+
     db:
         image: ghcr.io/ietf-tools/datatracker-db:latest
         restart: unless-stopped

docker/base.Dockerfile

@@ -19,7 +19,7 @@ RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesourc
 RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
 RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
     $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
-    
+
 # Add PostgreSQL Source 
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
@@ -83,15 +83,6 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends
 # Install kramdown-rfc2629 (ruby)
 RUN gem install kramdown-rfc2629
 
-# Install chromedriver
-COPY docker/scripts/app-install-chromedriver.sh /tmp/app-install-chromedriver.sh
-RUN sed -i 's/\r$//' /tmp/app-install-chromedriver.sh && \
-    chmod +x /tmp/app-install-chromedriver.sh
-RUN /tmp/app-install-chromedriver.sh
-
-# Fix /dev/shm permissions for chromedriver
-RUN chmod 1777 /dev/shm
-
 # GeckoDriver
 ARG GECKODRIVER_VERSION=latest
 RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0.33.0"; else echo $GECKODRIVER_VERSION; fi) \

docker/scripts/app-init.sh

@@ -18,9 +18,6 @@ sudo chown -R dev:dev "$WORKSPACEDIR/.vite"
 sudo chown -R dev:dev "$WORKSPACEDIR/.yarn/unplugged"
 sudo chown dev:dev "/assets"
 
-echo "Fix chromedriver /dev/shm permissions..."
-sudo chmod 1777 /dev/shm
-
 # Run nginx
 echo "Starting nginx..."
 sudo nginx

ietf/doc/tests_js.py

@@ -41,7 +41,7 @@ def _fill_in_author_form(form_elt, name, email, affiliation, country):
                     (By.CSS_SELECTOR, result_selector),
                     name
                 ))
-            input.send_keys('\n')  # select the object
+            self.driver.find_element(By.CSS_SELECTOR, result_selector).click()
 
             # After the author is selected, the email select options will be populated.
             # Wait for that, then click on the option corresponding to the requested email.
@@ -94,10 +94,8 @@ def _read_author_form(form_elt):
         # get the "add author" button so we can add blank author forms
         add_author_button = self.driver.find_element(By.ID, 'add-author-button')
         for index, auth in enumerate(authors):
-            self.driver.execute_script("arguments[0].scrollIntoView();", add_author_button)  # FIXME: no idea why this fails:
-            # self.scroll_to_element(add_author_button)  # Can only click if it's in view!
-            self.driver.execute_script("arguments[0].click();", add_author_button)  # FIXME: no idea why this fails:
-            # add_author_button.click()  # Create a new form. Automatically scrolls to it.
+            self.scroll_to_element(add_author_button)  # Can only click if it's in view!
+            add_author_button.click()  # Create a new form. Automatically scrolls to it.
             author_forms = authors_list.find_elements(By.CLASS_NAME, 'author-panel')
             authors_added = index + 1
             self.assertEqual(len(author_forms), authors_added + 1)  # Started with 1 author, hence +1
@@ -119,9 +117,8 @@ def _read_author_form(form_elt):
         self.driver.find_element(By.ID, 'id_basis').send_keys('change testing')
         # Now click the 'submit' button and check that the update was accepted.
         submit_button = self.driver.find_element(By.CSS_SELECTOR, '#content button[type="submit"]')
-        self.driver.execute_script("arguments[0].click();", submit_button)  # FIXME: no idea why this fails:
-        # self.scroll_to_element(submit_button)
-        # submit_button.click()
+        self.scroll_to_element(submit_button)
+        submit_button.click()
         # Wait for redirect to the document_main view
         self.wait.until(
             expected_conditions.url_to_be(

ietf/meeting/tests_js.py

@@ -499,7 +499,7 @@ def test_past_swap_days_buttons(self):
         clicked_index = 1
         # scroll so the button we want to click is just below the navbar, otherwise it may
         # fall beneath the sessions panel
-        navbar = self.driver.find_element_by_class_name('navbar')
+        navbar = self.driver.find_element(By.CSS_SELECTOR, '.navbar')
         self.driver.execute_script(
             'window.scrollBy({top: %s, behavior: "instant"})' % (
                     future_swap_days_buttons[1].location['y'] - navbar.size['height']
@@ -1232,10 +1232,13 @@ def _assert_ietf_tz_correct(meetings, tz):
         self.driver.get(self.absreverse('ietf.meeting.views.upcoming'))
         tz_select_input = self.driver.find_element(By.ID, 'timezone-select')
         tz_select_bottom_input = self.driver.find_element(By.ID, 'timezone-select-bottom')
-        local_tz_link = self.driver.find_element(By.ID, 'local-timezone')
-        utc_tz_link = self.driver.find_element(By.ID, 'utc-timezone')
-        local_tz_bottom_link = self.driver.find_element(By.ID, 'local-timezone-bottom')
-        utc_tz_bottom_link = self.driver.find_element(By.ID, 'utc-timezone-bottom')
+
+        # For things we click, need to click the labels / actually visible items. The actual inputs are hidden
+        # and managed by the JS.
+        local_tz_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="local-timezone"]')
+        utc_tz_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="utc-timezone"]')
+        local_tz_bottom_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="local-timezone-bottom"]')
+        utc_tz_bottom_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="utc-timezone-bottom"]')
 
         # wait for the select box to be updated - look for an arbitrary time zone to be in
         # its options list to detect this
@@ -1245,7 +1248,10 @@ def _assert_ietf_tz_correct(meetings, tz):
                 (By.CSS_SELECTOR, '#timezone-select > option[value="%s"]' % arbitrary_tz)
             )
         )
-
+        tz_selector_clickables = self.driver.find_elements(By.CSS_SELECTOR, ".tz-display .select2")
+        self.assertEqual(len(tz_selector_clickables), 2)
+        (tz_selector_top, tz_selector_bottom) = tz_selector_clickables
+
         arbitrary_tz_bottom_opt = tz_select_bottom_input.find_element(By.CSS_SELECTOR,
             '#timezone-select-bottom > option[value="%s"]' % arbitrary_tz)
 
@@ -1266,8 +1272,7 @@ def _assert_ietf_tz_correct(meetings, tz):
         _assert_ietf_tz_correct(ietf_meetings, local_tz)
 
         # click 'utc' button
-        self.driver.execute_script("arguments[0].click();", utc_tz_link)  # FIXME-LARS: not working:
-        # utc_tz_link.click()
+        utc_tz_link.click()
         self.wait.until(expected_conditions.element_to_be_selected(utc_tz_opt))
         self.assertFalse(local_tz_opt.is_selected())
         self.assertFalse(local_tz_bottom_opt.is_selected())
@@ -1279,8 +1284,7 @@ def _assert_ietf_tz_correct(meetings, tz):
         _assert_ietf_tz_correct(ietf_meetings, 'UTC')
 
         # click back to 'local'
-        self.driver.execute_script("arguments[0].click();", local_tz_link)  # FIXME-LARS: not working:
-        # local_tz_link.click()
+        local_tz_link.click()
         self.wait.until(expected_conditions.element_to_be_selected(local_tz_opt))
         self.assertTrue(local_tz_opt.is_selected())
         self.assertTrue(local_tz_bottom_opt.is_selected())
@@ -1292,7 +1296,12 @@ def _assert_ietf_tz_correct(meetings, tz):
         _assert_ietf_tz_correct(ietf_meetings, local_tz)
 
         # Now select a different item from the select input
-        arbitrary_tz_opt.click()
+        tz_selector_top.click()
+        self.wait.until(
+            expected_conditions.presence_of_element_located(
+                (By.CSS_SELECTOR, 'span.select2-container .select2-results li[id$="America/Halifax"]')
+            )
+        ).click()
         self.wait.until(expected_conditions.element_to_be_selected(arbitrary_tz_opt))
         self.assertFalse(local_tz_opt.is_selected())
         self.assertFalse(local_tz_bottom_opt.is_selected())
@@ -1305,8 +1314,8 @@ def _assert_ietf_tz_correct(meetings, tz):
 
         # Now repeat those tests using the widgets at the bottom of the page
         # click 'utc' button
-        self.driver.execute_script("arguments[0].click();", utc_tz_bottom_link)  # FIXME-LARS: not working:
-        # utc_tz_bottom_link.click()
+        self.scroll_to_element(utc_tz_bottom_link)
+        utc_tz_bottom_link.click()
         self.wait.until(expected_conditions.element_to_be_selected(utc_tz_opt))
         self.assertFalse(local_tz_opt.is_selected())
         self.assertFalse(local_tz_bottom_opt.is_selected())
@@ -1318,8 +1327,8 @@ def _assert_ietf_tz_correct(meetings, tz):
         _assert_ietf_tz_correct(ietf_meetings, 'UTC')
 
         # click back to 'local'
-        self.driver.execute_script("arguments[0].click();", local_tz_bottom_link)  # FIXME-LARS: not working:
-        # local_tz_bottom_link.click()
+        self.scroll_to_element(local_tz_bottom_link)
+        local_tz_bottom_link.click()
         self.wait.until(expected_conditions.element_to_be_selected(local_tz_opt))
         self.assertTrue(local_tz_opt.is_selected())
         self.assertTrue(local_tz_bottom_opt.is_selected())
@@ -1331,7 +1340,13 @@ def _assert_ietf_tz_correct(meetings, tz):
         _assert_ietf_tz_correct(ietf_meetings, local_tz)
 
         # Now select a different item from the select input
-        arbitrary_tz_bottom_opt.click()
+        self.scroll_to_element(tz_selector_bottom)
+        tz_selector_bottom.click()
+        self.wait.until(
+            expected_conditions.presence_of_element_located(
+                (By.CSS_SELECTOR, 'span.select2-container .select2-results li[id$="America/Halifax"]')
+            )
+        ).click()
         self.wait.until(expected_conditions.element_to_be_selected(arbitrary_tz_opt))
         self.assertFalse(local_tz_opt.is_selected())
         self.assertFalse(local_tz_bottom_opt.is_selected())
@@ -1382,6 +1397,7 @@ def test_upcoming_materials_modal(self):
             ),
             'Modal close button not found or not clickable',
         )
+        time.sleep(0.3)  # gross, but the button is clickable while still fading in
         close_modal_button.click()
         self.wait.until(
             expected_conditions.invisibility_of_element(modal_div),

ietf/settings.py

@@ -572,8 +572,6 @@ def skip_unreadable_post(record):
 
 TEST_DIFF_FAILURE_DIR = "/tmp/test/failure/"
 
-TEST_GHOSTDRIVER_LOG_PATH = "ghostdriver.log"
-
 # These are regexes
 TEST_URL_COVERAGE_EXCLUDE = [
     r"^\^admin/",

ietf/utils/jstest.py

@@ -1,18 +1,18 @@
 # Copyright The IETF Trust 2014-2021, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from django.conf import settings
+import os
+
 from django.urls import reverse as urlreverse
 from unittest import skipIf
 
 skip_selenium = False
 skip_message  = ""
 try:
     from selenium import webdriver
-    from selenium.webdriver.chrome.service import Service
-    from selenium.webdriver.chrome.options import Options
+    from selenium.webdriver.firefox.service import Service
+    from selenium.webdriver.firefox.options import Options
     from selenium.webdriver.common.by import By
-    from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 except ImportError as e:
     skip_selenium = True
     skip_message = "Skipping selenium tests: %s" % e
@@ -21,7 +21,7 @@
 from ietf.utils.pipe import pipe
 from ietf.utils.test_runner import IetfLiveServerTestCase
 
-executable_name = 'chromedriver'
+executable_name = 'geckodriver'
 code, out, err = pipe('{} --version'.format(executable_name))
 if code != 0:
     skip_selenium = True
@@ -30,20 +30,11 @@
     print("     "+skip_message)
 
 def start_web_driver():
-    service = Service(executable_path="chromedriver",
-                      log_path=settings.TEST_GHOSTDRIVER_LOG_PATH)
-    service.start()
+    service = Service(log_output=f"{executable_name}.log", service_args=['--log-no-truncate'])
     options = Options()
-    options.add_argument("headless")
-    options.add_argument("disable-extensions")
-    options.add_argument("disable-gpu") # headless needs this
-    options.add_argument("no-sandbox") # docker needs this
-    dc = DesiredCapabilities.CHROME
-    dc["goog:loggingPrefs"] = {"browser": "ALL"}
-    # For selenium 3:
-    return webdriver.Chrome("chromedriver", options=options, desired_capabilities=dc)
-    # For selenium 4:
-    # return webdriver.Chrome(service=service, options=options, desired_capabilities=dc)
+    options.add_argument("--headless")
+    os.environ["MOZ_REMOTE_SETTINGS_DEVTOOLS"] = "1"
+    return webdriver.Firefox(service=service, options=options)
 
 
 def selenium_enabled():

requirements.txt

@@ -65,12 +65,12 @@ types-requests>=2.27.1
 requests-mock>=1.9.3
 rfc2html>=2.0.3
 scout-apm>=2.24.2
-selenium>=3.141.0,<4.0
+selenium>=4.0
 tblib>=1.7.0    # So that the django test runner provides tracebacks
 tlds>=2022042700    # Used to teach bleach about which TLDs currently exist
 tqdm>=4.64.0
 Unidecode>=1.3.4
-urllib3<2        # v2 causes selenium tests to fail with "Timeout value was <object..." error
+urllib3>=2
 weasyprint>=59
 xml2rfc>=3.12.4
 xym>=0.6,<1.0