feat: make the application tzaware

Author: rjsparks

.github/workflows/build.yml

@@ -144,6 +144,8 @@ jobs:
       run: |
         echo "Running checks..."
         ./ietf/manage.py check
+        ./ietf/manage.py migrate || true
+        echo "USE_TZ = True" >> ./ietf/settings_local.py
         ./ietf/manage.py migrate
         echo "Validating migrations..."
         if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then

.github/workflows/ci-run-tests.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - 'main'
+      - 'feat/tzaware'
     paths:
       - 'ietf/**'
       - 'requirements.txt'
@@ -42,6 +43,10 @@ jobs:
       run: |
         echo "Running checks..."
         ./ietf/manage.py check
+        echo "Running migrations with USE_TZ=False..."
+        ./ietf/manage.py migrate || true
+        echo "USE_TZ = True" >> ./ietf/settings_local.py
+        echo "Running migrations with USE_TZ=True..."
         ./ietf/manage.py migrate
         echo "Validating migrations..."
         if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then

bin/add-old-drafts-from-archive.py

@@ -16,6 +16,7 @@
 from django.core.validators import validate_email, ValidationError
 from ietf.utils.draft import PlaintextDraft
 from ietf.submit.utils import update_authors
+from ietf.utils.timezone import date_today
 
 import debug                            # pyflakes:ignore
 
@@ -66,9 +67,9 @@
                     print name, rev, "Can't parse", p,":",e
                     continue
             if draft.errors and draft.errors.keys()!=['draftname',]:
-                print "Errors - could not process", name, rev, datetime.datetime.fromtimestamp(p.stat().st_mtime), draft.errors, draft.get_title().encode('utf8')
+                print "Errors - could not process", name, rev, datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc), draft.errors, draft.get_title().encode('utf8')
             else:
-                time = datetime.datetime.fromtimestamp(p.stat().st_mtime)
+                time = datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc)
                 if not doc:
                     doc = Document.objects.create(name=name,
                                                   time=time,
@@ -140,7 +141,7 @@ def turn_into_unicode(s):
                         doc = doc,
                         rev = rev,
                         by = system,
-                        desc = "Revision added from id-archive on %s by %s"%(datetime.date.today(),sys.argv[0]),
+                        desc = "Revision added from id-archive on %s by %s"%(date_today(),sys.argv[0]),
                         time=time,
                 )
                 events.append(e)

bin/check-copyright

@@ -162,7 +162,7 @@ def get_first_commit(path):
             else:
                 pass
     except OSError:
-        rev, who, when = None, None, datetime.datetime.now()
+        rev, who, when = None, None, datetime.datetime.now(datetime.timezone.utc)
     return { path: { 'rev': rev, 'who': who, 'date': when.strftime('%Y-%m-%d %H:%M:%S'), }, }
 
 

dev/deploy-to-container/start.sh

@@ -9,7 +9,31 @@ chmod +x ./app-create-dirs.sh
 ./app-create-dirs.sh
 echo "Running Datatracker checks..."
 ./ietf/manage.py check
-echo "Running Datatracker migrations..."
-./ietf/manage.py migrate
+
+# Migrate, adjusting to what the current state of the underlying database might be:
+WORKSPACEDIR=.
+if ietf/manage.py showmigrations | grep "\[ \] 0003_pause_to_change_use_tz"; then
+    if grep "USE_TZ" $WORKSPACEDIR/ietf/settings_local.py; then
+        cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = False/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+    else
+        echo "USE_TZ = False" >> $WORKSPACEDIR/ietf/settings_local.py
+    fi
+    echo "Running Datatracker migrations with USE_TZ = False..."
+    # This is expected to exit non-zero at the pause
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true
+    cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+    echo "Running Datatracker migrations with USE_TZ = True..."
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
+
+else
+    if grep "USE_TZ" $WORKSPACEDIR/ietf/settings_local.py; then
+        cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+    else
+        echo "USE_TZ = True" >> $WORKSPACEDIR/ietf/settings_local.py
+    echo "Running Datatracker migrations..."
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
+    fi
+fi
+
 echo "Starting Datatracker..."
 ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local

dev/diff/prepare.sh

@@ -12,5 +12,24 @@ yarn legacy:build
 echo "Creating data directories..."
 chmod +x ./docker/scripts/app-create-dirs.sh
 ./docker/scripts/app-create-dirs.sh
+
 ./ietf/manage.py check
-./ietf/manage.py migrate
+if ./ietf/manage.py showmigrations | grep "\[ \] 0003_pause_to_change_use_tz"; then
+    if grep "USE_TZ" ./ietf/settings_local.py; then
+        cat ./ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = False/' > /tmp/settings_local.py && mv /tmp/settings_local.py ./ietf/settings_local.py
+    else
+        echo "USE_TZ = False" >> ./ietf/settings_local.py
+    fi
+    # This is expected to exit non-zero at the pause
+    /usr/local/bin/python ./ietf/manage.py migrate  || true
+    cat ./ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py ./ietf/settings_local.py
+    /usr/local/bin/python ./ietf/manage.py migrate
+
+else
+    if grep "USE_TZ" ./ietf/settings_local.py; then
+        cat ./ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py ./ietf/settings_local.py
+    else
+        echo "USE_TZ = True" >> ./ietf/settings_local.py
+    /usr/local/bin/python ./ietf/manage.py migrate
+    fi
+fi

docker/README.md

@@ -27,9 +27,11 @@
 This project includes a devcontainer configuration which automates the setup of the development environment with all the required dependencies.
 
 ### Initial Setup
-        
+
 1. Launch [VS Code](https://code.visualstudio.com/)
-2. Under the **Extensions** tab, ensure you have the **Remote - Containers** ([ms-vscode-remote.remote-containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)) extension installed. On Windows, you also need the **Remote - WSL** ([ms-vscode-remote.remote-wsl](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-wsl)) extension to take advantage of the WSL 2 *(Windows Subsystem for Linux)* native integration.
+2. Under the **Extensions** tab, ensure you have the **Remote - Containers** ([ms-vscode-remote.remote-containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)) extension installed.
+  * On Linux, note that the Snap installation of VS Code is [incompatible with this plugin](https://code.visualstudio.com/docs/devcontainers/containers#_system-requirements:~:text=snap%20package%20is%20not%20supported). 
+  * On Windows, you also need the **Remote - WSL** ([ms-vscode-remote.remote-wsl](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-wsl)) extension to take advantage of the WSL 2 *(Windows Subsystem for Linux)* native integration.
 2. Open the top-level directory of the datatracker code you fetched above.
 3. A prompt inviting you to reopen the project in containers will appear in the bottom-right corner. Click the **Reopen in Container** button. If you missed the prompt, you can press `F1`, start typing `reopen in container` task and launch it.
 4. VS Code will relaunch in the dev environment and create the containers automatically.
@@ -44,8 +46,8 @@ You can also open the datatracker project folder and click the **Reopen in conta
 ### Usage
 
 - Under the **Run and Debug** tab, you can run the server with the debugger attached using **Run Server** (F5). Once the server is ready to accept connections, you'll be prompted to open in a browser. You can also open [http://localhost:8000](http://localhost:8000) in a browser.
-    
-    > An alternate profile **Run Server with Debug Toolbar** is also available from the dropdown menu, which displays various tools 
+
+    > An alternate profile **Run Server with Debug Toolbar** is also available from the dropdown menu, which displays various tools
     on top of the webpage. However, note that this configuration has a significant performance impact.
 
     To add a **Breakpoint**, simply click to the left of the line gutter you wish to stop at. You can also add **Conditional Breakpoints** and **Logpoint** by right-clicking at the same location.
@@ -88,7 +90,7 @@ You can also open the datatracker project folder and click the **Reopen in conta
     cd docker
     ./run
     ```
-    
+
     > Note that you can pass the `-r` flag to `./run` to force a rebuild of the containers. This is useful if you switched branches and that the existing containers still contain configurations from the old branch. You should also use this if you don't regularly keep up with main and your containers reflect a much older version of the branch.
 
     On Windows *(using Powershell)*:
@@ -157,7 +159,7 @@ cd docker
 On Windows:
 ```sh
 docker compose down -v --rmi all
-docker image prune 
+docker image prune
 ```
 
 ### Accessing MariaDB Port

docker/scripts/app-init.sh

@@ -100,7 +100,28 @@ echo "Starting memcached..."
 
 echo "Running initial checks..."
 /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local
-# /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
+
+# Migrate, adjusting to what the current state of the underlying database might be:
+
+if ietf/manage.py showmigrations | grep "\[ \] 0003_pause_to_change_use_tz"; then
+    if grep "USE_TZ" $WORKSPACEDIR/ietf/settings_local.py; then
+        cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = False/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+    else
+        echo "USE_TZ = False" >> $WORKSPACEDIR/ietf/settings_local.py
+    fi
+    # This is expected to exit non-zero at the pause
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true
+    cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
+
+else
+    if grep "USE_TZ" $WORKSPACEDIR/ietf/settings_local.py; then
+        cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+    else
+        echo "USE_TZ = True" >> $WORKSPACEDIR/ietf/settings_local.py
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
+    fi
+fi
 
 echo "-----------------------------------------------------------------"
 echo "Done!"

ietf/api/management/commands/makeresources.py

@@ -3,7 +3,6 @@
 
 
 import os
-import datetime
 import collections
 import io
 
@@ -14,6 +13,7 @@
 from django.core.management.base import AppCommand
 from django.db import models
 from django.template import Template, Context
+from django.utils import timezone
 
 from tastypie.resources import ModelResource
 
@@ -89,7 +89,7 @@ def handle_app_config(self, app, **options):
                     info = dict(
                         app=app.name,
                         app_label=app.label,
-                        date=datetime.datetime.now()
+                        date=timezone.now()
                     )
                     new_models = {}
                     for model, rclass_name in missing_resources:

ietf/api/tests.py

@@ -32,6 +32,7 @@
 from ietf.person.models import PersonalApiKey
 from ietf.stats.models import MeetingRegistration
 from ietf.utils.mail import outbox, get_payload_text
+from ietf.utils.models import DumpInfo
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized
 
 OMITTED_APPS = (
@@ -508,10 +509,17 @@ def test_api_new_meeting_registration(self):
         self.assertEqual(set(missing_fields), set(drop_fields))
 
     def test_api_version(self):
+        DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=timezone.utc), host='testapi.example.com',tz='UTC')
         url = urlreverse('ietf.api.views.version')
         r = self.client.get(url)
         data = r.json()
         self.assertEqual(data['version'], ietf.__version__+ietf.__patch__)
+        self.assertEqual(data['dumptime'], "2022-08-31 07:10:01 +0000")
+        DumpInfo.objects.update(tz='PST8PDT')
+        r = self.client.get(url)
+        data = r.json()        
+        self.assertEqual(data['dumptime'], "2022-08-31 07:10:01 -0700")
+
 
     def test_api_appauth(self):
         url = urlreverse('ietf.api.views.app_auth')