Merge branch 'feat/tzaware' into jennifer/main-for-tzaware-merge

# Conflicts:
#	ietf/name/fixtures/names.json
#	ietf/utils/management/commands/send_gdpr_consent_request.py

Author: jennifer-richards

.github/workflows/build.yml

@@ -139,6 +139,8 @@ jobs:
       run: |
         echo "Running checks..."
         ./ietf/manage.py check
+        ./ietf/manage.py migrate || true
+        echo "USE_TZ = True" >> ./ietf/settings_local.py
         ./ietf/manage.py migrate
         echo "Validating migrations..."
         if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then

bin/add-old-drafts-from-archive.py

@@ -16,6 +16,7 @@
 from django.core.validators import validate_email, ValidationError
 from ietf.utils.draft import PlaintextDraft
 from ietf.submit.utils import update_authors
+from ietf.utils.timezone import date_today
 
 import debug                            # pyflakes:ignore
 
@@ -66,9 +67,9 @@
                     print name, rev, "Can't parse", p,":",e
                     continue
             if draft.errors and draft.errors.keys()!=['draftname',]:
-                print "Errors - could not process", name, rev, datetime.datetime.fromtimestamp(p.stat().st_mtime), draft.errors, draft.get_title().encode('utf8')
+                print "Errors - could not process", name, rev, datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc), draft.errors, draft.get_title().encode('utf8')
             else:
-                time = datetime.datetime.fromtimestamp(p.stat().st_mtime)
+                time = datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc)
                 if not doc:
                     doc = Document.objects.create(name=name,
                                                   time=time,
@@ -140,7 +141,7 @@ def turn_into_unicode(s):
                         doc = doc,
                         rev = rev,
                         by = system,
-                        desc = "Revision added from id-archive on %s by %s"%(datetime.date.today(),sys.argv[0]),
+                        desc = "Revision added from id-archive on %s by %s"%(date_today(),sys.argv[0]),
                         time=time,
                 )
                 events.append(e)

bin/check-copyright

@@ -162,7 +162,7 @@ def get_first_commit(path):
             else:
                 pass
     except OSError:
-        rev, who, when = None, None, datetime.datetime.now()
+        rev, who, when = None, None, datetime.datetime.now(datetime.timezone.utc)
     return { path: { 'rev': rev, 'who': who, 'date': when.strftime('%Y-%m-%d %H:%M:%S'), }, }
 
 

dev/diff/prepare.sh

@@ -12,5 +12,24 @@ yarn legacy:build
 echo "Creating data directories..."
 chmod +x ./docker/scripts/app-create-dirs.sh
 ./docker/scripts/app-create-dirs.sh
+
 ./ietf/manage.py check
-./ietf/manage.py migrate
+if ./ietf/manage.py showmigrations | grep "\[ \] 0003_pause_to_change_use_tz"; then
+    if grep "USE_TZ" ./ietf/settings_local.py; then
+        cat ./ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = False/' > /tmp/settings_local.py && mv /tmp/settings_local.py ./ietf/settings_local.py
+    else
+        echo "USE_TZ = False" >> ./ietf/settings_local.py
+    fi
+    # This is expected to exit non-zero at the pause
+    /usr/local/bin/python ./ietf/manage.py migrate  || true
+    cat ./ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py ./ietf/settings_local.py
+    /usr/local/bin/python ./ietf/manage.py migrate
+
+else
+    if grep "USE_TZ" ./ietf/settings_local.py; then
+        cat ./ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py ./ietf/settings_local.py
+    else
+        echo "USE_TZ = True" >> ./ietf/settings_local.py
+    /usr/local/bin/python ./ietf/manage.py migrate
+    fi
+fi

docker/scripts/app-init.sh

@@ -100,7 +100,28 @@ echo "Starting memcached..."
 
 echo "Running initial checks..."
 /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local
-# /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
+
+# Migrate, adjusting to what the current state of the underlying database might be:
+
+if ietf/manage.py showmigrations | grep "\[ \] 0003_pause_to_change_use_tz"; then
+    if grep "USE_TZ" $WORKSPACEDIR/ietf/settings_local.py; then
+        cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = False/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+    else
+        echo "USE_TZ = False" >> $WORKSPACEDIR/ietf/settings_local.py
+    fi
+    # This is expected to exit non-zero at the pause
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true
+    cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
+
+else
+    if grep "USE_TZ" $WORKSPACEDIR/ietf/settings_local.py; then
+        cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+    else
+        echo "USE_TZ = True" >> $WORKSPACEDIR/ietf/settings_local.py
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
+    fi
+fi
 
 echo "-----------------------------------------------------------------"
 echo "Done!"

ietf/api/management/commands/makeresources.py

@@ -3,7 +3,6 @@
 
 
 import os
-import datetime
 import collections
 import io
 
@@ -14,6 +13,7 @@
 from django.core.management.base import AppCommand
 from django.db import models
 from django.template import Template, Context
+from django.utils import timezone
 
 from tastypie.resources import ModelResource
 
@@ -89,7 +89,7 @@ def handle_app_config(self, app, **options):
                     info = dict(
                         app=app.name,
                         app_label=app.label,
-                        date=datetime.datetime.now()
+                        date=timezone.now()
                     )
                     new_models = {}
                     for model, rclass_name in missing_resources:

ietf/api/tests.py

@@ -32,6 +32,7 @@
 from ietf.person.models import PersonalApiKey
 from ietf.stats.models import MeetingRegistration
 from ietf.utils.mail import outbox, get_payload_text
+from ietf.utils.models import DumpInfo
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized
 
 OMITTED_APPS = (
@@ -508,10 +509,17 @@ def test_api_new_meeting_registration(self):
         self.assertEqual(set(missing_fields), set(drop_fields))
 
     def test_api_version(self):
+        DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=timezone.utc), host='testapi.example.com',tz='UTC')
         url = urlreverse('ietf.api.views.version')
         r = self.client.get(url)
         data = r.json()
         self.assertEqual(data['version'], ietf.__version__+ietf.__patch__)
+        self.assertEqual(data['dumptime'], "2022-08-31 07:10:01 +0000")
+        DumpInfo.objects.update(tz='PST8PDT')
+        r = self.client.get(url)
+        data = r.json()        
+        self.assertEqual(data['dumptime'], "2022-08-31 07:10:01 -0700")
+
 
     def test_api_appauth(self):
         url = urlreverse('ietf.api.views.app_auth')

ietf/api/views.py

@@ -203,8 +203,13 @@ def err(code, text):
 
 
 def version(request):
+    dumpdate = None
     dumpinfo = DumpInfo.objects.order_by('-date').first()
-    dumptime = pytz.timezone(dumpinfo.tz).localize(dumpinfo.date).strftime('%Y-%m-%d %H:%M:%S %z') if dumpinfo else None
+    if dumpinfo:
+        dumpdate = dumpinfo.date
+        if dumpinfo.tz != "UTC":
+            dumpdate = pytz.timezone(dumpinfo.tz).localize(dumpinfo.date.replace(tzinfo=None))
+    dumptime = dumpdate.strftime('%Y-%m-%d %H:%M:%S %z') if dumpinfo else None
     return HttpResponse(
             json.dumps({
                         'version': ietf.__version__+ietf.__patch__,

ietf/bin/create-charter-newrevisiondocevents

@@ -28,7 +28,7 @@ def warn(string):
 # ------------------------------------------------------------------------------
 
 import re
-from datetime import datetime as Datetime
+import datetime
 
 import django
 django.setup()
@@ -44,7 +44,7 @@ system_entity = Person.objects.get(name="(System)")
 charterdir = Path(settings.CHARTER_PATH)
 for file in charterdir.files("charter-ietf-*.txt"):
     fname = file.name
-    ftime = Datetime.fromtimestamp(file.mtime)
+    ftime = datetime.datetime.fromtimestamp(file.mtime, datetime.timezone.utc)
     match = re.search("^(?P<name>[a-z0-9-]+)-(?P<rev>\d\d-\d\d)\.txt$", fname)
     if match:
         name = match.group("name")

ietf/bin/iana-changes-updates

@@ -18,6 +18,7 @@ django.setup()
 
 from django.conf import settings
 from optparse import OptionParser
+from zoneinfo import ZoneInfo
 
 parser = OptionParser()
 parser.add_option("-f", "--from", dest="start",
@@ -38,21 +39,30 @@ CLOCK_SKEW_COMPENSATION = 5 # seconds
 MAX_INTERVAL_ACCEPTED_BY_IANA = datetime.timedelta(hours=23)
 
 
+local_tzinfo = ZoneInfo(settings.TIME_ZONE)
 start = datetime.datetime.now() - datetime.timedelta(hours=23) + datetime.timedelta(seconds=CLOCK_SKEW_COMPENSATION)
 if options.start:
     start = datetime.datetime.strptime(options.start, "%Y-%m-%d %H:%M:%S")
+start = start.replace(tzinfo=local_tzinfo).astimezone(datetime.timezone.utc)
 
 end = start + datetime.timedelta(hours=23)
 if options.end:
-    end = datetime.datetime.strptime(options.end, "%Y-%m-%d %H:%M:%S")
+    end = datetime.datetime.strptime(options.end, "%Y-%m-%d %H:%M:%S").replace(tzinfo=local_tzinfo)
+end = end.astimezone(datetime.timezone.utc)
 
 syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER)
 
 # ----------------------------------------------------------------------
 
 from ietf.sync.iana import fetch_changes_json, parse_changes_json, update_history_with_changes
 
-syslog.syslog("Updating history log with new changes from IANA from %s, period %s - %s" % (settings.IANA_SYNC_CHANGES_URL, start, end))
+syslog.syslog(
+    "Updating history log with new changes from IANA from %s, period %s - %s" % (
+        settings.IANA_SYNC_CHANGES_URL,
+        start.astimezone(local_tzinfo),
+        end.astimezone(local_tzinfo),
+    )
+)
 
 t = start
 while t < end: