fix: fix failing tests and eliminate naive datetime warnings (ietf-tools#4402)

* test: fix timestamp construction in several doc tests

* refactor: rename date2datetime to datetime_from_date and clarify code

* chore: helper to get tzinfo for PRODUCTION_TIMEZONE

* fix: fix timezone handling in make_last_call()

* test: fix datetime generation in doc.tests_charter

* refactor: remove PRODUCTION_TIMEZONE setting

Replaces the PRODUCTION_TIMEZONE setting with a constant,
DEADLINE_TZINFO, in ietf.utils.timezone.

* test: be more careful about timezone in tests_charter.py

* test: be more careful about timezone in doc/tests.py

* fix: fix timezone handling affecting doc.tests_draft

* fix: fix timezone handling affecting tests_irsg_ballot.py

* fix: fix timezone handling affecting tests_review.py

* fix: fix timezone handling affecting last ietf.doc tests

* fix: fix timezone handling affecting last ietf.group tests

* fix: fix timezone handling affecting ietf.iesg tests

* fix: handle timezones in get_8989_eligibility_querysets

* fix: handle timezones affecting ietfauth tests

* fix: return tz-aware datetime from utc_from_string

* fix: specify timezone for constants in ipr_rfc_number()

* fix: specify tz for ipr deadlines

* fix: handle timezones affecting liaisons tests

* fix: treat leap day in get_8989_eligibility_querysets()

Manual cherry-pick of 248d647

* test: treat leap day properly in nomcom tests

* fix: fix timezone handling affecting nomcom tests

* test: fix timezone handling in review tests

* fix: fix timezone handling affecting secr.meetings tests

* fix: handle both pytz and zoneinfo timezones in ietf.utils.timezone

* fix: fix timezone handling affecting secr.proceedings tests

* refactor: use make_aware() helper in secr.meetings tests

* test: fix timezone handling in secr.telechat tests

* fix: fix timezone handling affecting stats tests

* fix: eliminate tz-naive helpers affecting sync email parsing

* fix: include timezone data when serializing DeletedEvent data

* fix: fix timezone handling affecting sync tests

* style: remove unused import

Author: jennifer-richards

ietf/doc/expire.py

@@ -18,6 +18,7 @@
 from ietf.meeting.models import Meeting
 from ietf.doc.utils import add_state_change_event, update_action_holders
 from ietf.mailtrigger.utils import gather_address_lists
+from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO
 
 
 nonexpirable_states: Optional[List[State]] = None
@@ -53,13 +54,13 @@ def expirable_drafts(queryset=None):
 
 
 def get_soon_to_expire_drafts(days_of_warning):
-    start_date = datetime.date.today() - datetime.timedelta(1)
+    start_date = datetime_today(DEADLINE_TZINFO) - datetime.timedelta(1)
     end_date = start_date + datetime.timedelta(days_of_warning)
 
     return expirable_drafts().filter(expires__gte=start_date, expires__lt=end_date)
 
 def get_expired_drafts():
-    return expirable_drafts().filter(expires__lt=datetime.date.today() + datetime.timedelta(1))
+    return expirable_drafts().filter(expires__lt=datetime_today(DEADLINE_TZINFO) + datetime.timedelta(1))
 
 def in_draft_expire_freeze(when=None):
     if when == None:

ietf/doc/migrations/0046_tzaware_deletedevents.py

@@ -0,0 +1,61 @@
+# Generated by Django 2.2.28 on 2022-08-31 20:26
+
+import datetime
+import json
+
+from zoneinfo import ZoneInfo
+
+from django.db import migrations
+
+
+TZ_BEFORE = ZoneInfo('PST8PDT')
+
+
+def forward(apps, schema_editor):
+    DeletedEvent = apps.get_model('doc', 'DeletedEvent')
+    for deleted_event in DeletedEvent.objects.all():
+        fields = json.loads(deleted_event.json)
+        replacements = {}
+        for k, v in fields.items():
+            if isinstance(v, str):
+                try:
+                    dt = datetime.datetime.strptime(v, '%Y-%m-%d %H:%M:%S')
+                except:
+                    pass
+                else:
+                    replacements[k] = dt.replace(tzinfo=TZ_BEFORE).astimezone(datetime.timezone.utc).isoformat()
+        if len(replacements) > 0:
+            fields.update(replacements)
+            deleted_event.json = json.dumps(fields)
+            deleted_event.save()
+
+
+def reverse(apps, schema_editor):
+    DeletedEvent = apps.get_model('doc', 'DeletedEvent')
+    for deleted_event in DeletedEvent.objects.all():
+        fields = json.loads(deleted_event.json)
+        replacements = {}
+        for k, v in fields.items():
+            if isinstance(v, str) and 'T' in v:
+                try:
+                    dt = datetime.datetime.fromisoformat(v)
+                except:
+                    pass
+                else:
+                    replacements[k] = dt.astimezone(TZ_BEFORE).replace(tzinfo=None).strftime('%Y-%m-%d %H:%M:%S')
+        if len(replacements) > 0:
+            fields.update(replacements)
+            deleted_event.json = json.dumps(fields)
+            deleted_event.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0045_use_timezone_now_for_doc_models'),
+        ('utils', '0003_pause_to_change_use_tz'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]

ietf/doc/tests.py

@@ -18,6 +18,7 @@
 from urllib.parse import urlparse, parse_qs
 from tempfile import NamedTemporaryFile
 from collections import defaultdict
+from zoneinfo import ZoneInfo
 
 from django.core.management import call_command
 from django.urls import reverse as urlreverse
@@ -57,6 +58,8 @@
 from ietf.utils.test_utils import login_testing_unauthorized, unicontent, reload_db_objects
 from ietf.utils.test_utils import TestCase
 from ietf.utils.text import normalize_text
+from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO
+
 
 class SearchTests(TestCase):
     def test_search(self):
@@ -1428,6 +1431,8 @@ def _run_test(username=None, expect_buttons=False):
 
     def test_draft_group_link(self):
         """Link to group 'about' page should have correct format"""
+        event_datetime = datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo('America/Los_Angeles'))
+
         for group_type_id in ['wg', 'rg', 'ag']:
             group = GroupFactory(type_id=group_type_id)
             draft = WgDraftFactory(name='draft-document-%s' % group_type_id, group=group)
@@ -1436,7 +1441,7 @@ def test_draft_group_link(self):
             self.assert_correct_wg_group_link(r, group)
 
             rfc = WgRfcFactory(name='draft-rfc-document-%s' % group_type_id, group=group)
-            DocEventFactory.create(doc=rfc, type='published_rfc', time = '2010-10-10')
+            DocEventFactory.create(doc=rfc, type='published_rfc', time=event_datetime)
             # get the rfc name to avoid a redirect
             rfc_name = rfc.docalias.filter(name__startswith='rfc').first().name
             r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=rfc_name)))
@@ -1451,7 +1456,7 @@ def test_draft_group_link(self):
             self.assert_correct_non_wg_group_link(r, group)
 
             rfc = WgRfcFactory(name='draft-rfc-document-%s' % group_type_id, group=group)
-            DocEventFactory.create(doc=rfc, type='published_rfc', time = '2010-10-10')
+            DocEventFactory.create(doc=rfc, type='published_rfc', time=event_datetime)
             # get the rfc name to avoid a redirect
             rfc_name = rfc.docalias.filter(name__startswith='rfc').first().name
             r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=rfc_name)))
@@ -1837,7 +1842,7 @@ def test_last_call_feed(self):
             desc="Last call\x0b",  # include a control character to be sure it does not break anything
             type="sent_last_call",
             by=Person.objects.get(user__username="secretary"),
-            expires=datetime.date.today() + datetime.timedelta(days=7))
+            expires=datetime_today(DEADLINE_TZINFO) + datetime.timedelta(days=7))
 
         r = self.client.get("/feed/last-call/")
         self.assertEqual(r.status_code, 200)
@@ -1885,10 +1890,14 @@ def test_document_bibtex(self):
                   #other_aliases = ['rfc6020',],
                   states = [('draft','rfc'),('draft-iesg','pub')],
                   std_level_id = 'ps',
-                  time = datetime.datetime(2010,10,10),
+                  time = datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo('America/Los_Angeles')),
               )
         num = rfc.rfc_number()
-        DocEventFactory.create(doc=rfc, type='published_rfc', time = '2010-10-10')
+        DocEventFactory.create(
+            doc=rfc,
+            type='published_rfc',
+            time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo('America/Los_Angeles')),
+        )
         #
         url = urlreverse('ietf.doc.views_doc.document_bibtex', kwargs=dict(name=rfc.name))
         r = self.client.get(url)
@@ -1906,10 +1915,14 @@ def test_document_bibtex(self):
                   stream_id =       'ise',
                   states =          [('draft','rfc'),('draft-iesg','pub')],
                   std_level_id =    'inf',
-                  time =            datetime.datetime(1990,0o4,0o1),
+                  time =            datetime.datetime(1990, 4, 1, tzinfo=ZoneInfo('America/Los_Angeles')),
               )
         num = april1.rfc_number()
-        DocEventFactory.create(doc=april1, type='published_rfc', time = '1990-04-01')
+        DocEventFactory.create(
+            doc=april1,
+            type='published_rfc',
+            time=datetime.datetime(1990, 4, 1, tzinfo=ZoneInfo('America/Los_Angeles')),
+        )
         #
         url = urlreverse('ietf.doc.views_doc.document_bibtex', kwargs=dict(name=april1.name))
         r = self.client.get(url)
@@ -2044,7 +2057,9 @@ def tearDown(self):
        super().tearDown()
 
    def testManagementCommand(self):
-       a_month_ago = timezone.now() - datetime.timedelta(30)
+       tz = ZoneInfo('America/Los_Angeles')
+       a_month_ago = (timezone.now() - datetime.timedelta(30)).astimezone(tz)
+       a_month_ago = a_month_ago.replace(hour=0, minute=0, second=0, microsecond=0)
        ad = RoleFactory(name_id='ad', group__type_id='area', group__state_id='active').person
        shepherd = PersonFactory()
        author1 = PersonFactory()
@@ -2059,9 +2074,9 @@ def testManagementCommand(self):
        doc1 = IndividualDraftFactory(authors=[author1], shepherd=shepherd.email(), ad=ad)
        doc2 = WgDraftFactory(name='draft-ietf-mars-test', group__acronym='mars', authors=[author2], ad=ad)
        doc3 = WgRfcFactory.create(name='draft-ietf-mars-finished', group__acronym='mars', authors=[author3], ad=ad, std_level_id='ps', states=[('draft','rfc'),('draft-iesg','pub')], time=a_month_ago)
-       DocEventFactory.create(doc=doc3, type='published_rfc', time=a_month_ago.strftime("%Y-%m-%d"))
-       doc4 = WgRfcFactory.create(authors=[author4,author5], ad=ad, std_level_id='ps', states=[('draft','rfc'),('draft-iesg','pub')], time=datetime.datetime(2010,10,10))
-       DocEventFactory.create(doc=doc4, type='published_rfc', time = '2010-10-10')
+       DocEventFactory.create(doc=doc3, type='published_rfc', time=a_month_ago)
+       doc4 = WgRfcFactory.create(authors=[author4,author5], ad=ad, std_level_id='ps', states=[('draft','rfc'),('draft-iesg','pub')], time=datetime.datetime(2010,10,10, tzinfo=tz))
+       DocEventFactory.create(doc=doc4, type='published_rfc', time=datetime.datetime(2010, 10, 10, tzinfo=tz))
        doc5 = IndividualDraftFactory(authors=[author6])
 
        args = [ ]

ietf/doc/tests_charter.py

@@ -26,6 +26,8 @@
 from ietf.utils.test_utils import TestCase
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import login_testing_unauthorized
+from ietf.utils.timezone import datetime_today, date_today, DEADLINE_TZINFO
+
 
 class ViewCharterTests(TestCase):
     def test_view_revisions(self):
@@ -402,7 +404,7 @@ def test_no_returning_item_for_different_ballot(self):
 
         # Make it so that the charter has been through internal review, and passed its external review
         # ballot on a previous telechat 
-        last_week = datetime.date.today()-datetime.timedelta(days=7)
+        last_week = datetime_today(DEADLINE_TZINFO) - datetime.timedelta(days=7)
         BallotDocEvent.objects.create(type='created_ballot',by=login,doc=charter, rev=charter.rev,
                                       ballot_type=BallotType.objects.get(doc_type=charter.type,slug='r-extrev'),
                                       time=last_week)
@@ -746,7 +748,7 @@ def test_approve(self):
 
         charter.set_state(State.objects.get(used=True, type="charter", slug="iesgrev"))
 
-        due_date = datetime.date.today() + datetime.timedelta(days=180)
+        due_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=180)
         m1 = GroupMilestone.objects.create(group=group,
                                            state_id="active",
                                            desc="Has been copied",
@@ -826,7 +828,7 @@ def test_charter_with_milestones(self):
         m = GroupMilestone.objects.create(group=charter.group,
                                           state_id="active",
                                           desc="Test milestone",
-                                          due=datetime.date.today(),
+                                          due=date_today(DEADLINE_TZINFO),
                                           resolved="")
 
         url = urlreverse('ietf.doc.views_charter.charter_with_milestones_txt', kwargs=dict(name=charter.name, rev=charter.rev))

ietf/doc/tests_draft.py

@@ -34,6 +34,7 @@
 from ietf.utils.test_utils import login_testing_unauthorized
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import TestCase
+from ietf.utils.timezone import date_today, datetime_from_date
 
 
 class ChangeStateTests(TestCase):
@@ -402,11 +403,11 @@ def test_edit_telechat_date(self):
 
         # change to a telechat that should cause returning item to be auto-detected
         # First, make it appear that the previous telechat has already passed
-        telechat_event.telechat_date = datetime.date.today()-datetime.timedelta(days=7)
+        telechat_event.telechat_date = date_today() - datetime.timedelta(days=7)
         telechat_event.save()
         ad = Person.objects.get(user__username="ad")
         ballot = create_ballot_if_not_open(None, draft, ad, 'approve')
-        ballot.time = telechat_event.telechat_date
+        ballot.time = datetime_from_date(telechat_event.telechat_date)
         ballot.save()
 
         r = self.client.post(url, data)
@@ -429,7 +430,7 @@ def test_edit_telechat_date(self):
         self.assertTrue("Telechat update" in outbox[-1]['Subject'])
 
         # Put it on an agenda that's very soon from now
-        next_week = datetime.date.today()+datetime.timedelta(days=7)
+        next_week = date_today() + datetime.timedelta(days=7)
         td =  TelechatDate.objects.active()[0]
         td.date = next_week
         td.save()

ietf/doc/tests_irsg_ballot.py

@@ -19,6 +19,7 @@
 from ietf.person.utils import get_active_irsg, get_active_ads
 from ietf.group.factories import RoleFactory
 from ietf.person.models import Person
+from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO
 
 
 class IssueIRSGBallotTests(TestCase):
@@ -254,7 +255,7 @@ def test_edit_ballot_position_permissions(self):
         irsgmember = get_active_irsg()[0]
         secr = RoleFactory(group__acronym='secretariat',name_id='secr')
         wg_ballot = create_ballot_if_not_open(None, wg_draft, ad.person, 'approve')
-        due = datetime.date.today()+datetime.timedelta(days=14)
+        due = datetime_today(DEADLINE_TZINFO) + datetime.timedelta(days=14)
         rg_ballot = create_ballot_if_not_open(None, rg_draft, secr.person, 'irsg-approve', due)
 
         url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=wg_draft.name, ballot_id=wg_ballot.pk))

ietf/doc/tests_review.py

@@ -10,7 +10,6 @@
 from mock import patch
 from requests import Response
 
-
 from django.apps import apps
 from django.urls import reverse as urlreverse
 from django.conf import settings
@@ -39,6 +38,7 @@
 from ietf.utils.test_utils import login_testing_unauthorized, reload_db_objects
 from ietf.utils.test_utils import TestCase
 from ietf.utils.text import strip_prefix, xslugify
+from ietf.utils.timezone import DEADLINE_TZINFO
 from django.utils.html import escape
 
 class ReviewTests(TestCase):
@@ -734,15 +734,15 @@ def test_complete_review_enter_content_by_secretary(self):
         # The secretary is allowed to set a custom completion date (#2590)
         assignment = reload_db_objects(assignment)
         self.assertEqual(assignment.state_id, "completed")
-        self.assertEqual(assignment.completed_on, datetime.datetime(2012, 12, 24, 12, 13, 14))
+        self.assertEqual(assignment.completed_on, datetime.datetime(2012, 12, 24, 12, 13, 14, tzinfo=DEADLINE_TZINFO))
 
         # There should be two events:
         # - the event logging when the change when it was entered, i.e. very close to now.
         # - the completion of the review, set to the provided date/time
         events = ReviewAssignmentDocEvent.objects.filter(doc=assignment.review_request.doc).order_by('-time')
         event0_time_diff = timezone.now() - events[0].time
         self.assertLess(event0_time_diff, datetime.timedelta(seconds=10))
-        self.assertEqual(events[1].time, datetime.datetime(2012, 12, 24, 12, 13, 14))
+        self.assertEqual(events[1].time, datetime.datetime(2012, 12, 24, 12, 13, 14, tzinfo=DEADLINE_TZINFO))
 
         with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f:
             self.assertEqual(f.read(), "This is a review\nwith two lines")

ietf/doc/tests_utils.py

@@ -143,7 +143,7 @@ def test_update_action_holders_resets_age(self):
         doc = self.doc_in_iesg_state('pub-req')
         doc.action_holders.set([self.ad])
         dah = doc.documentactionholder_set.get(person=self.ad)
-        dah.time_added = datetime.datetime(2020, 1, 1)  # arbitrary date in the past
+        dah.time_added = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)  # arbitrary date in the past
         dah.save()
 
         self.assertNotEqual(doc.documentactionholder_set.get(person=self.ad).time_added.date(), datetime.date.today())

ietf/doc/utils.py

@@ -39,6 +39,7 @@
 from ietf.utils import draft, log
 from ietf.utils.mail import send_mail
 from ietf.mailtrigger.utils import gather_address_lists
+from ietf.utils.timezone import date_today, datetime_from_date, datetime_today, DEADLINE_TZINFO
 from ietf.utils.xmldraft import XMLDraft
 
 
@@ -637,11 +638,22 @@ def has_same_ballot(doc, date1, date2=None):
     """ Test if the most recent ballot created before the end of date1
         is the same as the most recent ballot created before the
         end of date 2. """
+    datetime1 = datetime_from_date(date1, DEADLINE_TZINFO)
     if date2 is None:
-        date2 = datetime.date.today()
-    ballot1 = doc.latest_event(BallotDocEvent,type='created_ballot',time__lt=date1+datetime.timedelta(days=1))
-    ballot2 = doc.latest_event(BallotDocEvent,type='created_ballot',time__lt=date2+datetime.timedelta(days=1))
-    return ballot1==ballot2
+        datetime2 = datetime_today(DEADLINE_TZINFO)
+    else:
+        datetime2 = datetime_from_date(date2, DEADLINE_TZINFO)
+    ballot1 = doc.latest_event(
+        BallotDocEvent,
+        type='created_ballot',
+        time__lt=datetime1 + datetime.timedelta(days=1),
+    )
+    ballot2 = doc.latest_event(
+        BallotDocEvent,
+        type='created_ballot',
+        time__lt=datetime2 + datetime.timedelta(days=1),
+    )
+    return ballot1 == ballot2
 
 def make_notify_changed_event(request, doc, by, new_notify, time=None):
 
@@ -687,7 +699,7 @@ def update_telechat(request, doc, by, new_telechat_date, new_returning_item=None
          and on_agenda
          and prev_agenda
          and new_telechat_date != prev_telechat
-         and prev_telechat < datetime.date.today()
+         and prev_telechat < date_today(DEADLINE_TZINFO)
          and has_same_ballot(doc,prev.telechat_date)
        ):
         returning = True