Fixed a bunch of str/bytes issues.

 - Legacy-Id: 16341

Author: levkowetz

ietf/doc/mails.py

@@ -383,15 +383,15 @@ def generate_issue_ballot_mail(request, doc, ballot):
 def email_iana(request, doc, to, msg, cc=None):
     # fix up message and send it with extra info on doc in headers
     import email
-    parsed_msg = email.message_from_string(msg.encode("utf-8"))
+    parsed_msg = email.message_from_string(msg)
     parsed_msg.set_charset('UTF-8')
 
     extra = extra_automation_headers(doc)
     extra["Reply-To"] = ["noreply@ietf.org", ]
 
     send_mail_text(request, to,
                    parsed_msg["From"], parsed_msg["Subject"],
-                   parsed_msg.get_payload().decode(str(parsed_msg.get_charset())),
+                   parsed_msg.get_payload(),
                    extra=extra,
                    cc=cc)
 

ietf/doc/tests_charter.py

@@ -399,9 +399,9 @@ def test_submit_charter(self):
         # post
         prev_rev = charter.rev
 
-        latin_1_snippet = '\xe5' * 10
-        utf_8_snippet = '\xc3\xa5' * 10
-        test_file = StringIO("Windows line\r\nMac line\rUnix line\n" + latin_1_snippet)
+        latin_1_snippet = b'\xe5' * 10
+        utf_8_snippet = b'\xc3\xa5' * 10
+        test_file = StringIO("Windows line\r\nMac line\rUnix line\n" + latin_1_snippet.decode('latin-1'))
         test_file.name = "unnamed"
 
         r = self.client.post(url, dict(txt=test_file))
@@ -413,7 +413,7 @@ def test_submit_charter(self):
 
         with open(os.path.join(self.charter_dir, charter.canonical_name() + "-" + charter.rev + ".txt")) as f:
             self.assertEqual(f.read(),
-                              "Windows line\nMac line\nUnix line\n" + utf_8_snippet)
+                              "Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode('utf_8'))
 
     def test_submit_initial_charter(self):
         group = GroupFactory(type_id='wg',acronym='mars',list_email='mars-wg@ietf.org')

ietf/doc/views_review.py

@@ -620,13 +620,13 @@ def complete_review(request, name, assignment_id):
 
             # save file on disk
             if review_submission == "upload":
-                encoded_content = form.cleaned_data['review_file']
+                content = form.cleaned_data['review_file']
             else:
-                encoded_content = form.cleaned_data['review_content'].encode("utf-8")
+                content = form.cleaned_data['review_content']
 
             filename = os.path.join(review.get_file_path(), '{}.txt'.format(review.name, review.rev))
-            with open(filename, 'wb') as destination:
-                destination.write(encoded_content)
+            with open(filename, 'w', encoding='utf-8') as destination:
+                destination.write(content)
 
             completion_datetime = datetime.datetime.now()
             if "completion_date" in form.cleaned_data:
@@ -699,7 +699,7 @@ def complete_review(request, name, assignment_id):
                         cc=form.cleaned_data["cc"],
                         body = render_to_string("review/completed_review.txt", {
                             "assignment": assignment,
-                            "content": encoded_content.decode("utf-8"),
+                            "content": content,
                         }),
                     )
                 msg.related_groups.add(*related_groups)

ietf/ietfauth/htpasswd.py

@@ -1,14 +1,17 @@
+# Copyright The IETF Trust 2016-2019, All Rights Reserved
+
 import subprocess, hashlib
+from django.utils.encoding import force_bytes
 
 from django.conf import settings
 
 def update_htpasswd_file(username, password):
     if getattr(settings, 'USE_PYTHON_HTDIGEST', None):
         pass_file = settings.HTPASSWD_FILE
         realm = settings.HTDIGEST_REALM
-        prefix = '%s:%s:' % (username, realm)
-        key = hashlib.md5(prefix + password).hexdigest()
-        f = open(pass_file, 'r+')
+        prefix = force_bytes('%s:%s:' % (username, realm))
+        key = force_bytes(hashlib.md5(prefix + force_bytes(password)).hexdigest())
+        f = open(pass_file, 'r+b')
         pos = f.tell()
         line = f.readline()
         while line:
@@ -17,7 +20,7 @@ def update_htpasswd_file(username, password):
             pos=f.tell()
             line = f.readline()
         f.seek(pos)
-        f.write('%s%s\n' % (prefix, key))
+        f.write(b'%s%s\n' % (prefix, key))
         f.close()
     else:
         p = subprocess.Popen([settings.HTPASSWD_COMMAND, "-b", settings.HTPASSWD_FILE, username, password], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

ietf/meeting/models.py

@@ -255,7 +255,7 @@ def build_timeslices(self):
         days.sort()
         for ymd in time_slices:
             time_slices[ymd].sort()
-            slots[ymd].sort(lambda x,y: cmp(x.time, y.time))
+            slots[ymd].sort(key=lambda x: x.time)
         return days,time_slices,slots
 
     # this functions makes a list of timeslices and rooms, and
@@ -1080,22 +1080,22 @@ def json_dict(self, host_scheme):
             sess1['group_href']     = urljoin(host_scheme, self.group.json_url())
             if self.group.parent is not None:
                 sess1['area']           = self.group.parent.acronym.upper()
-            sess1['description']    = self.group.name.encode('utf-8')
+            sess1['description']    = self.group.name
             sess1['group_id']       = str(self.group.pk)
         reslist = []
         for r in self.resources.all():
             reslist.append(r.json_dict(host_scheme))
         sess1['resources']      = reslist
         sess1['session_id']     = str(self.pk)
-        sess1['name']           = self.name.encode('utf-8')
-        sess1['title']          = self.short_name.encode('utf-8')
-        sess1['short_name']     = self.short_name.encode('utf-8')
+        sess1['name']           = self.name
+        sess1['title']          = self.short_name
+        sess1['short_name']     = self.short_name
         sess1['bof']            = str(self.group.is_bof())
-        sess1['agenda_note']    = self.agenda_note.encode('utf-8')
+        sess1['agenda_note']    = self.agenda_note
         sess1['attendees']      = str(self.attendees)
-        sess1['status']         = self.status.name.encode('utf-8')
+        sess1['status']         = self.status.name
         if self.comments is not None:
-            sess1['comments']       = self.comments.encode('utf-8')
+            sess1['comments']       = self.comments
         sess1['requested_time'] = self.requested.strftime("%Y-%m-%d")
         # the related person object sometimes does not exist in the dataset.
         try:

ietf/message/utils.py

@@ -41,7 +41,7 @@ def send_scheduled_message_from_send_queue(send_queue):
         # make body a real message so we can parse it
         body = ("MIME-Version: 1.0\r\nContent-Type: %s\r\n" % message.content_type) + body
 
-        msg = email.message_from_string(body.encode("utf-8"))
+        msg = email.message_from_string(body)
         send_mail_mime(None, message.to, message.frm, message.subject,
                        msg, cc=message.cc, bcc=message.bcc)
 

ietf/nomcom/fields.py

@@ -1,3 +1,4 @@
+# Copyright The IETF Trust 2012-2019, All Rights Reserved
 from django.conf import settings
 from django.db import models
 from django.utils.encoding import smart_str
@@ -19,7 +20,7 @@ def pre_save(self, instance, add):
                 raise ValueError("Trying to read the NomCom public key: " + str(e))
 
             command = "%s smime -encrypt -in /dev/stdin %s" % (settings.OPENSSL_COMMAND, cert_file)
-            code, out, error = pipe(command, comments)
+            code, out, error = pipe(command, comments.encode())
             if code != 0:
                 log("openssl error: %s:\n  Error %s: %s" %(command, code, error))
             if not error:

ietf/nomcom/test_data.py

@@ -1,3 +1,4 @@
+# Copyright The IETF Trust 2012-2019, All Rights Reserved
 import tempfile
 import os
 
@@ -38,7 +39,7 @@
 
 def generate_cert():
     """Function to generate cert"""
-    config = """
+    config = b"""
             [ req ]
             distinguished_name = req_distinguished_name
             string_mask        = utf8only
@@ -89,7 +90,7 @@ def check_comments(encryped, plain, privatekey_file):
 
     decrypted_file.close()
     encrypted_file.close()
-    decrypted_comments = open(decrypted_file.name, 'r').read().decode('utf8')
+    decrypted_comments = open(decrypted_file.name, 'r').read()
     os.unlink(encrypted_file.name)
     os.unlink(decrypted_file.name)
 

ietf/nomcom/utils.py

@@ -87,7 +87,7 @@ def get_user_email(user):
     return user._email_cache
 
 def get_hash_nominee_position(date, nominee_position_id):
-    return hashlib.md5('%s%s%s' % (settings.SECRET_KEY, date, nominee_position_id)).hexdigest()
+    return hashlib.md5(('%s%s%s' % (settings.SECRET_KEY, date, nominee_position_id)).encode()).hexdigest()
 
 
 def initialize_templates_for_group(group):
@@ -160,7 +160,7 @@ def retrieve_nomcom_private_key(request, year):
 
     command = "%s bf -d -in /dev/stdin -k \"%s\" -a"
     code, out, error = pipe(command % (settings.OPENSSL_COMMAND,
-                                       settings.SECRET_KEY), private_key)
+                                       settings.SECRET_KEY), private_key.encode())
     if code != 0:
         log("openssl error: %s:\n  Error %s: %s" %(command, code, error))        
     return out
@@ -399,9 +399,8 @@ def make_nomineeposition_for_newperson(nomcom, candidate_name, candidate_email,
 def getheader(header_text, default="ascii"):
     """Decode the specified header"""
 
-    headers = decode_header(header_text)
-    header_sections = [str(text, charset or default)
-                       for text, charset in headers]
+    tuples = decode_header(header_text)
+    header_sections = [ text.decode(charset or default) if isinstance(text, bytes) else text for text, charset in tuples]
     return "".join(header_sections)
 
 

ietf/review/mailarch.py

@@ -11,6 +11,7 @@
 from pyquery import PyQuery
 
 from django.conf import settings
+from django.utils.encoding import force_bytes
 
 def list_name_from_email(list_email):
     if not list_email.endswith("@ietf.org"):
@@ -23,9 +24,9 @@ def hash_list_message_id(list_name, msgid):
     # https://www.mail-archive.com/faq.html#listserver except the list
     # name (without "@ietf.org") is used instead of the full address,
     # and rightmost "=" signs are (optionally) stripped
-    sha = hashlib.sha1(msgid)
-    sha.update(list_name)
-    return base64.urlsafe_b64encode(sha.digest()).rstrip("=")
+    sha = hashlib.sha1(force_bytes(msgid))
+    sha.update(force_bytes(list_name))
+    return base64.urlsafe_b64encode(sha.digest()).rstrip(b"=")
 
 def construct_query_urls(review_req, query=None):
     list_name = list_name_from_email(review_req.team.list_email)