Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions ietf/api/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
from ietf.meeting.models import Meeting
from ietf.stats.models import MeetingRegistration
from ietf.utils.decorators import require_api_key
from ietf.utils.log import log
from ietf.utils.models import DumpInfo


Expand Down Expand Up @@ -154,7 +153,6 @@ def err(code, text):
if value is None and item in required_fields:
missing_fields.append(item)
data[item] = value
log("Meeting registration notification: %s" % json.dumps(data))
if missing_fields:
return err(400, "Missing parameters: %s" % ', '.join(missing_fields))
number = data['meeting']
Expand Down
1 change: 0 additions & 1 deletion ietf/bin/merge-person-records
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ from django.contrib import admin
from ietf.person.models import Person
from ietf.person.utils import (merge_persons, send_merge_notification, handle_users,
determine_merge_order)
from ietf.utils.log import log


def main():
Expand Down
23 changes: 4 additions & 19 deletions ietf/doc/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,13 @@
import io
import os
import rfc2html
import time

from typing import Optional, TYPE_CHECKING
from weasyprint import HTML as wpHTML

from django.db import models
from django.core import checks
from django.core.cache import caches
from django.core.exceptions import ValidationError
from django.core.validators import URLValidator, RegexValidator
from django.urls import reverse as urlreverse
from django.contrib.contenttypes.models import ContentType
Expand Down Expand Up @@ -221,12 +219,8 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS):
if not hasattr(self, '_cached_href'):
validator = URLValidator()
if self.external_url and self.external_url.split(':')[0] in validator.schemes:
try:
validator(self.external_url)
return self.external_url
except ValidationError:
log.unreachable('2018-12-28')
pass
validator(self.external_url)
return self.external_url

if self.type_id in settings.DOC_HREFS and self.type_id in meeting_doc_refs:
if self.meeting_related():
Expand Down Expand Up @@ -582,7 +576,6 @@ def pdfized(self):
try:
pdf = wpHTML(string=html.replace('\xad','')).write_pdf(stylesheets=[io.BytesIO(b'html { font-size: 94%;}')])
except AssertionError:
log.log(f'weasyprint failed with an assert on {self.name}')
pdf = None
if pdf:
cache.set(cache_key, pdf, settings.PDFIZER_CACHE_TIME)
Expand Down Expand Up @@ -819,16 +812,12 @@ def save_with_history(self, events):
assert events, "You must always add at least one event to describe the changes in the history log"
self.time = max(self.time, events[0].time)

mark = time.time()
self._has_an_event_so_saving_is_allowed = True
self.save()
del self._has_an_event_so_saving_is_allowed
log.log(f'{time.time()-mark:.3f} seconds to save {self.name} Document')

mark = time.time()
from ietf.doc.utils import save_document_in_history
save_document_in_history(self)
log.log(f'{time.time()-mark:.3f} seconds to save {self.name} DocHistory')

def save(self, *args, **kwargs):
# if there's no primary key yet, we can allow the save to go
Expand Down Expand Up @@ -1235,11 +1224,7 @@ def get_dochistory(self):

def __str__(self):
return u"%s %s by %s at %s" % (self.doc.name, self.get_type_display().lower(), self.by.plain_name(), self.time)

def save(self, *args, **kwargs):
super(DocEvent, self).save(*args, **kwargs)
log.assertion('self.rev != None')


class Meta:
ordering = ['-time', '-id']
indexes = [
Expand Down Expand Up @@ -1421,4 +1406,4 @@ class BofreqEditorDocEvent(DocEvent):

class BofreqResponsibleDocEvent(DocEvent):
""" Capture the responsible leadership (IAB and IESG members) for a BOF Request """
responsible = models.ManyToManyField('person.Person', blank=True)
responsible = models.ManyToManyField('person.Person', blank=True)
20 changes: 1 addition & 19 deletions ietf/doc/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,9 @@
from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, is_individual_draft_author, is_bofreq_editor
from ietf.person.models import Person
from ietf.review.models import ReviewWish
from ietf.utils import draft, text
from ietf.utils import draft, log
from ietf.utils.mail import send_mail
from ietf.mailtrigger.utils import gather_address_lists
from ietf.utils import log
from ietf.utils.xmldraft import XMLDraft


Expand Down Expand Up @@ -396,23 +395,6 @@ def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'):

return raw_content

def get_document_content(key, filename, split=True, markup=True):
log.unreachable("2017-12-05")
try:
with io.open(filename, 'rb') as f:
raw_content = f.read()
except IOError:
if settings.DEBUG:
error = "Error; cannot read ("+filename+")"
else:
error = "Error; cannot read ("+key+")"
return error

# if markup:
# return markup_txt.markup(raw_content, split)
# else:
# return raw_content
return text.decode(raw_content)

def tags_suffix(tags):
return ("::" + "::".join(t.name for t in tags)) if tags else ""
Expand Down
12 changes: 0 additions & 12 deletions ietf/doc/views_ballot.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
from ietf.message.utils import infer_message
from ietf.name.models import BallotPositionName, DocTypeName
from ietf.person.models import Person
from ietf.utils import log
from ietf.utils.mail import send_mail_text, send_mail_preformatted
from ietf.utils.decorators import require_api_key
from ietf.utils.response import permission_denied
Expand Down Expand Up @@ -95,17 +94,6 @@ def do_undefer_ballot(request, doc):

email_ballot_undeferred(request, doc, by.plain_name(), telechat_date)

def position_to_ballot_choice(position):
log.unreachable('2018-04-25')
for v, label in BALLOT_CHOICES:
if v and getattr(position, v):
return v
return ""

def position_label(position_value):
log.unreachable('2018-04-25')
return dict(BALLOT_CHOICES).get(position_value, "")

# -------------------------------------------------
class EditPositionForm(forms.Form):
position = forms.ModelChoiceField(queryset=BallotPositionName.objects.all(), widget=forms.RadioSelect, initial="norecord", required=True)
Expand Down
2 changes: 1 addition & 1 deletion ietf/doc/views_doc.py
Original file line number Diff line number Diff line change
Expand Up @@ -1972,7 +1972,7 @@ def rfcdiff_latest_json(request, name, rev=None):
if doc.name != doc.canonical_name():
prev_rev = doc.rev
# not sure what to do if non-numeric values come back, so at least log it
log.assertion('doc.rfc_number().isdigit()')
log.assertion('doc.rfc_number().isdigit()') # .rfc_number() is expensive...

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Expensive, but at least cached. At the moment the value is used again in real code immediately after - could catch exceptions there instead of asserting ahead of time to reduce the risk that the assertions outlive the code they protect.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, as you note, it's not clear what to do if the expectations of those functions are violated, and I view the assertions as a short term safety net checking that we didn't have broken data. We could probably just pull the asserts, but since they aren't really adding load, leaving them for awhile longer won't hurt us.

log.assertion('doc.rev.isdigit()')
if int(doc.rfc_number()) in HAS_TOMBSTONE and prev_rev != '00':
prev_rev = f'{(int(doc.rev)-1):02d}'
Expand Down
13 changes: 2 additions & 11 deletions ietf/mailinglists/management/commands/import_mailman_listinfo.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,13 @@ def log_time(msg):
return
mailman_export = json.loads(result.stdout)

log("Starting import of list info from Mailman")
names = sorted(mailman_export.keys())
log_time("Fetched list of mailman list names")
addr_max_length = Subscribed._meta.get_field('email').max_length

subscribed = { l.name: set(l.subscribed_set.values_list('email', flat=True)) for l in List.objects.all().prefetch_related('subscribed_set') }
log_time("Computed dictionary of list members")

for name in names:
note("List: %s" % mailman_export[name]['internal_name'])
log_time("Fetched Mailman list object for %s" % name)

lists = List.objects.filter(name=mailman_export[name]['real_name'])
if lists.count() > 1:
Expand All @@ -68,12 +64,12 @@ def log_time(msg):
dirty = True
if dirty:
mmlist.save()
log_time(" Updated database List object for %s" % name)
# The following calls return lowercased addresses
if mailman_export[name]['advertised']:
members = set(mailman_export[name]['members'])
if not mailman_export[name]['real_name'] in subscribed:
log("Note: didn't find '%s' in the dictionary of subscriptions" % mailman_export[name]['real_name'])
# 2022-7-29: lots of these going into the logs but being ignored...
# log("Note: didn't find '%s' in the dictionary of subscriptions" % mailman_export[name]['real_name'])
continue
known = subscribed[mailman_export[name]['real_name']]
log_time(" Fetched known list members from database")
Expand All @@ -82,14 +78,10 @@ def log_time(msg):
for addr in to_remove:
note(" Removing subscription: %s" % (addr))
old = Subscribed.objects.get(email=addr)
log_time(" Fetched subscribed object")
old.lists.remove(mmlist)
log_time(" Removed %s from %s" % (mmlist, old))
if old.lists.count() == 0:
note(" Removing address with no subscriptions: %s" % (addr))
old.delete()
log_time(" Removed %s" % old)
log_time(" Removed addresses no longer subscribed")
if to_remove:
log(" Removed %s addresses from %s" % (len(to_remove), name))
for addr in to_add:
Expand All @@ -103,7 +95,6 @@ def log_time(msg):
sys.stderr.write(" ** Error handling %s in %s: %s\n" % (addr, name, e))
continue
new.lists.add(mmlist)
log_time(" Added new addresses")
if to_add:
log(" Added %s addresses to %s" % (len(to_add), name))
log("Completed import of list info from Mailman")
Expand Down
18 changes: 7 additions & 11 deletions ietf/meeting/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
validate_file_extension,
)
from ietf.utils.fields import MissingOkImageField
from ietf.utils.log import unreachable

countries = list(pytz.country_names.items())
countries.sort(key=lambda x: x[1])
Expand Down Expand Up @@ -274,16 +273,13 @@ def proceedings_format_version(self):
else:
version = len(settings.PROCEEDINGS_VERSION_CHANGES) # start assuming latest version
mtg_number = self.get_number()
if mtg_number is None:
unreachable('2021-08-10')
else:
# Find the index of the first entry in the version change array that
# is >= this meeting's number. The first entry in the array is 0, so the
# version is always >= 1 for positive meeting numbers.
for vers, threshold in enumerate(settings.PROCEEDINGS_VERSION_CHANGES):
if mtg_number < threshold:
version = vers
break
# Find the index of the first entry in the version change array that
# is >= this meeting's number. The first entry in the array is 0, so the
# version is always >= 1 for positive meeting numbers.
for vers, threshold in enumerate(settings.PROCEEDINGS_VERSION_CHANGES):
if mtg_number < threshold:
version = vers
break
self._proceedings_format_version = version # save this for later
return self._proceedings_format_version

Expand Down
21 changes: 1 addition & 20 deletions ietf/secr/meetings/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,25 +172,6 @@ def send_notifications(meeting, groups, person):
by=person,desc='sent scheduled notification for %s' % meeting)


# -------------------------------------------------
# AJAX Functions
# -------------------------------------------------
# def ajax_get_times(request, meeting_id, day):
# '''
# Ajax function to get timeslot times for a given day.
# returns JSON format response: [{id:start_time, value:start_time-end_time},...]
# '''
# # TODO strip duplicates if there are any
# from ietf.utils import log
# log.unreachable("2017-07-08")
# results=[]
# room = Room.objects.filter(meeting__number=meeting_id)[0]
# slots = TimeSlot.objects.filter(meeting__number=meeting_id,time__week_day=day,location=room).order_by('time')
# for slot in slots:
# d = {'id': slot.time.strftime('%H%M'), 'value': '%s-%s' % (slot.time.strftime('%H%M'), slot.end_time().strftime('%H%M'))}
# results.append(d)
#
# return HttpResponse(json.dumps(results), content_type='application/javascript')

# --------------------------------------------------
# STANDARD VIEW FUNCTIONS
Expand Down Expand Up @@ -922,4 +903,4 @@ def view(request, meeting_id):

return render(request, 'meetings/view.html', {
'meeting': meeting},
)
)
7 changes: 4 additions & 3 deletions ietf/stats/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
from ietf.stats.models import MeetingRegistration, CountryAlias
from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries, compute_hirsch_index
from ietf.ietfauth.utils import has_role
from ietf.utils.log import log
from ietf.utils.response import permission_denied

def stats_index(request):
Expand Down Expand Up @@ -744,7 +743,8 @@ def generate_canonical_names(values):
"eu_countries": sorted(eu_countries or [], key=lambda c: c.name),
"content_template": "stats/document_stats_{}.html".format(template_name),
}
log("Cache miss for '%s'. Data size: %sk" % (cache_key, len(str(data))/1000))
# Logs are full of these, but nobody is using them
# log("Cache miss for '%s'. Data size: %sk" % (cache_key, len(str(data))/1000))
cache.set(cache_key, data, 24*60*60)
return render(request, "stats/document_stats.html", data)

Expand Down Expand Up @@ -996,7 +996,8 @@ def reg_name(r):
"eu_countries": sorted(eu_countries or [], key=lambda c: c.name),
"content_template": "stats/meeting_stats_{}.html".format(template_name),
}
log("Cache miss for '%s'. Data size: %sk" % (cache_key, len(str(data))/1000))
# Logs are full of these, but nobody is using them...
# log("Cache miss for '%s'. Data size: %sk" % (cache_key, len(str(data))/1000))
cache.set(cache_key, data, 24*60*60)
#
return render(request, "stats/meeting_stats.html", data)
Expand Down
14 changes: 5 additions & 9 deletions ietf/submit/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,7 @@ def find_submission_filenames(draft):

@transaction.atomic
def post_submission(request, submission, approved_doc_desc, approved_subm_desc):
# This is very chatty into the logs, but these could still be useful for quick diagnostics
log.log(f"{submission.name}: start")
system = Person.objects.get(name="(System)")
submitter_parsed = submission.submitter_parsed()
Expand Down Expand Up @@ -589,15 +590,10 @@ def ensure_person_email_info_exists(name, email, docname):
else:
person.name_from_draft = name

# make sure we have an email address
if addr and (addr.startswith('unknown-email-') or is_valid_email(addr)):
active = True
addr = addr.lower()
else:
log.unreachable('2019-10-02')
# we're in trouble, use a fake one
active = False
addr = "unknown-email-%s" % person.plain_ascii().replace(" ", "-")

active = True
addr = addr.lower()


try:
email = person.email_set.get(address=addr)
Expand Down