refactor!: replace os.listdir() with os.scandir()

In many places we did a listdir() then a stat to see if it's a file or
directory. This change removes the need for the stat call. Also for
larger directories, scandir() is an iterator, so less memory use.

There is one remnant of listdir used in an error handler. That
requires a stat on each element in the directory, so there is no
benefit to using scandir() other than a slight memory saving on a
rarely used piece of code.

BREAKING CHANGE:

Python 2 requires installation of scandir pip package after this
commit.

Author: rouilj

roundup/admin.py

@@ -1226,16 +1226,17 @@ class colon_separated(csv.excel):
             delimiter = ':'
 
         # import all the files
-        for file in os.listdir(import_dir):
-            classname, ext = os.path.splitext(file)
+        for dir_entry in os.scandir(import_dir):
+            filename = dir_entry.name
+            classname, ext = os.path.splitext(filename)
             # we only care about CSV files
             if ext != '.csv' or classname.endswith('-journals'):
                 continue
 
             cl = self.get_class(classname)
 
             # ensure that the properties and the CSV file headings match
-            with open(os.path.join(import_dir, file), 'r') as f:
+            with open(os.path.join(import_dir, filename), 'r') as f:
                 reader = csv.reader(f, colon_separated, lineterminator='\n')
                 file_props = None
                 maxid = 1

roundup/backends/blobfiles.py

@@ -27,12 +27,11 @@ def files_in_dir(dir):
     if not os.path.exists(dir):
         return 0
     num_files = 0
-    for dir_entry in os.listdir(dir):
-        full_filename = os.path.join(dir, dir_entry)
-        if os.path.isfile(full_filename):
+    for dir_entry in os.scandir(dir):
+        if dir_entry.is_file():
             num_files = num_files + 1
-        elif os.path.isdir(full_filename):
-            num_files = num_files + files_in_dir(full_filename)
+        elif dir_entry.is_dir():
+            num_files = num_files + files_in_dir(dir_entry.path)
     return num_files
 
 

roundup/cgi/templating.py

@@ -334,9 +334,10 @@ def check(self, name):
 
     def precompile(self):
         """ Precompile templates in load directory by loading them """
-        for filename in os.listdir(self.template_dir):
+        for dir_entry in os.scandir(self.template_dir):
+            filename = dir_entry.name
             # skip subdirs
-            if os.path.isdir(filename):
+            if dir_entry.is_dir():
                 continue
 
             # skip files without ".html" or ".xml" extension - .css, .js etc.

roundup/init.py

@@ -44,7 +44,7 @@ def copytree(src, dst, symlinks=0):
     """
 
     # Prevent 'hidden' files (those starting with '.') from being considered.
-    names = [f for f in os.listdir(src) if not f.startswith('.')]
+    names = [f.name for f in os.scandir(src) if not f.name.startswith('.')]
     try:
         os.mkdir(dst)
     except OSError as error:
@@ -111,17 +111,16 @@ def install(instance_home, template, settings={}):
         config.save(config_ini_file)
 
 
-def listTemplates(dir):
+def listTemplates(directory):
     ''' List all the Roundup template directories in a given directory.
 
         Find all the dirs that contain a TEMPLATE-INFO.txt and parse it.
 
         Return a list of dicts of info about the templates.
     '''
     ret = {}
-    for idir in os.listdir(dir):
-        idir = os.path.join(dir, idir)
-        ti = loadTemplateInfo(idir)
+    for dir_entry in os.scandir(directory):
+        ti = loadTemplateInfo(dir_entry.path)
         if ti:
             ret[ti['name']] = ti
     return ret

roundup/instance.py

@@ -188,7 +188,8 @@ def get_extensions(self, dirname):
         dirpath = os.path.join(self.tracker_home, dirname)
         if os.path.isdir(dirpath):
             sys.path.insert(1, dirpath)
-            for name in os.listdir(dirpath):
+            for dir_entry in os.scandir(dirpath):
+                name = dir_entry.name
                 if not name.endswith('.py'):
                     continue
                 env = {}

roundup/scripts/roundup_gettext.py

@@ -64,6 +64,8 @@ class Options:
 TEMPLATE_FILE = "messages.pot"
 
 
+# FIXME: html directory can be specified in config.ini. Should be
+#        a parameter, or config.ini should be loaded.
 def run():
     # return unless command line arguments contain single directory path
     if (len(sys.argv) != 2) or (sys.argv[1] in ("-h", "--help")):
@@ -76,9 +78,9 @@ def run():
     if os.path.isdir(htmldir):
         # glob is not used because i want to match file names
         # without case sensitivity, and that is easier done this way.
-        htmlfiles = [filename for filename in os.listdir(htmldir)
-                     if os.path.isfile(os.path.join(htmldir, filename))
-                     and filename.lower().endswith(".html")]
+        htmlfiles = [e.name for e in os.scandir(htmldir)
+                     if e.is_file()
+                     and e.name.lower().endswith(".html")]
     else:
         htmlfiles = []
     # return if no html files found