better log formatting when using syslog

Author: joeshaw

pyres/__init__.py

@@ -15,7 +15,13 @@ def special_log_file(filename):
         return True
     return False
 
-def get_logging_handler(filename):
+def get_logging_handler(filename, procname, namespace=None):
+    if namespace:
+        message_format = namespace + ': %(message)s'
+    else:
+        message_format = '%(message)s'
+    format = '%(asctime)s %(levelname)-8s ' + message_format
+
     if not filename:
         filename = "stderr"
     if filename == "stderr":
@@ -35,6 +41,7 @@ def get_logging_handler(filename):
             raise Exception("Unable to figure out the syslog socket path")
 
         handler = SysLogHandler(syslog_path, facility)
+        format = procname + "[%(process)d]: " + message_format
     else:
         try:
             from logging.handlers import WatchedFileHandler
@@ -43,16 +50,15 @@ def get_logging_handler(filename):
             from logging.handlers import RotatingFileHandler
             handler = RotatingFileHandler(filename,maxBytes=52428800,
                                           backupCount=7)
+    handler.setFormatter(logging.Formatter(format, '%Y-%m-%d %H:%M:%S'))
     return handler
 
-def setup_logging(log_level=logging.INFO, filename=None):
+def setup_logging(procname, log_level=logging.INFO, filename=None):
     if log_level == logging.NOTSET:
         return
     logger = logging.getLogger()
     logger.setLevel(log_level)
-    handler = get_logging_handler(filename)
-    handler.setFormatter(logging.Formatter(
-        '%(asctime)s %(levelname)-8s %(message)s', '%Y-%m-%d %H:%M:%S'))
+    handler = get_logging_handler(filename, procname)
     logger.addHandler(handler)
 
 def setup_pidfile(path):

pyres/horde.py

@@ -19,14 +19,12 @@
     def setproctitle(name):
         pass
 
-def setup_logging(namespace='', log_level=logging.INFO, log_file=None):
+def setup_logging(procname, namespace='', log_level=logging.INFO, log_file=None):
 
     logger = multiprocessing.get_logger()
     #logger = multiprocessing.log_to_stderr()
     logger.setLevel(log_level)
-    format = '%(asctime)s %(levelname)s '+namespace+': %(message)s'
-    handler = get_logging_handler(log_file)
-    handler.setFormatter(logging.Formatter((format)))
+    handler = get_logging_handler(procname, log_file, namespace)
     logger.addHandler(handler)
     return logger
 
@@ -156,7 +154,7 @@ def run(self):
             else:
                 self.log_file = os.path.join(self.log_path, 'minion-%s.log' % self.pid)
         namespace = 'minion:%s' % self.pid
-        self.logger = setup_logging(namespace, self.log_level, self.log_file)
+        self.logger = setup_logging('minion', namespace, self.log_level, self.log_file)
         #self.clear_logger()
         if isinstance(self.server,basestring):
             self.resq = ResQ(server=self.server, password=self.password)
@@ -311,13 +309,12 @@ def setup_minions(self):
             self._add_minion()
 
     def _setup_logging(self):
-        self.logger = setup_logging('khan', self.logging_level, self.log_file)
+        self.logger = setup_logging('khan', 'khan', self.logging_level, self.log_file)
 
     def work(self, interval=2):
         setproctitle('pyres_manager: Starting')
         self.startup()
         self.setup_minions()
-        #self.logger = setup_logging('khan', self.logging_level, self.log_file)
         self._setup_logging()
         self.logger.info('Running as pid: %s' % self.pid)
         self.logger.info('Added %s child processes' % self.pool_size)

pyres/scripts.py

@@ -54,7 +54,7 @@ def pyres_scheduler():
     (options,args) = parser.parse_args()
     log_level = getattr(logging, options.log_level.upper(),'INFO')
     #logging.basicConfig(level=log_level, format="%(module)s: %(asctime)s: %(levelname)s: %(message)s")
-    setup_logging(log_level=log_level, filename=options.logfile)
+    setup_logging(procname="pyres_scheduler", log_level=log_level, filename=options.logfile)
     setup_pidfile(options.pidfile)
     server = '%s:%s' % (options.host, options.port)
     Scheduler.run(server)
@@ -103,7 +103,7 @@ def pyres_worker():
         parser.error("Argument must be a comma seperated list of queues")
 
     log_level = getattr(logging, options.log_level.upper(), 'INFO')
-    setup_logging(log_level=log_level, filename=options.logfile)
+    setup_logging(procname="pyres_worker", log_level=log_level, filename=options.logfile)
     setup_pidfile(options.pidfile)
 
     interval = options.interval