gppylib.gplog在文件lib/python/gppylib/gplog.py下。该模块包含一些建立python内建logging模块的帮助函数。主要是方便tool、helper等模块建立各自的Logger(setup_helper_tool_logging和setup_tool_logging函数)。我们通过一个例子来对该文件中相关函数进行分析。

from gppylib import gplog
logger = gplog.setup_tool_logging(EXECNAME, hostname, username, logdir)
if options.verbose:
gplog.enable_verbose_logging()
if options.quiet:
gplog.quiet_stdout_logging()
logger.info("Start myTool")
...

首先是调用setup_tool_logging函数,参数是EXECNAME, hostname, username, logdir。该函数返回单例模式的logger,可用于向stdout和文件中输出记录。主要是个性化定制logger的名字name,向SOUT_HANDLER和FILE_HANDLER设置Formatter,并使用个性化的格式化字符串。

def setup_tool_logging(appName, hostname, userName, logdir=None, nonuser=False):
"""
Returns a singleton logger for standard Greenplum tools:
- Logs output to stdout
- Logs output to a file, typically in ~/gpAdminLogs
"""
global _DEFAULT_FORMATTER
global _APP_NAME_FOR_DEFAULT_FORMAT
logger_name = "%s:%s" % (hostname, userName)
if nonuser:
appName = appName + "_" + logger_name
_APP_NAME_FOR_DEFAULT_FORMAT = appName
_enable_gpadmin_logging(appName, logdir)
#
# now reset the default formatter (someone may have called get_default_logger before calling setup_tool_logging)
#
logger = get_default_logger()
logger.name = logger_name
_DEFAULT_FORMATTER = None
f = _get_default_formatter()
_SOUT_HANDLER.setFormatter(f)
_FILE_HANDLER.setFormatter(f)
return

该函数用于为default logger设置文件输出handler,验证log的目录的正确性和存在性,最后通过调用_set_file_logging添加file handler。stream handler是在get_default_logger中加入logger的。

def _enable_gpadmin_logging(name, logdir=None):
"""
Sets up the file output handler for the default logger.
- if logdir is not specified it uses ~/gpAdminLogs
- the file is constructed as appended with "<logdir>/<name>_<date>.log"

NOTE: internal use only
"""
global _FILE_HANDLER
get_default_logger()
now = datetime.date.today()
if logdir is None:
home_dir = os.path.expanduser("~")
gpadmin_logs_dir = home_dir + "/gpAdminLogs"
else:
gpadmin_logs_dir = logdir
if not os.path.exists(gpadmin_logs_dir):
os.mkdir(gpadmin_logs_dir)
filename = "%s/%s_%s.log" % (gpadmin_logs_dir, name, now.strftime("%Y%m%d"))
_set_file_logging(filename)

get_default_logger单例模式返回默认logger,值创建stream handle也就是项stdout输出记录,并设置file handler。

def get_default_logger():
"""
Return the singleton default logger.
If a logger has not yet been established it creates one that:
- Logs output to stdout
- Does not setup file logging.
Typical usage would be to call one of the setup_*_logging() functions
at the beginning of a script in order to establish the exact type of
logging desired, after which later calls to get_default_logger() can be
used to return a reference to the logger.
"""
global _LOGGER, _SOUT_HANDLER
if _LOGGER is None:
_LOGGER = logging.getLogger('default')
f = _get_default_formatter()
_SOUT_HANDLER = EncodingStreamHandler(sys.stdout)
_SOUT_HANDLER.setFormatter(f)
_LOGGER.addHandler(_SOUT_HANDLER)
_LOGGER.setLevel(logging.INFO)
return

自定义的Handler类,继承自logging.StreamHandler,主要用于确保message的编码为utf-8。在重写的emit函数中进行处理,最终还需调用父类的emit的函数对记录进行处理(写到Stream中)

class EncodingStreamHandler(logging.StreamHandler):
"""This handler makes sure that the encoding of the message is utf-8 before
passing it along to the StreamHandler. This will prevent encode/decode
errors later on."""

def __init__(self, strm=None):
logging.StreamHandler.__init__(self, strm)

def emit(self, record):
if not isinstance(record.msg, str) and not isinstance(record.msg, unicode):
record.msg = str(record.msg)
if not isinstance(record.msg, unicode):
record.msg = unicode(record.msg, 'utf-8')
logging.StreamHandler.emit(self, record)

同样这里设置file handle,用于向文件中输入记录,其中的EncodingFileHandler继承ogging.FileHandler,也是确保message的编码为utf-8。

def _set_file_logging(filename):
"""
Establishes a file output HANDLER for the default formatter.

NOTE: internal use only
"""
global _LOGGER, _SOUT_HANDLER, _FILENAME, _FILE_HANDLER
_FILENAME = filename
_FILE_HANDLER = EncodingFileHandler(filename, 'a')
_FILE_HANDLER.setFormatter(_get_default_formatter())
_LOGGER.addHandler(_FILE_HANDLER)

class EncodingFileHandler(logging.FileHandler):
"""This handler makes sure that the encoding of the message is utf-8 before
passing it along to the FileHandler. This will prevent encode/decode
errors later on."""
def __init__(self, filename, mode='a', encoding=None, delay=0):
logging.FileHandler.__init__(self, filename, mode, encoding, delay)
def emit(self, record):
if not isinstance(record.msg, str) and not isinstance(record.msg, unicode):
record.msg = str(record.msg)
if not isinstance(record.msg, unicode):
record.msg = unicode(record.msg, 'utf-8')
logging.FileHandler.emit(self, record)

_get_default_formatter函数用于对formatter中的格式化进行定制后,返回Formatter对象。

def _get_default_formatter():
"""
Returns the default formatter, constructing it if needed.
The default formatter formats things using Greenplum standard logging:
<date>:<pid> <programname>:<hostname>:<username>:[LEVEL]:-message
NOTE: internal use only
"""
global _DEFAULT_FORMATTER
global _APP_NAME_FOR_DEFAULT_FORMAT
if _DEFAULT_FORMATTER is None:
format_str = "%(asctime)s:%(programname)s:%(name)s-[%(levelname)-s]:-%(message)s"
app_name = _APP_NAME_FOR_DEFAULT_FORMAT.replace("%", "") # to make sure we don't produce a format string
format_str = format_str.replace("%(programname)s", "%06d %s" % (os.getpid(), app_name))
_DEFAULT_FORMATTER = logging.Formatter(format_str, "%Y%m%d:%H:%M:%S")
return

enable_verbose_logging和quiet_stdout_logging调用Logger对象的setLevel函数用于设置输出记录的级别阈值

def enable_verbose_logging():
"""
Increases the log level to be verbose.
- Applies to all logging handlers (stdout/file).
"""
_LOGGER.setLevel(logging.DEBUG)

def quiet_stdout_logging():
"""
Reduce log level for stdout logging
"""
global _SOUT_HANDLER
_SOUT_HANDLER.setLevel(logging.WARN)