python日志线程安全

Question

我试图理解python日志在以下情况下是如何实现线程安全的： 1.我在主线程中为同一个文件创建不同的处理程序。然后让一个线程登录到它。 2.我从不同的线程中创建指向同一文件的多个处理程序

我在FileHandler的源代码中看到，创建的每个处理程序都有自己的锁，但在我提到的情况下，这是如何工作的？ 如果我理解正确，锁的作用域就是FileHandler对象，这意味着如果我创建两个不同的处理程序，它们就不会共享锁，并且可能会遇到竞争条件

所以我的问题是：这个线程如何安全

这是相关代码。请注意，FileHandler类继承StreamHandler，StreamHandler又继承Handler

class StreamHandler(Handler):
    """
    A handler class which writes logging records, appropriately formatted,
    to a stream. Note that this class does not close the stream, as
    sys.stdout or sys.stderr may be used.
    """

    terminator = '\n'

    def __init__(self, stream=None):
        """
        Initialize the handler.

        If stream is not specified, sys.stderr is used.
        """
        Handler.__init__(self)
        if stream is None:
            stream = sys.stderr
        self.stream = stream

    def flush(self):
        """
        Flushes the stream.
        """
        self.acquire()
        try:
            if self.stream and hasattr(self.stream, "flush"):
                self.stream.flush()
        finally:
            self.release()

    def emit(self, record):
        """
        Emit a record.

        If a formatter is specified, it is used to format the record.
        The record is then written to the stream with a trailing newline.  If
        exception information is present, it is formatted using
        traceback.print_exception and appended to the stream.  If the stream
        has an 'encoding' attribute, it is used to determine how to do the
        output to the stream.
        """
        try:
            msg = self.format(record)
            stream = self.stream
            stream.write(msg)
            stream.write(self.terminator)
            self.flush()
        except Exception:
            self.handleError(record)

    def __repr__(self):
        level = getLevelName(self.level)
        name = getattr(self.stream, 'name', '')
        if name:
            name += ' '
        return '<%s %s(%s)>' % (self.__class__.__name__, name, level)


class FileHandler(StreamHandler):
    """
    A handler class which writes formatted logging records to disk files.
    """
    def __init__(self, filename, mode='a', encoding=None, delay=False):
        """
        Open the specified file and use it as the stream for logging.
        """
        # Issue #27493: add support for Path objects to be passed in
        filename = os.fspath(filename)
        #keep the absolute path, otherwise derived classes which use this
        #may come a cropper when the current directory changes
        self.baseFilename = os.path.abspath(filename)
        self.mode = mode
        self.encoding = encoding
        self.delay = delay
        if delay:
            #We don't open the stream, but we still need to call the
            #Handler constructor to set level, formatter, lock etc.
            Handler.__init__(self)
            self.stream = None
        else:
            StreamHandler.__init__(self, self._open())

    def close(self):
        """
        Closes the stream.
        """
        self.acquire()
        try:
            try:
                if self.stream:
                    try:
                        self.flush()
                    finally:
                        stream = self.stream
                        self.stream = None
                        if hasattr(stream, "close"):
                            stream.close()
            finally:
                # Issue #19523: call unconditionally to
                # prevent a handler leak when delay is set
                StreamHandler.close(self)
        finally:
            self.release()

    def _open(self):
        """
        Open the current base file with the (original) mode and encoding.
        Return the resulting stream.
        """
        return open(self.baseFilename, self.mode, encoding=self.encoding)

    def emit(self, record):
        """
        Emit a record.

        If the stream was not opened because 'delay' was specified in the
        constructor, open it before calling the superclass's emit.
        """
        if self.stream is None:
            self.stream = self._open()
        StreamHandler.emit(self, record)

    def __repr__(self):
        level = getLevelName(self.level)
        return '<%s %s (%s)>' % (self.__class__.__name__, self.baseFilename, level)

Answer 1

我已经实现了线程安全的日志记录。目标是使用线程并行地收集具有不同标记的tweet

以下是我的日志记录实现：

import sys
import logging
import threading


class Singleton:
    __lock = threading.Lock()
    __instance = None

    def __init__(self):
        if self.__class__.__instance:
            raise Exception('Tried to allocate a second instance of a singleton.\nUse getInstance() instead.')
            sys.exit(-1)

    @classmethod
    def get_instance(cls):
        if cls.__instance is None:
            with cls.__lock:
                if cls.__instance is None:
                    cls.__instance = cls()
        return cls.__instance


class Logger(Singleton):
    __FORMAT = '%(asctime)s - %(levelname)s - [%(threadName)s] - %(message)s'

    def __init__(self, name: str = __name__):
        super().__init__()

        logger = logging.getLogger(name)
        logger.setLevel(logging.DEBUG)

        log_formatter = logging.Formatter(self.__class__.__FORMAT)

        console_handler = logging.StreamHandler(sys.stdout)
        console_handler.setFormatter(log_formatter)
        console_handler.setLevel(logging.DEBUG)

        file_handler = logging.FileHandler('log.log')
        file_handler.setFormatter(log_formatter)
        file_handler.setLevel(logging.INFO)

        if logger.handlers:
            logger.handlers = []


        logger.addHandler(file_handler)
        logger.addHandler(console_handler)

        self.__logger = logger

        self.__class__.__instance = self

    @property
    def logger(self):
        return self.__logger

    def debug(self, message: str):
        self.__logger.debug(message)

    def info(self, message: str):
        self.__logger.info(message)

    def warn(self, message: str):
        self.__logger.warning(message)

    def error(self, message: str):
        self.__logger.error(message)

    def critical(self, message: str):
        self.__logger.critical(message)

    def exception(self, message: str):
        self.__logger.exception(message)

然后在线程中，只需调用Logger类的get_实例方法

Logger.get_instance().info(f'Your log here')

我希望这对你有帮助

Answer 2

换句话说，在这两种情况下，处理程序都可以同时打开文件，同时写入文件

这意味着，从某种意义上说，消息可能在文件中“混在一起”（例如，开始写入msg1，然后开始写入msg2，然后继续写入msg1）是不“安全的”

除此之外，我看不出还有什么害处。单独的FileHandler实例不会相互干扰

因此，总而言之，日志文件的内容可能并不完美，但没有其他内容真正破坏

Answer 3

这是一个需要一些逆向工程的好问题

简单的答案是：FileHandler在本例中本身不是线程安全的，但不能使用构造函数创建它。而是使用工厂方法，确保线程安全：

# see here: https://github.com/python/cpython/blob/586be6f3ff68ab4034e555f1434a4427e129ad0b/Lib/logging/__init__.py#L1985
 if handlers is None:
                filename = kwargs.pop("filename", None)
                mode = kwargs.pop("filemode", 'a')
                if filename:
                    if 'b'in mode:
                        errors = None
                    h = FileHandler(filename, mode,
                                    encoding=encoding, errors=errors)
                else:
                    stream = kwargs.pop("stream", None)
                    h = StreamHandler(stream)
                handlers = [h]

以及：

# https://github.com/python/cpython/blob/586be6f3ff68ab4034e555f1434a4427e129ad0b/Lib/logging/__init__.py#L1272
    def getLogger(self, name):
        """
        Get a logger with the specified name (channel name), creating it
        if it doesn't yet exist. This name is a dot-separated hierarchical
        name, such as "a", "a.b", "a.b.c" or similar.
        If a PlaceHolder existed for the specified name [i.e. the logger
        didn't exist but a child of it did], replace it with the created
        logger and fix up the parent/child references which pointed to the
        placeholder to now point to the logger.
        """
        rv = None
        if not isinstance(name, str):
            raise TypeError('A logger name must be a string')
        _acquireLock()
        try:
            if name in self.loggerDict:
                rv = self.loggerDict[name]
                if isinstance(rv, PlaceHolder):
                    ph = rv
                    rv = (self.loggerClass or _loggerClass)(name)
                    rv.manager = self
                    self.loggerDict[name] = rv
                    self._fixupChildren(ph, rv)
                    self._fixupParents(rv)
            else:
                rv = (self.loggerClass or _loggerClass)(name)
                rv.manager = self
                self.loggerDict[name] = rv
                self._fixupParents(rv)
        finally:
            _releaseLock()
        return rv

请注意这里发生的两件事：
1） 在创建新的Handler之前获取锁
2） 如果已创建名称记录器，则返回该名称。因此，对于同一个文件，应该只获得一个FileHandler实例