Python, je veux une journalisation avec rotation et compression des journaux

Question

Quelqu'un peut-il suggérer un moyen en python de se connecter avec :

• rotation des journaux tous les jours
• compression des journaux lors de leur rotation
• facultatif - supprimez le fichier journal le plus ancien pour conserver X Mo d'espace libre
• facultatif - fichiers journaux sftp vers le serveur

Merci pour toute réponse, Fred

Answer 1

L'autre façon de compresser le fichier journal pendant la rotation (nouveau dans python 3.3) consiste à utiliser le rotateur d' attributs de classe BaseRotatingHandler (et tous hérités) par exemple :

 import gzip
import os
import logging
import logging.handlers

class GZipRotator:
    def __call__(self, source, dest):
        os.rename(source, dest)
        f_in = open(dest, 'rb')
        f_out = gzip.open("%s.gz" % dest, 'wb')
        f_out.writelines(f_in)
        f_out.close()
        f_in.close()
        os.remove(dest)

logformatter = logging.Formatter('%(asctime)s;%(levelname)s;%(message)s')
log = logging.handlers.TimedRotatingFileHandler('debug.log', 'midnight', 1, backupCount=5)
log.setLevel(logging.DEBUG)
log.setFormatter(logformatter)
log.rotator = GZipRotator()

logger = logging.getLogger('main')
logger.addHandler(log)    
logger.setLevel(logging.DEBUG)

....

Plus vous pouvez voir ici .

Answer 2

En plus de la réponse d'unutbu : voici comment modifier le TimedRotatingFileHandler pour compresser à l'aide de fichiers zip.

 import logging
import logging.handlers
import zipfile
import codecs
import sys
import os
import time
import glob


class TimedCompressedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler):
    """
    Extended version of TimedRotatingFileHandler that compress logs on rollover.
    """
    def doRollover(self):
        """
        do a rollover; in this case, a date/time stamp is appended to the filename
        when the rollover happens.  However, you want the file to be named for the
        start of the interval, not the current time.  If there is a backup count,
        then we have to get a list of matching filenames, sort them and remove
        the one with the oldest suffix.
        """

        self.stream.close()
        # get the time that this sequence started at and make it a TimeTuple
        t = self.rolloverAt - self.interval
        timeTuple = time.localtime(t)
        dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple)
        if os.path.exists(dfn):
            os.remove(dfn)
        os.rename(self.baseFilename, dfn)
        if self.backupCount > 0:
            # find the oldest log file and delete it
            s = glob.glob(self.baseFilename + ".20*")
            if len(s) > self.backupCount:
                s.sort()
                os.remove(s[0])
        #print "%s -> %s" % (self.baseFilename, dfn)
        if self.encoding:
            self.stream = codecs.open(self.baseFilename, 'w', self.encoding)
        else:
            self.stream = open(self.baseFilename, 'w')
        self.rolloverAt = self.rolloverAt + self.interval
        if os.path.exists(dfn + ".zip"):
            os.remove(dfn + ".zip")
        file = zipfile.ZipFile(dfn + ".zip", "w")
        file.write(dfn, os.path.basename(dfn), zipfile.ZIP_DEFLATED)
        file.close()
        os.remove(dfn)

if __name__=='__main__':
    ## Demo of using TimedCompressedRotatingFileHandler() to log every 5 seconds,
    ##     to one uncompressed file and five rotated and compressed files

    os.nice(19)   # I always nice test code

    logHandler = TimedCompressedRotatingFileHandler("mylog", when="S",
        interval=5, backupCount=5) # Total of six rotated log files, rotating every 5 secs
    logFormatter = logging.Formatter(
        fmt='%(asctime)s.%(msecs)03d %(message)s', 
        datefmt='%Y-%m-%d %H:%M:%S'
        )
    logHandler.setFormatter(logFormatter)
    mylogger = logging.getLogger('MyLogRef')
    mylogger.addHandler(logHandler)
    mylogger.setLevel(logging.DEBUG)

    # Write lines non-stop into the logger and rotate every 5 seconds
    ii = 0
    while True:
        mylogger.debug("Test {0}".format(ii))
        ii += 1

Answer 3

Soyez averti : les signatures de classe ont changé dans python 3. Voici mon exemple de travail pour python 3.6

 import logging.handlers
import os
import zlib


def namer(name):
    return name + ".gz"


def rotator(source, dest):
    print(f'compressing {source} -> {dest}')
    with open(source, "rb") as sf:
        data = sf.read()
        compressed = zlib.compress(data, 9)
        with open(dest, "wb") as df:
            df.write(compressed)
    os.remove(source)


err_handler = logging.handlers.TimedRotatingFileHandler('/data/errors.log', when="M", interval=1,
                                                        encoding='utf-8', backupCount=30, utc=True)
err_handler.rotator = rotator
err_handler.namer = namer

logger = logging.getLogger("Rotating Log")
logger.setLevel(logging.ERROR)

logger.addHandler(err_handler)

Answer 4

Je suppose qu'il est trop tard pour rejoindre la fête, mais voici ce que j'ai fait. J'ai créé une nouvelle classe héritant de la classe logging.handlers.RotatingFileHandler et ajouté quelques lignes pour compresser le fichier avant de le déplacer.

https://github.com/rkreddy46/python_code_reference/blob/master/compressed_log_rotator.py

 #!/usr/bin/env python

# Import all the needed modules
import logging.handlers
import sys
import time
import gzip
import os
import shutil
import random
import string

__version__ = 1.0
__descr__ = "This logic is written keeping in mind UNIX/LINUX/OSX platforms only"


# Create a new class that inherits from RotatingFileHandler. This is where we add the new feature to compress the logs
class CompressedRotatingFileHandler(logging.handlers.RotatingFileHandler):
    def doRollover(self):
        """
        Do a rollover, as described in __init__().
        """
        if self.stream:
            self.stream.close()
        if self.backupCount > 0:
            for i in range(self.backupCount - 1, 0, -1):
                sfn = "%s.%d.gz" % (self.baseFilename, i)
                dfn = "%s.%d.gz" % (self.baseFilename, i + 1)
                if os.path.exists(sfn):
                    # print "%s -> %s" % (sfn, dfn)
                    if os.path.exists(dfn):
                        os.remove(dfn)
                    os.rename(sfn, dfn)
            dfn = self.baseFilename + ".1.gz"
            if os.path.exists(dfn):
                os.remove(dfn)
            # These two lines below are the only new lines. I commented out the os.rename(self.baseFilename, dfn) and
            #  replaced it with these two lines.
            with open(self.baseFilename, 'rb') as f_in, gzip.open(dfn, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
            # os.rename(self.baseFilename, dfn)
            # print "%s -> %s" % (self.baseFilename, dfn)
        self.mode = 'w'
        self.stream = self._open()

# Specify which file will be used for our logs
log_filename = "/Users/myname/Downloads/test_logs/sample_log.txt"

# Create a logger instance and set the facility level
my_logger = logging.getLogger()
my_logger.setLevel(logging.DEBUG)

# Create a handler using our new class that rotates and compresses
file_handler = CompressedRotatingFileHandler(filename=log_filename, maxBytes=1000000, backupCount=10)

# Create a stream handler that shows the same log on the terminal (just for debug purposes)
view_handler = logging.StreamHandler(stream=sys.stdout)

# Add all the handlers to the logging instance
my_logger.addHandler(file_handler)
my_logger.addHandler(view_handler)

# This is optional to beef up the logs
random_huge_data = "".join(random.choice(string.ascii_letters) for _ in xrange(10000))

# All this code is user-specific, write your own code if you want to play around
count = 0
while True:
    my_logger.debug("This is the message number %s" % str(count))
    my_logger.debug(random_huge_data)
    count += 1
    if count % 100 == 0:
        count = 0
        time.sleep(2)