|
| 1 | +import os |
| 2 | +from collections import defaultdict |
| 3 | +import logging |
| 4 | +from colorlog import ColoredFormatter |
| 5 | + |
| 6 | +import pandas |
| 7 | +import numpy as np |
| 8 | + |
| 9 | +from tabulate import tabulate |
| 10 | + |
| 11 | + |
| 12 | +class LoggerClass(object): |
| 13 | + GLOBAL_LOGGER_NAME = '_global_logger' |
| 14 | + |
| 15 | + _color_formatter = ColoredFormatter( |
| 16 | + "%(asctime)s %(log_color)s%(name)-10s %(levelname)-8s%(reset)s %(white)s%(message)s", |
| 17 | + datefmt='%m-%d %H:%M:%S', |
| 18 | + reset=True, |
| 19 | + log_colors={ |
| 20 | + 'DEBUG': 'cyan', |
| 21 | + 'INFO': 'green', |
| 22 | + 'WARNING': 'yellow', |
| 23 | + 'ERROR': 'red', |
| 24 | + 'CRITICAL': 'red,bg_white', |
| 25 | + }, |
| 26 | + secondary_log_colors={}, |
| 27 | + style='%' |
| 28 | + ) |
| 29 | + |
| 30 | + _normal_formatter = logging.Formatter( |
| 31 | + '%(asctime)s %(name)-10s %(levelname)-8s %(message)s', |
| 32 | + datefmt='%m-%d %H:%M:%S', |
| 33 | + style='%' |
| 34 | + ) |
| 35 | + |
| 36 | + def __init__(self): |
| 37 | + self._dir = None |
| 38 | + self._logger = None |
| 39 | + self._log_path = None |
| 40 | + self._csv_path = None |
| 41 | + self._tabular = defaultdict(list) |
| 42 | + self._curr_recorded = list() |
| 43 | + self._num_dump_tabular_calls = 0 |
| 44 | + |
| 45 | + @property |
| 46 | + def dir(self): |
| 47 | + return self._dir |
| 48 | + |
| 49 | + ############# |
| 50 | + ### Setup ### |
| 51 | + ############# |
| 52 | + |
| 53 | + def setup(self, display_name, log_path, lvl): |
| 54 | + self._dir = os.path.dirname(log_path) |
| 55 | + self._logger = self._get_logger(LoggerClass.GLOBAL_LOGGER_NAME, |
| 56 | + log_path, |
| 57 | + lvl=lvl, |
| 58 | + display_name=display_name) |
| 59 | + self._csv_path = os.path.splitext(log_path)[0] + '.csv' |
| 60 | + |
| 61 | + ### load csv if exists |
| 62 | + if os.path.exists(self._csv_path): |
| 63 | + self._tabular = {k: list(v) for k, v in pandas.read_csv(self._csv_path).items()} |
| 64 | + self._num_dump_tabular_calls = len(tuple(self._tabular.values())[0]) |
| 65 | + |
| 66 | + def _get_logger(self, name, log_path, lvl=logging.INFO, display_name=None): |
| 67 | + if isinstance(lvl, str): |
| 68 | + lvl = lvl.lower().strip() |
| 69 | + if lvl == 'debug': |
| 70 | + lvl = logging.DEBUG |
| 71 | + elif lvl == 'info': |
| 72 | + lvl = logging.INFO |
| 73 | + elif lvl == 'warn' or lvl == 'warning': |
| 74 | + lvl = logging.WARN |
| 75 | + elif lvl == 'error': |
| 76 | + lvl = logging.ERROR |
| 77 | + elif lvl == 'fatal' or lvl == 'critical': |
| 78 | + lvl = logging.CRITICAL |
| 79 | + else: |
| 80 | + raise ValueError('unknown logging level') |
| 81 | + |
| 82 | + file_handler = logging.FileHandler(log_path) |
| 83 | + file_handler.setLevel(logging.DEBUG) |
| 84 | + file_handler.setFormatter(LoggerClass._normal_formatter) |
| 85 | + console_handler = logging.StreamHandler() |
| 86 | + console_handler.setLevel(lvl) |
| 87 | + console_handler.setFormatter(LoggerClass._color_formatter) |
| 88 | + if display_name is None: |
| 89 | + display_name = name |
| 90 | + logger = logging.getLogger(display_name) |
| 91 | + logger.setLevel(logging.DEBUG) |
| 92 | + logger.addHandler(console_handler) |
| 93 | + logger.addHandler(file_handler) |
| 94 | + |
| 95 | + return logger |
| 96 | + |
| 97 | + ############### |
| 98 | + ### Logging ### |
| 99 | + ############### |
| 100 | + |
| 101 | + def debug(self, s): |
| 102 | + assert (self._logger is not None) |
| 103 | + self._logger.debug(s) |
| 104 | + |
| 105 | + def info(self, s): |
| 106 | + assert (self._logger is not None) |
| 107 | + self._logger.info(s) |
| 108 | + |
| 109 | + def warn(self, s): |
| 110 | + assert (self._logger is not None) |
| 111 | + self._logger.warn(s) |
| 112 | + |
| 113 | + def error(self, s): |
| 114 | + assert (self._logger is not None) |
| 115 | + self._logger.error(s) |
| 116 | + |
| 117 | + def critical(self, s): |
| 118 | + assert (self._logger is not None) |
| 119 | + self._logger.critical(s) |
| 120 | + |
| 121 | + #################### |
| 122 | + ### Data logging ### |
| 123 | + #################### |
| 124 | + |
| 125 | + def record_tabular(self, key, val): |
| 126 | + assert (str(key) not in self._curr_recorded) |
| 127 | + self._curr_recorded.append(str(key)) |
| 128 | + |
| 129 | + if key in self._tabular: |
| 130 | + self._tabular[key].append(val) |
| 131 | + else: |
| 132 | + self._tabular[key] = [np.nan] * self._num_dump_tabular_calls + [val] |
| 133 | + |
| 134 | + def dump_tabular(self, print_func=None): |
| 135 | + if len(self._curr_recorded) == 0: |
| 136 | + return '' |
| 137 | + |
| 138 | + ### reset |
| 139 | + self._curr_recorded = list() |
| 140 | + self._num_dump_tabular_calls += 1 |
| 141 | + |
| 142 | + ### make sure all same length |
| 143 | + for k, v in self._tabular.items(): |
| 144 | + if len(v) == self._num_dump_tabular_calls: |
| 145 | + pass |
| 146 | + elif len(v) == self._num_dump_tabular_calls - 1: |
| 147 | + self._tabular[k].append(np.nan) |
| 148 | + else: |
| 149 | + raise ValueError('key {0} should not have {1} items when {2} calls have been made'.format( |
| 150 | + k, len(v), self._num_dump_tabular_calls)) |
| 151 | + |
| 152 | + ### print |
| 153 | + if print_func is not None: |
| 154 | + log_str = tabulate(sorted([(k, v[-1]) for k, v in self._tabular.items()], key=lambda kv: kv[0])) |
| 155 | + for line in log_str.split('\n'): |
| 156 | + print_func(line) |
| 157 | + |
| 158 | + ### write to file |
| 159 | + tabular_pandas = pandas.DataFrame({k: pandas.Series(v) for k, v in self._tabular.items()}) |
| 160 | + tabular_pandas.to_csv(self._csv_path) |
| 161 | + |
| 162 | + |
| 163 | +logger = LoggerClass() |
0 commit comments