security_vision/trap
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Threaded logging to avoid pauses in code

Browse source
This commit is contained in:
Ruben van de Ven 2024-11-12 21:36:37 +01:00
parent f2d71a9da3
commit a2ced9646f
2 changed files with 304 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

279
test_tracking_data.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

37
trap/plumber.py
View file

@ -1,6 +1,6 @@
import atexit import atexit
import logging import logging
from logging.handlers import SocketHandler from logging.handlers import SocketHandler, QueueHandler, QueueListener
from multiprocessing import Event, Process, Queue from multiprocessing import Event, Process, Queue
import multiprocessing import multiprocessing
import signal import signal
@ -18,6 +18,7 @@ from trap.tracker import run_tracker
logger = logging.getLogger("trap.plumbing") logger = logging.getLogger("trap.plumbing")
class ExceptionHandlingProcess(Process): class ExceptionHandlingProcess(Process):
def run(self): def run(self):
@ -45,25 +46,37 @@ def start():
loglevel = logging.NOTSET if args.verbose > 1 else logging.DEBUG if args.verbose > 0 else logging.INFO loglevel = logging.NOTSET if args.verbose > 1 else logging.DEBUG if args.verbose > 0 else logging.INFO
# print(args) # print(args)
# exit() # exit()
logging.basicConfig(
level=loglevel,
)
# set per handler, so we can set it lower for the root logger if remote logging is enabled
root_logger = logging.getLogger()
[h.setLevel(loglevel) for h in root_logger.handlers]
isRunning = Event() isRunning = Event()
isRunning.set() isRunning.set()
q = multiprocessing.Queue(-1)
queue_handler = QueueHandler(q)
stream_handler = logging.StreamHandler()
log_handlers = [stream_handler]
if args.remote_log_addr: if args.remote_log_addr:
logging.captureWarnings(True) logging.captureWarnings(True)
root_logger.setLevel(logging.NOTSET) # to send all records to cutelog # root_logger.setLevel(logging.NOTSET) # to send all records to cutelog
socket_handler = SocketHandler(args.remote_log_addr, args.remote_log_port) socket_handler = SocketHandler(args.remote_log_addr, args.remote_log_port)
root_logger.addHandler(socket_handler) socket_handler.setLevel(logging.NOTSET)
log_handlers.append(socket_handler)
queue_listener = QueueListener(q, *log_handlers, respect_handler_level=True)
queue_listener.start()
# root = logging.getLogger()
logging.basicConfig(
level=loglevel,
handlers=[queue_handler]
)
# root_logger = logging.getLogger()
# # set per handler, so we can set it lower for the root logger if remote logging is enabled
# [h.setLevel(loglevel) for h in root_logger.handlers]
# queue_listener.handlers.append(socket_handler)