security_vision/trap
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix remote logging

Browse source
This commit is contained in:
Ruben van de Ven 2025-05-19 14:11:56 +02:00
parent 98b8f0977e
commit 010775f5d5
6 changed files with 153 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
pyproject.toml
View file

@ -34,6 +34,7 @@ dependencies = [
"facenet-pytorch>=2.5.3", "facenet-pytorch>=2.5.3",
"simplification>=0.7.12", "simplification>=0.7.12",
"supervisor>=4.2.5", "supervisor>=4.2.5",
"superfsmon>=1.2.3",
] ]
[project.scripts] [project.scripts]

8
supervisord.conf
View file

@ -4,7 +4,7 @@ port = *:8293
# password = 123 # password = 123
[supervisord] [supervisord]
nodaemon = True nodaemon = false
; The rpcinterface:supervisor section must remain in the config file for ; The rpcinterface:supervisor section must remain in the config file for
@ -46,3 +46,9 @@ environment=DISPLAY=":0"
autostart=false autostart=false
; can be long to quit if rendering to video file ; can be long to quit if rendering to video file
stopwaitsecs=60 stopwaitsecs=60
# during development auto restart some services when the code changes
[program:superfsmon]
command=superfsmon trap/stage.py stage
directory=%(here)s

37
test_training.ipynb
View file

@ -36,7 +36,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -151,7 +151,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -161,7 +161,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -187,7 +187,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": 5,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -196,7 +196,34 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(tracks)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [

51
trap/node.py
View file

@ -1,4 +1,5 @@
import logging import logging
from logging.handlers import QueueHandler, QueueListener, SocketHandler
import multiprocessing import multiprocessing
from multiprocessing.synchronize import Event as BaseEvent from multiprocessing.synchronize import Event as BaseEvent
from argparse import ArgumentParser, Namespace from argparse import ArgumentParser, Namespace
@ -45,6 +46,31 @@ class Node():
@classmethod @classmethod
def arg_parser(cls) -> ArgumentParser: def arg_parser(cls) -> ArgumentParser:
raise RuntimeError("Not implemented arg_parser()") raise RuntimeError("Not implemented arg_parser()")
@classmethod
def _get_arg_parser(cls) -> ArgumentParser:
parser = cls.arg_parser()
# add some defaults
parser.add_argument(
'--verbose',
'-v',
help="Increase verbosity. Add multiple times to increase further.",
action='count', default=0
)
parser.add_argument(
'--remote-log-addr',
help="Connect to a remote logger like cutelog. Specify the ip",
type=str,
default="100.72.38.82"
)
parser.add_argument(
'--remote-log-port',
help="Connect to a remote logger like cutelog. Specify the port",
type=int,
default=19996
)
return parser
def sub(self, addr: str): def sub(self, addr: str):
"Default zmq sub configuration" "Default zmq sub configuration"
@ -69,11 +95,32 @@ class Node():
@classmethod @classmethod
def parse_and_start(cls): def parse_and_start(cls):
config = cls.arg_parser().parse_args() """To start the node from CLI/supervisor"""
config = cls._get_arg_parser().parse_args()
setup_logging(config) # running from cli, we need to setup logging
is_running = multiprocessing.Event() is_running = multiprocessing.Event()
is_running.set() is_running.set()
statsender = CounterSender() statsender = CounterSender()
counter = CounterFpsSender(f"trap.{cls.__name__}", statsender) counter = CounterFpsSender(f"trap.{cls.__name__}", statsender)
# timer_counter = Timer(cls.__name__) # timer_counter = Timer(cls.__name__)
cls.start(config, is_running, counter) cls.start(config, is_running, counter)
def setup_logging(config: Namespace):
loglevel = logging.NOTSET if config.verbose > 1 else logging.DEBUG if config.verbose > 0 else logging.INFO
stream_handler = logging.StreamHandler()
log_handlers = [stream_handler]
if config.remote_log_addr:
logging.captureWarnings(True)
# root_logger.setLevel(logging.NOTSET) # to send all records to cutelog
socket_handler = SocketHandler(config.remote_log_addr, config.remote_log_port)
print(socket_handler.host, socket_handler.port)
socket_handler.setLevel(logging.NOTSET)
log_handlers.append(socket_handler)
logging.basicConfig(
level=loglevel,
handlers=log_handlers # [queue_handler]
)

14
trap/stage.py
View file

@ -35,10 +35,6 @@ logger = logging.getLogger('trap.stage')
Coordinate = Tuple[float, float] Coordinate = Tuple[float, float]
DeltaT = float # delta_t in seconds DeltaT = float # delta_t in seconds
# current_fraction = line_locate_point(new_line_string, Point(old_ls.coords[-1]), normalized=True)
# new_fraction = current_fraction + stepsize
# grown_string = shapely.ops.substring(new_line_string, 0, new_fraction, normalized=True)
class LineGenerator(ABC): class LineGenerator(ABC):
@abstractmethod @abstractmethod
def update_drawn_positions(self, dt: DeltaT): def update_drawn_positions(self, dt: DeltaT):
@ -60,6 +56,7 @@ class AppendableLine(LineGenerator):
self.ready = len(self.points) == 0 self.ready = len(self.points) == 0
self.draw_decay_speed = draw_decay_speed self.draw_decay_speed = draw_decay_speed
def update_drawn_positions(self, dt: DeltaT): def update_drawn_positions(self, dt: DeltaT):
if len(self.points) == 0: if len(self.points) == 0:
# nothing to draw yet # nothing to draw yet
@ -90,6 +87,8 @@ class AppendableLine(LineGenerator):
self._drawn_points[-1] = (float(x), float(y)) self._drawn_points[-1] = (float(x), float(y))
class ProceduralChain(LineGenerator): class ProceduralChain(LineGenerator):
"""A line that can be 'dragged' to a target. In which
it disappears."""
MOVE_DECAY_SPEED = 80 # speed at which the drawing head should approach the next point MOVE_DECAY_SPEED = 80 # speed at which the drawing head should approach the next point
VELOCITY_DAMPING = 10 VELOCITY_DAMPING = 10
VELOCITY_FACTOR = 2 VELOCITY_FACTOR = 2
@ -157,6 +156,13 @@ class ProceduralChain(LineGenerator):
class DiffSegment(): class DiffSegment():
"""
A segment of a prediction track, that can be diffed
with a track. The track is continously update.
If a new prediction comes in, the diff is marked as
finished. After which it is animated and added to the
Scenario's anomaly score.
"""
DRAW_DECAY_SPEED = 25 DRAW_DECAY_SPEED = 25
def __init__(self, prediction: ProjectedTrack): def __init__(self, prediction: ProjectedTrack):

58
uv.lock
View file

@ -1824,12 +1824,11 @@ wheels = [
[[package]] [[package]]
name = "pywin32" name = "pywin32"
version = "310" version = "306"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/95/da/a5f38fffbba2fb99aa4aa905480ac4b8e83ca486659ac8c95bce47fb5276/pywin32-310-cp310-cp310-win32.whl", hash = "sha256:6dd97011efc8bf51d6793a82292419eba2c71cf8e7250cfac03bba284454abc1", size = 8848240 }, { url = "https://files.pythonhosted.org/packages/08/dc/28c668097edfaf4eac4617ef7adf081b9cf50d254672fcf399a70f5efc41/pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d", size = 8506422 },
{ url = "https://files.pythonhosted.org/packages/aa/fe/d873a773324fa565619ba555a82c9dabd677301720f3660a731a5d07e49a/pywin32-310-cp310-cp310-win_amd64.whl", hash = "sha256:c3e78706e4229b915a0821941a84e7ef420bf2b77e08c9dae3c76fd03fd2ae3d", size = 9601854 }, { url = "https://files.pythonhosted.org/packages/d3/d6/891894edec688e72c2e308b3243fad98b4066e1839fd2fe78f04129a9d31/pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8", size = 9226392 },
{ url = "https://files.pythonhosted.org/packages/3c/84/1a8e3d7a15490d28a5d816efa229ecb4999cdc51a7c30dd8914f669093b8/pywin32-310-cp310-cp310-win_arm64.whl", hash = "sha256:33babed0cf0c92a6f94cc6cc13546ab24ee13e3e800e61ed87609ab91e4c8213", size = 8522963 },
] ]
[[package]] [[package]]
@ -2191,6 +2190,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a0/4b/528ccf7a982216885a1ff4908e886b8fb5f19862d1962f56a3fce2435a70/starlette-0.46.1-py3-none-any.whl", hash = "sha256:77c74ed9d2720138b25875133f3a2dae6d854af2ec37dceb56aef370c1d8a227", size = 71995 }, { url = "https://files.pythonhosted.org/packages/a0/4b/528ccf7a982216885a1ff4908e886b8fb5f19862d1962f56a3fce2435a70/starlette-0.46.1-py3-none-any.whl", hash = "sha256:77c74ed9d2720138b25875133f3a2dae6d854af2ec37dceb56aef370c1d8a227", size = 71995 },
] ]
[[package]]
name = "superfsmon"
version = "1.2.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "supervisor", marker = "sys_platform != 'win32'" },
{ name = "supervisor-win", marker = "sys_platform == 'win32'" },
{ name = "watchdog" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e9/c2/269264babce3c29f5721cdb7c79ab4930562b67786bb6e5cc838e36e3530/superfsmon-1.2.3.tar.gz", hash = "sha256:fe5918872dc258eacff98cd054b28b73531f9897f72f8583fb2bbd448fc33928", size = 5186 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/bd/c5/d8fbf5c3901db69f7b1e25708fc865570712264026d06f75c5d535ec4ab1/superfsmon-1.2.3-py3-none-any.whl", hash = "sha256:da798e2a2c260fa633213df9f2f26d504fe234f78886e5f62ae4d81f0130bdf7", size = 4738 },
]
[[package]] [[package]]
name = "supervisor" name = "supervisor"
version = "4.2.5" version = "4.2.5"
@ -2203,6 +2216,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2c/7a/0ad3973941590c040475046fef37a2b08a76691e61aa59540828ee235a6e/supervisor-4.2.5-py2.py3-none-any.whl", hash = "sha256:2ecaede32fc25af814696374b79e42644ecaba5c09494c51016ffda9602d0f08", size = 319561 }, { url = "https://files.pythonhosted.org/packages/2c/7a/0ad3973941590c040475046fef37a2b08a76691e61aa59540828ee235a6e/supervisor-4.2.5-py2.py3-none-any.whl", hash = "sha256:2ecaede32fc25af814696374b79e42644ecaba5c09494c51016ffda9602d0f08", size = 319561 },
] ]
[[package]]
name = "supervisor-win"
version = "4.7.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pywin32", marker = "sys_platform == 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/98/48/5d6cd1b7492bf2c11452fd638de45519d2c103caed70c5bdb4ecebbac568/supervisor-win-4.7.0.tar.gz", hash = "sha256:c474d92edc7050b55adae2f7c7789d5d69f180dee7868a27673b1d38f8bea484", size = 397342 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/69/4d/3a493f15f5b80608857ef157f382ace494f51d9031e6bee6082437dd1403/supervisor_win-4.7.0-py2.py3-none-any.whl", hash = "sha256:bd98554c2a0878704c3f3fd95e38965d9986eae6a2ad29f34d73d0aee138a481", size = 303996 },
]
[[package]] [[package]]
name = "tensorboard" name = "tensorboard"
version = "2.19.0" version = "2.19.0"
@ -2507,6 +2532,7 @@ dependencies = [
{ name = "setproctitle" }, { name = "setproctitle" },
{ name = "shapely" }, { name = "shapely" },
{ name = "simplification" }, { name = "simplification" },
{ name = "superfsmon" },
{ name = "supervisor" }, { name = "supervisor" },
{ name = "tensorboardx" }, { name = "tensorboardx" },
{ name = "torch", version = "1.12.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, { name = "torch", version = "1.12.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" },
@ -2540,6 +2566,7 @@ requires-dist = [
{ name = "setproctitle", specifier = ">=1.3.3,<2" }, { name = "setproctitle", specifier = ">=1.3.3,<2" },
{ name = "shapely", specifier = ">=2.1" }, { name = "shapely", specifier = ">=2.1" },
{ name = "simplification", specifier = ">=0.7.12" }, { name = "simplification", specifier = ">=0.7.12" },
{ name = "superfsmon", specifier = ">=1.2.3" },
{ name = "supervisor", specifier = ">=4.2.5" }, { name = "supervisor", specifier = ">=4.2.5" },
{ name = "tensorboardx", specifier = ">=2.6.2.2,<3" }, { name = "tensorboardx", specifier = ">=2.6.2.2,<3" },
{ name = "torch", marker = "python_full_version < '3.10' or python_full_version >= '4' or sys_platform != 'linux'", specifier = "==1.12.1" }, { name = "torch", marker = "python_full_version < '3.10' or python_full_version >= '4' or sys_platform != 'linux'", specifier = "==1.12.1" },
@ -2704,6 +2731,29 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a6/3d/7b22abbdb059d551507275a2815bc2b1974e3b9f6a13781c1eac9e858965/vbuild-0.8.2-py2.py3-none-any.whl", hash = "sha256:d76bcc976a1c53b6a5776ac947606f9e7786c25df33a587ebe33ed09dd8a1076", size = 9371 }, { url = "https://files.pythonhosted.org/packages/a6/3d/7b22abbdb059d551507275a2815bc2b1974e3b9f6a13781c1eac9e858965/vbuild-0.8.2-py2.py3-none-any.whl", hash = "sha256:d76bcc976a1c53b6a5776ac947606f9e7786c25df33a587ebe33ed09dd8a1076", size = 9371 },
] ]
[[package]]
name = "watchdog"
version = "6.0.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/0c/56/90994d789c61df619bfc5ce2ecdabd5eeff564e1eb47512bd01b5e019569/watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26", size = 96390 },
{ url = "https://files.pythonhosted.org/packages/55/46/9a67ee697342ddf3c6daa97e3a587a56d6c4052f881ed926a849fcf7371c/watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112", size = 88389 },
{ url = "https://files.pythonhosted.org/packages/44/65/91b0985747c52064d8701e1075eb96f8c40a79df889e59a399453adfb882/watchdog-6.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3", size = 89020 },
{ url = "https://files.pythonhosted.org/packages/30/ad/d17b5d42e28a8b91f8ed01cb949da092827afb9995d4559fd448d0472763/watchdog-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881", size = 87902 },
{ url = "https://files.pythonhosted.org/packages/5c/ca/c3649991d140ff6ab67bfc85ab42b165ead119c9e12211e08089d763ece5/watchdog-6.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11", size = 88380 },
{ url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079 },
{ url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078 },
{ url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076 },
{ url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077 },
{ url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078 },
{ url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077 },
{ url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078 },
{ url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065 },
{ url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070 },
{ url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067 },
]
[[package]] [[package]]
name = "watchfiles" name = "watchfiles"
version = "1.0.4" version = "1.0.4"