From e756e0dbc3d79634c9058f05fb401a471b983c3e Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Fri, 9 Sep 2022 17:09:54 +0200 Subject: [PATCH 01/10] Common: Implement EventSerializer for pydantic objects --- monkey/common/event_serializers/__init__.py | 2 +- .../pydantic_event_serializer.py | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 monkey/common/event_serializers/pydantic_event_serializer.py diff --git a/monkey/common/event_serializers/__init__.py b/monkey/common/event_serializers/__init__.py index 2b60471b1..c335fbc58 100644 --- a/monkey/common/event_serializers/__init__.py +++ b/monkey/common/event_serializers/__init__.py @@ -1,2 +1,2 @@ -from .i_event_serialize import IEventSerializer +from .i_event_serialize import IEventSerializer, JSONSerializable from .event_serializer_registry import EventSerializerRegistry diff --git a/monkey/common/event_serializers/pydantic_event_serializer.py b/monkey/common/event_serializers/pydantic_event_serializer.py new file mode 100644 index 000000000..63332f00c --- /dev/null +++ b/monkey/common/event_serializers/pydantic_event_serializer.py @@ -0,0 +1,28 @@ +import logging +from typing import Type + +from common.base_models import InfectionMonkeyBaseModel +from common.events import AbstractAgentEvent + +from . import IEventSerializer, JSONSerializable + +logger = logging.getLogger(__name__) + + +class PydanticEventSerializer(IEventSerializer): + def __init__(self, event_class: Type[AbstractAgentEvent]): + self._event_class = event_class + + def serialize(self, event: AbstractAgentEvent) -> JSONSerializable: + if not issubclass(event.__class__, self._event_class): + raise TypeError(f"Event object must be of type: {InfectionMonkeyBaseModel.__name__}") + + try: + return event.dict() + except AttributeError as err: + logger.error(f"Error occured while serializing an event {event}: {err}") + + return None + + def deserialize(self, serialized_event: JSONSerializable) -> AbstractAgentEvent: + return self._event_class.parse_obj(serialized_event) From 1755d72ae78281937c66986fb8c67813a9551e57 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Fri, 9 Sep 2022 17:10:34 +0200 Subject: [PATCH 02/10] Common: Export PydanticEventSerializer from __init__ --- monkey/common/event_serializers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monkey/common/event_serializers/__init__.py b/monkey/common/event_serializers/__init__.py index c335fbc58..1adb5e3dd 100644 --- a/monkey/common/event_serializers/__init__.py +++ b/monkey/common/event_serializers/__init__.py @@ -1,2 +1,3 @@ from .i_event_serialize import IEventSerializer, JSONSerializable from .event_serializer_registry import EventSerializerRegistry +from .pydantic_event_serializer import PydanticEventSerializer From 9e9160304cbba52c2224c022cb42f612406823f7 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Fri, 9 Sep 2022 17:11:01 +0200 Subject: [PATCH 03/10] UT: Add unit tests for PydanticEventSerializer --- .../test_pydantic_event_serializer.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 monkey/tests/unit_tests/common/event_serializers/test_pydantic_event_serializer.py diff --git a/monkey/tests/unit_tests/common/event_serializers/test_pydantic_event_serializer.py b/monkey/tests/unit_tests/common/event_serializers/test_pydantic_event_serializer.py new file mode 100644 index 000000000..ce38f2680 --- /dev/null +++ b/monkey/tests/unit_tests/common/event_serializers/test_pydantic_event_serializer.py @@ -0,0 +1,50 @@ +from abc import ABC +from dataclasses import dataclass, field + +import pytest +from pydantic import ValidationError + +from common.base_models import InfectionMonkeyBaseModel +from common.event_serializers import PydanticEventSerializer +from common.events import AbstractAgentEvent + + +@dataclass(frozen=True) +class NotAgentEvent(ABC): + some_field: int + other_field: float + + +@dataclass(frozen=True) +class SomeAgentEvent(AbstractAgentEvent): + bogus: int = field(default_factory=int) + + +class PydanticEvent(InfectionMonkeyBaseModel): + some_field: str + + +@pytest.fixture +def pydantic_event_serializer(): + return PydanticEventSerializer(PydanticEvent) + + +@pytest.mark.parametrize("event", [NotAgentEvent(1, 2.0), SomeAgentEvent(2)]) +def test_pydantic_event_serializer__serialize_wrong_type(pydantic_event_serializer, event): + with pytest.raises(TypeError): + pydantic_event_serializer.serialize(event) + + +def test_pydantic_event_serializer__deserialize_wrong_type(pydantic_event_serializer): + with pytest.raises(ValidationError): + pydantic_event_serializer.deserialize("bla") + + +def test_pydanitc_event_serializer__de_serialize(pydantic_event_serializer): + pydantic_event = PydanticEvent(some_field="some_field") + + serialized_event = pydantic_event_serializer.serialize(pydantic_event) + deserialized_object = pydantic_event_serializer.deserialize(serialized_event) + + assert type(serialized_event) != type(deserialized_object) + assert deserialized_object == pydantic_event From 88d65f40aef079a96f0c328633ecb0e060160dff Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Mon, 12 Sep 2022 15:15:00 +0200 Subject: [PATCH 04/10] Project: Add PydanticEventSerializer to vulture_allowlist --- vulture_allowlist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vulture_allowlist.py b/vulture_allowlist.py index a8369b7da..c77390784 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -298,6 +298,7 @@ serialize event deserialize serialized_event +PydanticEventSerializer # pydantic base models underscore_attrs_are_private From 62ab6e5a778f3e6f7fc628179b469a7226445a95 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Tue, 13 Sep 2022 13:36:38 +0200 Subject: [PATCH 05/10] Common: Add generic events to PydanticEventSerializer --- .../pydantic_event_serializer.py | 15 ++++++++------- .../test_pydantic_event_serializer.py | 7 +++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/monkey/common/event_serializers/pydantic_event_serializer.py b/monkey/common/event_serializers/pydantic_event_serializer.py index 63332f00c..fab4f8c06 100644 --- a/monkey/common/event_serializers/pydantic_event_serializer.py +++ b/monkey/common/event_serializers/pydantic_event_serializer.py @@ -1,21 +1,22 @@ import logging -from typing import Type +from typing import Type, TypeVar -from common.base_models import InfectionMonkeyBaseModel from common.events import AbstractAgentEvent from . import IEventSerializer, JSONSerializable logger = logging.getLogger(__name__) +T = TypeVar("T", bound=AbstractAgentEvent) + class PydanticEventSerializer(IEventSerializer): - def __init__(self, event_class: Type[AbstractAgentEvent]): + def __init__(self, event_class: Type[T]): self._event_class = event_class - def serialize(self, event: AbstractAgentEvent) -> JSONSerializable: + def serialize(self, event: T) -> JSONSerializable: if not issubclass(event.__class__, self._event_class): - raise TypeError(f"Event object must be of type: {InfectionMonkeyBaseModel.__name__}") + raise TypeError(f"Event object must be of type: {self._event_class.__name__}") try: return event.dict() @@ -24,5 +25,5 @@ class PydanticEventSerializer(IEventSerializer): return None - def deserialize(self, serialized_event: JSONSerializable) -> AbstractAgentEvent: - return self._event_class.parse_obj(serialized_event) + def deserialize(self, serialized_event: JSONSerializable) -> T: + return self._event_class(**serialized_event) diff --git a/monkey/tests/unit_tests/common/event_serializers/test_pydantic_event_serializer.py b/monkey/tests/unit_tests/common/event_serializers/test_pydantic_event_serializer.py index ce38f2680..e3cb7a561 100644 --- a/monkey/tests/unit_tests/common/event_serializers/test_pydantic_event_serializer.py +++ b/monkey/tests/unit_tests/common/event_serializers/test_pydantic_event_serializer.py @@ -2,10 +2,9 @@ from abc import ABC from dataclasses import dataclass, field import pytest -from pydantic import ValidationError from common.base_models import InfectionMonkeyBaseModel -from common.event_serializers import PydanticEventSerializer +from common.event_serializers import IEventSerializer, PydanticEventSerializer from common.events import AbstractAgentEvent @@ -25,7 +24,7 @@ class PydanticEvent(InfectionMonkeyBaseModel): @pytest.fixture -def pydantic_event_serializer(): +def pydantic_event_serializer() -> IEventSerializer: return PydanticEventSerializer(PydanticEvent) @@ -36,7 +35,7 @@ def test_pydantic_event_serializer__serialize_wrong_type(pydantic_event_serializ def test_pydantic_event_serializer__deserialize_wrong_type(pydantic_event_serializer): - with pytest.raises(ValidationError): + with pytest.raises(TypeError): pydantic_event_serializer.deserialize("bla") From 34e843f7f36814636dd8dbc7558341924bd6d954 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 13 Sep 2022 07:50:38 -0400 Subject: [PATCH 06/10] Common: Make PydanticEventSerializer generic --- monkey/common/event_serializers/pydantic_event_serializer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/common/event_serializers/pydantic_event_serializer.py b/monkey/common/event_serializers/pydantic_event_serializer.py index fab4f8c06..87ac201f8 100644 --- a/monkey/common/event_serializers/pydantic_event_serializer.py +++ b/monkey/common/event_serializers/pydantic_event_serializer.py @@ -1,5 +1,5 @@ import logging -from typing import Type, TypeVar +from typing import Generic, Type, TypeVar from common.events import AbstractAgentEvent @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) T = TypeVar("T", bound=AbstractAgentEvent) -class PydanticEventSerializer(IEventSerializer): +class PydanticEventSerializer(IEventSerializer, Generic[T]): def __init__(self, event_class: Type[T]): self._event_class = event_class From 780595cf19804e0d788902201b2e59956de041e0 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 13 Sep 2022 07:53:30 -0400 Subject: [PATCH 07/10] Common: Use `simplify=True` in PydanticEventSerializer --- monkey/common/event_serializers/pydantic_event_serializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/common/event_serializers/pydantic_event_serializer.py b/monkey/common/event_serializers/pydantic_event_serializer.py index 87ac201f8..e1cb56142 100644 --- a/monkey/common/event_serializers/pydantic_event_serializer.py +++ b/monkey/common/event_serializers/pydantic_event_serializer.py @@ -19,7 +19,7 @@ class PydanticEventSerializer(IEventSerializer, Generic[T]): raise TypeError(f"Event object must be of type: {self._event_class.__name__}") try: - return event.dict() + return event.dict(simplify=True) except AttributeError as err: logger.error(f"Error occured while serializing an event {event}: {err}") From 6c0b63aa295f55951d9dc43356115158a5183c41 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 13 Sep 2022 07:55:15 -0400 Subject: [PATCH 08/10] Common: Don't hide AttributeError from the caller --- .../common/event_serializers/pydantic_event_serializer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/monkey/common/event_serializers/pydantic_event_serializer.py b/monkey/common/event_serializers/pydantic_event_serializer.py index e1cb56142..692a3cea8 100644 --- a/monkey/common/event_serializers/pydantic_event_serializer.py +++ b/monkey/common/event_serializers/pydantic_event_serializer.py @@ -18,12 +18,7 @@ class PydanticEventSerializer(IEventSerializer, Generic[T]): if not issubclass(event.__class__, self._event_class): raise TypeError(f"Event object must be of type: {self._event_class.__name__}") - try: - return event.dict(simplify=True) - except AttributeError as err: - logger.error(f"Error occured while serializing an event {event}: {err}") - - return None + return event.dict(simplify=True) def deserialize(self, serialized_event: JSONSerializable) -> T: return self._event_class(**serialized_event) From d3a4f255f0f74011ab57e94b05bbf28d16bb3a20 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Tue, 13 Sep 2022 14:09:54 +0200 Subject: [PATCH 09/10] Common: Use isinstance in PydanticEventSerializer --- monkey/common/event_serializers/pydantic_event_serializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/common/event_serializers/pydantic_event_serializer.py b/monkey/common/event_serializers/pydantic_event_serializer.py index 692a3cea8..263e3463b 100644 --- a/monkey/common/event_serializers/pydantic_event_serializer.py +++ b/monkey/common/event_serializers/pydantic_event_serializer.py @@ -15,7 +15,7 @@ class PydanticEventSerializer(IEventSerializer, Generic[T]): self._event_class = event_class def serialize(self, event: T) -> JSONSerializable: - if not issubclass(event.__class__, self._event_class): + if not isinstance(event, self._event_class): raise TypeError(f"Event object must be of type: {self._event_class.__name__}") return event.dict(simplify=True) From 69e11d6b503c86f8988c835a2e835df7e7875c6d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 13 Sep 2022 08:35:43 -0400 Subject: [PATCH 10/10] Common: Ignore mypy cyclical definition warning for JSONSerializable --- monkey/common/event_serializers/i_event_serialize.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/monkey/common/event_serializers/i_event_serialize.py b/monkey/common/event_serializers/i_event_serialize.py index 75d95be3c..a092946ab 100644 --- a/monkey/common/event_serializers/i_event_serialize.py +++ b/monkey/common/event_serializers/i_event_serialize.py @@ -3,8 +3,14 @@ from typing import Dict, List, Union from common.events import AbstractAgentEvent -JSONSerializable = Union[ - Dict[str, "JSONSerializable"], List["JSONSerializable"], int, str, float, bool, None +JSONSerializable = Union[ # type: ignore[misc] + Dict[str, "JSONSerializable"], # type: ignore[misc] + List["JSONSerializable"], # type: ignore[misc] + int, + str, + float, + bool, + None, ]