Skip to content

Wrap dec_hook errors in ValidationError #460

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jcrist merged 1 commit into from
Jun 30, 2023
Merged

Wrap dec_hook errors in ValidationError #460

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 16 additions & 11 deletions docs/source/extending.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,12 @@ These should have the following signatures:

def dec_hook(type: Type, obj: Any) -> Any:
"""Given a type in a schema, convert ``obj`` (composed of natively
supported objects) into an object of type ``type``"""
supported objects) into an object of type ``type``.

Any `TypeError` or `ValueError` exceptions raised by this method will
be considered "user facing" and converted into a `ValidationError` with
additional context. All other exceptions will be raised directly.
"""
pass

def ext_hook(code: int, data: memoryview) -> Any:
Expand Down Expand Up @@ -78,8 +83,8 @@ objects to/from objects, which are then natively handled by ``msgspec``.
# convert the complex to a tuple of real, imag
return (obj.real, obj.imag)
else:
# Raise a TypeError for other types
raise TypeError(f"Objects of type {type(obj)} are not supported")
# Raise a NotImplementedError for other types
raise NotImplementedError(f"Objects of type {type(obj)} are not supported")


def dec_hook(type: Type, obj: Any) -> Any:
Expand All @@ -89,8 +94,8 @@ objects to/from objects, which are then natively handled by ``msgspec``.
real, imag = obj
return complex(real, imag)
else:
# Raise a TypeError for other types
raise TypeError(f"Objects of type {type} are not supported")
# Raise a NotImplementedError for other types
raise NotImplementedError(f"Objects of type {type} are not supported")


# Define a message that contains a complex type
Expand Down Expand Up @@ -161,8 +166,8 @@ buffer.
+---------+---------+
| real | imag |
+---------+---------+
8 bytes 8 bytes
8 bytes 8 bytes


Here we define ``enc_hook`` and ``ext_hook`` callbacks to convert `complex`
objects to/from this binary representation as a MessagePack extension.
Expand All @@ -186,8 +191,8 @@ objects to/from this binary representation as a MessagePack extension.
# Return an `Ext` object so msgspec serializes it as an extension type.
return msgspec.msgpack.Ext(COMPLEX_TYPE_CODE, data)
else:
# Raise a TypeError for other types
raise TypeError(f"Objects of type {type(obj)} are not supported")
# Raise a NotImplementedError for other types
raise NotImplementedError(f"Objects of type {type(obj)} are not supported")


def ext_hook(code: int, data: memoryview) -> Any:
Expand All @@ -197,8 +202,8 @@ objects to/from this binary representation as a MessagePack extension.
real, imag = struct.unpack('dd', data)
return complex(real, imag)
else:
# Raise a TypeError for other extension type codes
raise TypeError(f"Extension type code {code} is not supported")
# Raise a NotImplementedError for other extension type codes
raise NotImplementedError(f"Extension type code {code} is not supported")


# Create an encoder and a decoder using the custom callbacks
Expand Down
86 changes: 71 additions & 15 deletions msgspec/_core.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8574,8 +8574,59 @@ ms_decode_custom(PyObject *obj, PyObject *dec_hook, TypeNode* type, PathNode *pa
if (dec_hook != NULL) {
out = PyObject_CallFunctionObjArgs(dec_hook, custom_obj, obj, NULL);
Py_DECREF(obj);
if (out == NULL)
if (out == NULL) {
PyObject *exc_type, *exc, *tb;

/* Fetch the exception state */
PyErr_Fetch(&exc_type, &exc, &tb);

/* If null, some other c-extension has borked, just return */
if (exc_type == NULL) return NULL;

/* If it's a TypeError or ValueError, wrap it in a ValidationError.
* Otherwise we reraise the original error below */
if (
PyType_IsSubtype(
(PyTypeObject *)exc_type, (PyTypeObject *)PyExc_ValueError
) ||
PyType_IsSubtype(
(PyTypeObject *)exc_type, (PyTypeObject *)PyExc_TypeError
)
) {
PyObject *exc_type2, *exc2, *tb2;

/* Normalize the original exception */
PyErr_NormalizeException(&exc_type, &exc, &tb);
if (tb != NULL) {
PyException_SetTraceback(exc, tb);
Py_DECREF(tb);
}
Py_DECREF(exc_type);

/* Raise a new validation error with context based on the
* original exception */
ms_raise_validation_error(path, "%S%U", exc);

/* Fetch the new exception */
PyErr_Fetch(&exc_type2, &exc2, &tb2);
/* Normalize the new exception */
PyErr_NormalizeException(&exc_type2, &exc2, &tb2);
/* Set the original exception as the cause and context */
Py_INCREF(exc);
PyException_SetCause(exc2, exc);
PyException_SetContext(exc2, exc);

/* At this point the original exc_type/exc/tb are all dropped,
* replace them with the new values */
exc_type = exc_type2;
exc = exc2;
tb = tb2;
}
/* Restore the new exception state */
PyErr_Restore(exc_type, exc, tb);

return NULL;
}
}
else {
out = obj;
Expand Down Expand Up @@ -10136,8 +10187,9 @@ PyDoc_STRVAR(Encoder__doc__,
"Parameters\n"
"----------\n"
"enc_hook : callable, optional\n"
" A callable to call for objects that aren't supported msgspec types. Takes the\n"
" unsupported object and should return a supported object, or raise a TypeError."
" A callable to call for objects that aren't supported msgspec types. Takes\n"
" the unsupported object and should return a supported object, or raise a\n"
" ``NotImplementedError`` if unsupported."
);

enum mpack_code {
Expand Down Expand Up @@ -11108,8 +11160,9 @@ PyDoc_STRVAR(msgspec_msgpack_encode__doc__,
"obj : Any\n"
" The object to serialize.\n"
"enc_hook : callable, optional\n"
" A callable to call for objects that aren't supported msgspec types. Takes the\n"
" unsupported object and should return a supported object, or raise a TypeError.\n"
" A callable to call for objects that aren't supported msgspec types. Takes\n"
" the unsupported object and should return a supported object, or raise a\n"
" ``NotImplementedError`` if unsupported.\n"
"\n"
"Returns\n"
"-------\n"
Expand Down Expand Up @@ -11139,8 +11192,9 @@ PyDoc_STRVAR(JSONEncoder__doc__,
"Parameters\n"
"----------\n"
"enc_hook : callable, optional\n"
" A callable to call for objects that aren't supported msgspec types. Takes the\n"
" unsupported object and should return a supported object, or raise a TypeError."
" A callable to call for objects that aren't supported msgspec types. Takes\n"
" the unsupported object and should return a supported object, or raise a\n"
" ``NotImplementedError`` if unsupported."
);

static int json_encode_inline(EncoderState*, PyObject*);
Expand Down Expand Up @@ -11956,8 +12010,9 @@ PyDoc_STRVAR(msgspec_json_encode__doc__,
"obj : Any\n"
" The object to serialize.\n"
"enc_hook : callable, optional\n"
" A callable to call for objects that aren't supported msgspec types. Takes the\n"
" unsupported object and should return a supported object, or raise a TypeError.\n"
" A callable to call for objects that aren't supported msgspec types. Takes\n"
" the unsupported object and should return a supported object, or raise a\n"
" ``NotImplementedError`` if unsupported.\n"
"\n"
"Returns\n"
"-------\n"
Expand Down Expand Up @@ -12025,7 +12080,7 @@ PyDoc_STRVAR(Decoder__doc__,
" signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` is the\n"
" expected message type, and ``obj`` is the decoded representation composed\n"
" of only basic MessagePack types. This hook should transform ``obj`` into\n"
" type ``type``, or raise a ``TypeError`` if unsupported.\n"
" type ``type``, or raise a ``NotImplementedError`` if unsupported.\n"
"ext_hook : callable, optional\n"
" An optional callback for decoding MessagePack extensions. Should have the\n"
" signature ``ext_hook(code: int, data: memoryview) -> Any``. If provided,\n"
Expand Down Expand Up @@ -13758,7 +13813,7 @@ PyDoc_STRVAR(msgspec_msgpack_decode__doc__,
" signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` is the\n"
" expected message type, and ``obj`` is the decoded representation composed\n"
" of only basic MessagePack types. This hook should transform ``obj`` into\n"
" type ``type``, or raise a ``TypeError`` if unsupported.\n"
" type ``type``, or raise a ``NotImplementedError`` if unsupported.\n"
"ext_hook : callable, optional\n"
" An optional callback for decoding MessagePack extensions. Should have the\n"
" signature ``ext_hook(code: int, data: memoryview) -> Any``. If provided,\n"
Expand Down Expand Up @@ -13934,7 +13989,7 @@ PyDoc_STRVAR(JSONDecoder__doc__,
" signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` is the\n"
" expected message type, and ``obj`` is the decoded representation composed\n"
" of only basic JSON types. This hook should transform ``obj`` into type\n"
" ``type``, or raise a ``TypeError`` if unsupported."
" ``type``, or raise a ``NotImplementedError`` if unsupported."
);
static int
JSONDecoder_init(JSONDecoder *self, PyObject *args, PyObject *kwds)
Expand Down Expand Up @@ -17602,8 +17657,9 @@ PyDoc_STRVAR(msgspec_to_builtins__doc__,
"str_keys: bool, optional\n"
" Whether to convert all object keys to strings. Default is False.\n"
"enc_hook : callable, optional\n"
" A callable to call for objects that aren't supported msgspec types. Takes the\n"
" unsupported object and should return a supported object, or raise a TypeError.\n"
" A callable to call for objects that aren't supported msgspec types. Takes\n"
" the unsupported object and should return a supported object, or raise a\n"
" ``NotImplementedError`` if unsupported.\n"
"\n"
"Returns\n"
"-------\n"
Expand Down Expand Up @@ -19011,7 +19067,7 @@ PyDoc_STRVAR(msgspec_convert__doc__,
" signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` is the\n"
" expected message type, and ``obj`` is the decoded representation composed\n"
" of only basic MessagePack types. This hook should transform ``obj`` into\n"
" type ``type``, or raise a ``TypeError`` if unsupported.\n"
" type ``type``, or raise a ``NotImplementedError`` if unsupported.\n"
"builtin_types: Iterable[type], optional\n"
" Useful for wrapping other serialization protocols. An iterable of types to\n"
" treat as additional builtin types. Passing a type here indicates that the\n"
Expand Down
4 changes: 2 additions & 2 deletions msgspec/toml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def encode(obj: Any, *, enc_hook: Optional[Callable[[Any], Any]] = None) -> byte
enc_hook : callable, optional
A callable to call for objects that aren't supported msgspec types.
Takes the unsupported object and should return a supported object, or
raise a TypeError.
raise a ``NotImplementedError`` if unsupported.

Returns
-------
Expand Down Expand Up @@ -137,7 +137,7 @@ def decode(buf, *, type=Any, strict=True, dec_hook=None):
the signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type``
is the expected message type, and ``obj`` is the decoded representation
composed of only basic TOML types. This hook should transform ``obj``
into type ``type``, or raise a ``TypeError`` if unsupported.
into type ``type``, or raise a ``NotImplementedError`` if unsupported.

Returns
-------
Expand Down
4 changes: 2 additions & 2 deletions msgspec/yaml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def encode(obj: Any, *, enc_hook: Optional[Callable[[Any], Any]] = None) -> byte
enc_hook : callable, optional
A callable to call for objects that aren't supported msgspec types.
Takes the unsupported object and should return a supported object, or
raise a TypeError.
raise a ``NotImplementedError`` if unsupported.

Returns
-------
Expand Down Expand Up @@ -129,7 +129,7 @@ def decode(buf, *, type=Any, strict=True, dec_hook=None):
the signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type``
is the expected message type, and ``obj`` is the decoded representation
composed of only basic YAML types. This hook should transform ``obj``
into type ``type``, or raise a ``TypeError`` if unsupported.
into type ``type``, or raise a ``NotImplementedError`` if unsupported.

Returns
-------
Expand Down
131 changes: 131 additions & 0 deletions tests/test_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,137 @@ def test_decoder_runtime_type_parameters(self, proto):
msg = proto.encode(2)
assert dec.decode(msg) == 2

def test_decoder_dec_hook_attribute(self, proto):
def dec_hook(typ, obj):
pass

dec = proto.Decoder()
assert dec.dec_hook is None

dec = proto.Decoder(dec_hook=None)
assert dec.dec_hook is None

dec = proto.Decoder(dec_hook=dec_hook)
assert dec.dec_hook is dec_hook

def test_decoder_dec_hook_not_callable(self, proto):
with pytest.raises(TypeError):
proto.Decoder(dec_hook=1)

def test_decode_dec_hook(self, proto):
def dec_hook(typ, obj):
assert typ is Custom
return typ(*obj)

msg = proto.encode([1, 2])
res = proto.decode(msg, type=Custom, dec_hook=dec_hook)
assert res == Custom(1, 2)
assert isinstance(res, Custom)

def test_decoder_dec_hook(self, proto):
called = False

def dec_hook(typ, obj):
nonlocal called
called = True
assert typ is Custom
return Custom(*obj)

dec = proto.Decoder(type=List[Custom], dec_hook=dec_hook)
buf = proto.encode([[1, 2], [3, 4], [5, 6]])
msg = dec.decode(buf)
assert called
assert msg == [Custom(1, 2), Custom(3, 4), Custom(5, 6)]
assert isinstance(msg[0], Custom)

def test_decoder_dec_hook_optional_custom_type(self, proto):
called = False

def dec_hook(typ, obj):
nonlocal called
called = True

dec = proto.Decoder(type=Optional[Custom], dec_hook=dec_hook)
msg = dec.decode(proto.encode(None))
assert not called
assert msg is None

@pytest.mark.parametrize("err_cls", [TypeError, ValueError])
def test_decode_dec_hook_errors_wrapped(self, err_cls, proto):
def dec_hook(typ, obj):
assert obj == "some string"
raise err_cls("Oh no!")

msg = proto.encode("some string")
with pytest.raises(msgspec.ValidationError, match="Oh no!") as rec:
proto.decode(msg, type=Custom, dec_hook=dec_hook)

assert rec.value.__cause__ is rec.value.__context__
assert type(rec.value.__cause__) is err_cls

msg = proto.encode(["some string"])
with pytest.raises(msgspec.ValidationError, match=r"Oh no! - at `\$\[0\]`"):
proto.decode(msg, type=List[Custom], dec_hook=dec_hook)

def test_decode_dec_hook_errors_passthrough(self, proto):
def dec_hook(typ, obj):
assert obj == "some string"
raise NotImplementedError("Oh no!")

msg = proto.encode("some string")
with pytest.raises(NotImplementedError, match="Oh no!"):
proto.decode(msg, type=Custom, dec_hook=dec_hook)

msg = proto.encode(["some string"])
with pytest.raises(NotImplementedError, match=r"Oh no!"):
proto.decode(msg, type=List[Custom], dec_hook=dec_hook)

def test_decode_dec_hook_wrong_type(self, proto):
dec = proto.Decoder(type=Custom, dec_hook=lambda t, o: o)

msg = proto.encode([1, 2])
with pytest.raises(
msgspec.ValidationError,
match="Expected `Custom`, got `list`",
):
dec.decode(msg)

def test_decode_dec_hook_wrong_type_in_struct(self, proto):
class Test(msgspec.Struct):
point: Custom
other: int

dec = proto.Decoder(type=Test, dec_hook=lambda t, o: o)

msg = proto.encode({"point": [1, 2], "other": 3})
with pytest.raises(msgspec.ValidationError) as rec:
dec.decode(msg)

assert "Expected `Custom`, got `list` - at `$.point`" == str(rec.value)

def test_decode_dec_hook_wrong_type_generic(self, proto):
dec = proto.Decoder(type=Deque[int], dec_hook=lambda t, o: o)

msg = proto.encode([1, 2, 3])
with pytest.raises(msgspec.ValidationError) as rec:
dec.decode(msg)

assert "Expected `collections.deque`, got `list`" == str(rec.value)

def test_decode_dec_hook_isinstance_errors(self, proto):
class Metaclass(type):
def __instancecheck__(self, obj):
raise TypeError("Oh no!")

class Custom(metaclass=Metaclass):
pass

dec = proto.Decoder(type=Custom)

msg = proto.encode(1)
with pytest.raises(TypeError, match="Oh no!"):
dec.decode(msg)


class TestThreadSafe:
def test_encode_threadsafe(self, proto):
Expand Down
Loading