diff --git a/benchmarks/bench_encodings.py b/benchmarks/bench_encodings.py index a4919a7e..6ccfa789 100644 --- a/benchmarks/bench_encodings.py +++ b/benchmarks/bench_encodings.py @@ -31,7 +31,7 @@ class Directory(msgspec.Struct, tag="directory"): def bench(dumps, loads, ndata, schema=None): data = make_filesystem_data(ndata) if schema: - data = msgspec.from_builtins(data, schema) + data = msgspec.convert(data, schema) timer = timeit.Timer("func(data)", globals={"func": dumps, "data": data}) n, t = timer.autorange() dumps_time = t / n diff --git a/benchmarks/bench_validation.py b/benchmarks/bench_validation.py index 69ead67b..923f21d6 100644 --- a/benchmarks/bench_validation.py +++ b/benchmarks/bench_validation.py @@ -40,7 +40,7 @@ def bench_msgspec(n): dec = msgspec.json.Decoder(Directory) def convert(data): - return msgspec.from_builtins(data, Directory) + return msgspec.convert(data, Directory) return bench(enc.encode, dec.decode, n, convert) diff --git a/docs/source/supported-types.rst b/docs/source/supported-types.rst index 3ecbca80..b6bd360d 100644 --- a/docs/source/supported-types.rst +++ b/docs/source/supported-types.rst @@ -87,6 +87,15 @@ lacks a ``null`` value, attempted to encode a message containing ``None`` to >>> msgspec.json.decode(b'null') None +If ``strict=False`` is specified, a string value of ``"null"`` (case +insensitive) may also be coerced to ``None``. See :ref:`strict-vs-lax` for more +information. + +.. code-block:: python + + >>> msgspec.json.decode(b'"null"', type=None, strict=False) + None + ``bool`` -------- @@ -101,6 +110,18 @@ supported protocols. >>> msgspec.json.decode(b'true') True +If ``strict=False`` is specified, string values of ``"true"``/``"1"`` or +``"false"``/``"0"`` (case insensitive) may also be coerced to +``True``/``False`` respectively. See :ref:`strict-vs-lax` for more information. + +.. code-block:: python + + >>> msgspec.json.decode(b'"false"', type=bool, strict=False) + False + + >>> msgspec.json.decode(b'"TRUE"', type=bool, strict=False) + True + ``int`` ------- @@ -122,6 +143,15 @@ Support for large integers varies by protocol: >>> msgspec.json.decode(b"123", type=int) 123 +If ``strict=False`` is specified, string values may also be coerced to +integers, following the same restrictions as above. See :ref:`strict-vs-lax` +for more information. + +.. code-block:: python + + >>> msgspec.json.decode(b'"123"', type=int, strict=False) + 123 + ``float`` --------- @@ -151,6 +181,19 @@ provided, the `int` will be automatically converted. ... msgspec.json.decode(b"123", type=float) 123.0 +If ``strict=False`` is specified, string values may also be coerced to floats. +Note that in this case the strings ``"nan"``, ``"inf"``/``"infinity"``, +``"-inf"``/``"-infinity"`` (case insensitive) will coerce to +``nan``/``inf``/``-inf``. See :ref:`strict-vs-lax` for more information. + +.. code-block:: python + + >>> msgspec.json.decode(b'"123.45"', type=float, strict=False) + 123.45 + + >>> msgspec.json.decode(b'"-inf"', type=float, strict=False) + -inf + ``str`` ------- diff --git a/docs/source/usage.rst b/docs/source/usage.rst index c70e0d20..419ffbfb 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -131,10 +131,15 @@ If a message doesn't match the expected type, an error is raised. File "", line 1, in msgspec.ValidationError: Expected `str`, got `int` - at `$.groups[1]` +.. _strict-vs-lax: + +"Strict" vs "Lax" Mode +~~~~~~~~~~~~~~~~~~~~~~ + Unlike some other libraries (e.g. pydantic_), ``msgspec`` won't perform any -unsafe implicit conversion. For example, if an integer is specified and a -string is decoded instead, an error is raised rather than attempting to cast -the string to an int. +unsafe implicit conversion by default ("strict" mode). For example, if an +integer is specified and a string is provided instead, an error is raised +rather than attempting to cast the string to an int. .. code-block:: python @@ -143,6 +148,17 @@ the string to an int. File "", line 1, in msgspec.ValidationError: Expected `int`, got `str` - at `$[2]` +For cases where you'd like a more lax set of conversion rules, you can pass +``strict=False`` to any ``decode`` function or ``Decoder`` class ("lax" mode). +See :doc:`supported-types` for information on how this affects individual +types. + +.. code-block:: python + + >>> msgspec.json.decode(b'[1, 2, "3"]', type=list[int], strict=False) + [1, 2, 3] + + .. _JSON: https://json.org .. _MessagePack: https://msgpack.org .. _YAML: https://yaml.org diff --git a/msgspec/_core.c b/msgspec/_core.c index 334ccc4a..11864832 100644 --- a/msgspec/_core.c +++ b/msgspec/_core.c @@ -355,6 +355,7 @@ typedef struct { PyObject *str_enc_hook; PyObject *str_dec_hook; PyObject *str_ext_hook; + PyObject *str_strict; PyObject *str_utcoffset; PyObject *str___origin__; PyObject *str___args__; @@ -8789,20 +8790,17 @@ ms_decode_float(double x, TypeNode *type, PathNode *path) { } static MS_NOINLINE PyObject * -ms_decode_constr_pyfloat(PyObject *obj, TypeNode *type, PathNode *path) { +_ms_check_float_constraints(PyObject *obj, TypeNode *type, PathNode *path) { double x = PyFloat_AS_DOUBLE(obj); - if (!ms_passes_float_constraints_inline(x, type, path)) return NULL; - Py_INCREF(obj); - return obj; + if (ms_passes_float_constraints_inline(x, type, path)) return obj; + Py_DECREF(obj); + return NULL; } static MS_INLINE PyObject * -ms_decode_pyfloat(PyObject *obj, TypeNode *type, PathNode *path) { - if (MS_UNLIKELY(type->types & MS_FLOAT_CONSTRS)) { - return ms_decode_constr_pyfloat(obj, type, path); - } - Py_INCREF(obj); - return obj; +ms_check_float_constraints(PyObject *obj, TypeNode *type, PathNode *path) { + if (MS_LIKELY(!(type->types & MS_FLOAT_CONSTRS))) return obj; + return _ms_check_float_constraints(obj, type, path); } static MS_NOINLINE bool @@ -9828,6 +9826,183 @@ ms_decode_decimal(const char *view, Py_ssize_t size, bool is_ascii, PathNode *pa return out; } +/************************************************************************* + * strict=False Utilities * + *************************************************************************/ + +static PyObject * +ms_maybe_decode_int_from_str( + const char *p, Py_ssize_t size, TypeNode *type, PathNode *path, bool *invalid +) { + uint64_t mantissa = 0; + bool is_negative = false; + const char *end = p + size; + + if (size == 0) goto invalid_int; + + char c = *p; + if (c == '-') { + p++; + is_negative = true; + if (p == end) goto invalid_int; + c = *p; + } + + if (MS_UNLIKELY(c == '0')) { + /* Value is either 0 or invalid */ + p++; + if (p == end) goto done; + goto invalid_int; + } + + /* We can read the first 19 digits safely into a uint64 without checking + * for overflow. */ + size_t remaining = end - p; + const char *safe_end = p + Py_MIN(19, remaining); + while (p < safe_end) { + c = *p; + if (!is_digit(c)) goto end_digits; + p++; + mantissa = mantissa * 10 + (uint64_t)(c - '0'); + } + if (MS_UNLIKELY(remaining > 19)) { + /* Reading a 20th digit may or may not cause overflow. Any additional + * digits definitely will. Read the 20th digit (and check for a 21st), + * erroring upon overflow. */ + c = *p; + if (MS_UNLIKELY(is_digit(c))) { + p++; + uint64_t mantissa2 = mantissa * 10 + (uint64_t)(c - '0'); + bool out_of_range = ( + (mantissa2 < mantissa) || + ((mantissa2 - (uint64_t)(c - '0')) / 10) != mantissa || + (p != end) + ); + if (out_of_range) goto out_of_range; + mantissa = mantissa2; + } + } + +end_digits: + /* There must be at least one digit */ + if (MS_UNLIKELY(mantissa == 0)) goto invalid_int; + + /* Check for trailing characters */ + if (p != end) goto invalid_int; + +done: + if (MS_UNLIKELY(is_negative)) { + if (MS_UNLIKELY(mantissa > 1ull << 63)) { + goto out_of_range; + } + if (MS_LIKELY(type->types & MS_TYPE_INT)) { + return ms_decode_int(-1 * (int64_t)mantissa, type, path); + } + return ms_decode_int_enum_or_literal_int64(-1 * (int64_t)mantissa, type, path); + } + if (MS_LIKELY(type->types & MS_TYPE_INT)) { + return ms_decode_uint(mantissa, type, path); + } + return ms_decode_int_enum_or_literal_uint64(mantissa, type, path); + +out_of_range: + return ms_error_with_path("Integer value out of range%U", path); + +invalid_int: + *invalid = true; + return NULL; +} + +static PyObject * +ms_decode_int_from_str( + const char *p, Py_ssize_t size, TypeNode *type, PathNode *path +) { + bool invalid = false; + PyObject *out = ms_maybe_decode_int_from_str(p, size, type, path, &invalid); + if (MS_UNLIKELY(invalid)) { + ms_error_with_path("Invalid integer string%U", path); + return NULL; + } + return out; +} + +static PyObject * +ms_decode_str_lax( + const char *view, + Py_ssize_t size, + TypeNode *type, + PathNode *path, + bool *invalid +) { + if (type->types & (MS_TYPE_INT | MS_TYPE_INTENUM | MS_TYPE_INTLITERAL)) { + bool invalid_int = false; + PyObject *out = ms_maybe_decode_int_from_str( + view, size, type, path, &invalid_int + ); + if (MS_LIKELY(!invalid_int)) return out; + } + + if (type->types & MS_TYPE_FLOAT) { + /* TODO: with some refactoring, we should be able to use our own str -> + * float routine rather than relying on CPython's */ + PyObject *temp = PyBytes_FromStringAndSize(view, size); + if (temp == NULL) return NULL; + PyObject *out = PyFloat_FromString(temp); + Py_DECREF(temp); + if (out == NULL) { + PyErr_Clear(); + } + else { + return ms_check_float_constraints(out, type, path); + } + } + + if (type->types & MS_TYPE_BOOL) { + if (size == 1) { + if (*view == '0') { + Py_RETURN_FALSE; + } + else if (*view == '1') { + Py_RETURN_TRUE; + } + } + else if (size == 4) { + if ( + (view[0] == 't' || view[0] == 'T') && + (view[1] == 'r' || view[1] == 'R') && + (view[2] == 'u' || view[2] == 'U') && + (view[3] == 'e' || view[3] == 'E') + ) { + Py_RETURN_TRUE; + } + } + else if (size == 5) { + if ( + (view[0] == 'f' || view[0] == 'F') && + (view[1] == 'a' || view[1] == 'A') && + (view[2] == 'l' || view[2] == 'L') && + (view[3] == 's' || view[3] == 'S') && + (view[4] == 'e' || view[4] == 'E') + ) { + Py_RETURN_FALSE; + } + } + } + + if (type->types & MS_TYPE_NONE) { + if (size == 4 && + (view[0] == 'n' || view[0] == 'N') && + (view[1] == 'u' || view[1] == 'U') && + (view[2] == 'l' || view[2] == 'L') && + (view[3] == 'l' || view[3] == 'L') + ) { + Py_RETURN_NONE; + } + } + *invalid = true; + return NULL; +} + /************************************************************************* * MessagePack Encoder * *************************************************************************/ @@ -11650,6 +11825,7 @@ typedef struct DecoderState { TypeNode *type; PyObject *dec_hook; PyObject *ext_hook; + bool strict; /* Per-message attributes */ PyObject *buffer_obj; @@ -11664,12 +11840,13 @@ typedef struct Decoder { /* Configuration */ TypeNode *type; + char strict; PyObject *dec_hook; PyObject *ext_hook; } Decoder; PyDoc_STRVAR(Decoder__doc__, -"Decoder(type='Any', *, dec_hook=None, ext_hook=None)\n" +"Decoder(type='Any', *, strict=True, dec_hook=None, ext_hook=None)\n" "--\n" "\n" "A MessagePack decoder.\n" @@ -11681,6 +11858,10 @@ PyDoc_STRVAR(Decoder__doc__, " provided, the message will be type checked and decoded as the specified\n" " type. Defaults to `Any`, in which case the message will be decoded using\n" " the default MessagePack types.\n" +"strict : bool, optional\n" +" Whether type coercion rules should be strict. Setting to False enables a\n" +" wider set of coercion rules from string to non-string types for all values.\n" +" Default is True.\n" "dec_hook : callable, optional\n" " An optional callback for handling decoding custom types. Should have the\n" " signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` is the\n" @@ -11699,18 +11880,22 @@ PyDoc_STRVAR(Decoder__doc__, static int Decoder_init(Decoder *self, PyObject *args, PyObject *kwds) { - char *kwlist[] = {"type", "dec_hook", "ext_hook", NULL}; + char *kwlist[] = {"type", "strict", "dec_hook", "ext_hook", NULL}; MsgspecState *st = msgspec_get_global_state(); PyObject *type = st->typing_any; PyObject *ext_hook = NULL; PyObject *dec_hook = NULL; + int strict = 1; if (!PyArg_ParseTupleAndKeywords( - args, kwds, "|O$OO", kwlist, &type, &dec_hook, &ext_hook + args, kwds, "|O$pOO", kwlist, &type, &strict, &dec_hook, &ext_hook )) { return -1; } + /* Handle strict */ + self->strict = strict; + /* Handle dec_hook */ if (dec_hook == Py_None) { dec_hook = NULL; @@ -12226,38 +12411,38 @@ mpack_decode_float(DecoderState *self, double val, TypeNode *type, PathNode *pat static PyObject * mpack_decode_str(DecoderState *self, Py_ssize_t size, TypeNode *type, PathNode *path) { - if (MS_LIKELY( - type->types & ( - MS_TYPE_ANY | MS_TYPE_STR | MS_TYPE_ENUM | MS_TYPE_STRLITERAL | - MS_TYPE_DATETIME | MS_TYPE_DATE | MS_TYPE_TIME | - MS_TYPE_UUID | MS_TYPE_DECIMAL - ) - ) - ) { - char *s = NULL; - if (MS_UNLIKELY(mpack_read(self, &s, size) < 0)) return NULL; - if (MS_UNLIKELY(type->types & (MS_TYPE_ENUM | MS_TYPE_STRLITERAL))) { - return ms_decode_str_enum_or_literal(s, size, type, path); - } - if (MS_UNLIKELY(type->types & MS_TYPE_DATETIME)) { - return ms_decode_datetime(s, size, type, path); - } - if (MS_UNLIKELY(type->types & MS_TYPE_DATE)) { - return ms_decode_date(s, size, path); - } - if (MS_UNLIKELY(type->types & MS_TYPE_TIME)) { - return ms_decode_time(s, size, type, path); - } - if (MS_UNLIKELY(type->types & MS_TYPE_UUID)) { - return ms_decode_uuid(s, size, path); - } - if (MS_UNLIKELY(type->types & MS_TYPE_DECIMAL)) { - return ms_decode_decimal(s, size, false, path); - } + char *s = NULL; + if (MS_UNLIKELY(mpack_read(self, &s, size) < 0)) return NULL; + + if (MS_UNLIKELY(!self->strict)) { + bool invalid = false; + PyObject *out = ms_decode_str_lax(s, size, type, path, &invalid); + if (!invalid) return out; + } + + if (MS_LIKELY(type->types & (MS_TYPE_STR | MS_TYPE_ANY))) { return ms_check_str_constraints( PyUnicode_DecodeUTF8(s, size, NULL), type, path ); } + else if (MS_UNLIKELY(type->types & (MS_TYPE_ENUM | MS_TYPE_STRLITERAL))) { + return ms_decode_str_enum_or_literal(s, size, type, path); + } + else if (MS_UNLIKELY(type->types & MS_TYPE_DATETIME)) { + return ms_decode_datetime(s, size, type, path); + } + else if (MS_UNLIKELY(type->types & MS_TYPE_DATE)) { + return ms_decode_date(s, size, path); + } + else if (MS_UNLIKELY(type->types & MS_TYPE_TIME)) { + return ms_decode_time(s, size, type, path); + } + else if (MS_UNLIKELY(type->types & MS_TYPE_UUID)) { + return ms_decode_uuid(s, size, path); + } + else if (MS_UNLIKELY(type->types & MS_TYPE_DECIMAL)) { + return ms_decode_decimal(s, size, false, path); + } return ms_validation_error("str", type, path); } @@ -13328,6 +13513,7 @@ Decoder_decode(Decoder *self, PyObject *const *args, Py_ssize_t nargs) DecoderState state = { .type = self->type, + .strict = self->strict, .dec_hook = self->dec_hook, .ext_hook = self->ext_hook }; @@ -13365,6 +13551,7 @@ static struct PyMethodDef Decoder_methods[] = { static PyMemberDef Decoder_members[] = { {"type", T_OBJECT_EX, offsetof(Decoder, orig_type), READONLY, "The Decoder type"}, + {"strict", T_BOOL, offsetof(Decoder, strict), READONLY, "The Decoder strict setting"}, {"dec_hook", T_OBJECT, offsetof(Decoder, dec_hook), READONLY, "The Decoder dec_hook"}, {"ext_hook", T_OBJECT, offsetof(Decoder, ext_hook), READONLY, "The Decoder ext_hook"}, {NULL}, @@ -13387,7 +13574,7 @@ static PyTypeObject Decoder_Type = { PyDoc_STRVAR(msgspec_msgpack_decode__doc__, -"msgpack_decode(buf, *, type='Any', dec_hook=None, ext_hook=None)\n" +"msgpack_decode(buf, *, type='Any', strict=True, dec_hook=None, ext_hook=None)\n" "--\n" "\n" "Deserialize an object from bytes.\n" @@ -13401,6 +13588,10 @@ PyDoc_STRVAR(msgspec_msgpack_decode__doc__, " provided, the message will be type checked and decoded as the specified\n" " type. Defaults to `Any`, in which case the message will be decoded using\n" " the default MessagePack types.\n" +"strict : bool, optional\n" +" Whether type coercion rules should be strict. Setting to False enables a\n" +" wider set of coercion rules from string to non-string types for all values.\n" +" Default is True.\n" "dec_hook : callable, optional\n" " An optional callback for handling decoding custom types. Should have the\n" " signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` is the\n" @@ -13428,8 +13619,10 @@ PyDoc_STRVAR(msgspec_msgpack_decode__doc__, static PyObject* msgspec_msgpack_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - PyObject *res = NULL, *buf = NULL, *type = NULL, *dec_hook = NULL, *ext_hook = NULL; + PyObject *res = NULL, *buf = NULL, *type = NULL, *strict_obj = NULL; + PyObject *dec_hook = NULL, *ext_hook = NULL; MsgspecState *mod = msgspec_get_global_state(); + int strict = 1; /* Parse arguments */ if (!check_positional_nargs(nargs, 1, 1)) return NULL; @@ -13437,6 +13630,7 @@ msgspec_msgpack_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, if (kwnames != NULL) { Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames); if ((type = find_keyword(kwnames, args + nargs, mod->str_type)) != NULL) nkwargs--; + if ((strict_obj = find_keyword(kwnames, args + nargs, mod->str_strict)) != NULL) nkwargs--; if ((dec_hook = find_keyword(kwnames, args + nargs, mod->str_dec_hook)) != NULL) nkwargs--; if ((ext_hook = find_keyword(kwnames, args + nargs, mod->str_ext_hook)) != NULL) nkwargs--; if (nkwargs > 0) { @@ -13448,6 +13642,12 @@ msgspec_msgpack_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, } } + /* Handle strict */ + if (strict_obj != NULL) { + strict = PyObject_IsTrue(strict_obj); + if (strict < 0) return NULL; + } + /* Handle dec_hook */ if (dec_hook == Py_None) { dec_hook = NULL; @@ -13471,6 +13671,7 @@ msgspec_msgpack_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, } DecoderState state = { + .strict = strict, .dec_hook = dec_hook, .ext_hook = ext_hook }; @@ -13526,6 +13727,7 @@ typedef struct JSONDecoderState { /* Configuration */ TypeNode *type; PyObject *dec_hook; + bool strict; /* Temporary scratch space */ unsigned char *scratch; @@ -13545,11 +13747,12 @@ typedef struct JSONDecoder { /* Configuration */ TypeNode *type; + char strict; PyObject *dec_hook; } JSONDecoder; PyDoc_STRVAR(JSONDecoder__doc__, -"Decoder(type='Any', *, dec_hook=None)\n" +"Decoder(type='Any', *, strict=True, dec_hook=None)\n" "--\n" "\n" "A JSON decoder.\n" @@ -13561,6 +13764,10 @@ PyDoc_STRVAR(JSONDecoder__doc__, " provided, the message will be type checked and decoded as the specified\n" " type. Defaults to `Any`, in which case the message will be decoded using\n" " the default JSON types.\n" +"strict : bool, optional\n" +" Whether type coercion rules should be strict. Setting to False enables a\n" +" wider set of coercion rules from string to non-string types for all values.\n" +" Default is True.\n" "dec_hook : callable, optional\n" " An optional callback for handling decoding custom types. Should have the\n" " signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` is the\n" @@ -13571,12 +13778,15 @@ PyDoc_STRVAR(JSONDecoder__doc__, static int JSONDecoder_init(JSONDecoder *self, PyObject *args, PyObject *kwds) { - char *kwlist[] = {"type", "dec_hook", NULL}; + char *kwlist[] = {"type", "strict", "dec_hook", NULL}; MsgspecState *st = msgspec_get_global_state(); PyObject *type = st->typing_any; PyObject *dec_hook = NULL; + int strict = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$O", kwlist, &type, &dec_hook)) { + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "|O$pO", kwlist, &type, &strict, &dec_hook) + ) { return -1; } @@ -13593,6 +13803,9 @@ JSONDecoder_init(JSONDecoder *self, PyObject *args, PyObject *kwds) } self->dec_hook = dec_hook; + /* Handle strict */ + self->strict = strict; + /* Handle type */ self->type = TypeNode_Convert(type); if (self->type == NULL) return -1; @@ -14300,168 +14513,50 @@ json_decode_binary( return ms_error_with_path("Invalid base64 encoded string%U", path); } -static bool -json_decode_int_from_str_inner( - const char *p, Py_ssize_t size, bool err_invalid, - TypeNode *type, PathNode *path, PyObject **out -) { - /* This function signature has gotten kinda weird due to being shared - * between `json_decode` and `convert`. Read the comments below for - * more info */ - uint64_t mantissa = 0; - bool is_negative = false; - const char *end = p + size; - - if (size == 0) goto invalid; +static PyObject * +json_decode_string(JSONDecoderState *self, TypeNode *type, PathNode *path) { + char *view = NULL; + bool is_ascii = true; + Py_ssize_t size = json_decode_string_view(self, &view, &is_ascii); + if (size < 0) return NULL; - char c = *p; - if (c == '-') { - p++; - is_negative = true; - if (p == end) goto invalid; - c = *p; + if (MS_UNLIKELY(!self->strict)) { + bool invalid = false; + PyObject *out = ms_decode_str_lax(view, size, type, path, &invalid); + if (!invalid) return out; } - if (MS_UNLIKELY(c == '0')) { - /* Value is either 0 or invalid */ - p++; - if (p == end) goto done; - goto invalid; + if (MS_LIKELY(type->types & (MS_TYPE_STR | MS_TYPE_ANY))) { + PyObject *out; + if (MS_LIKELY(is_ascii)) { + out = PyUnicode_New(size, 127); + memcpy(ascii_get_buffer(out), view, size); + } + else { + out = PyUnicode_DecodeUTF8(view, size, NULL); + } + return ms_check_str_constraints(out, type, path); } - - /* We can read the first 19 digits safely into a uint64 without checking - * for overflow. */ - size_t remaining = end - p; - const char *safe_end = p + Py_MIN(19, remaining); - while (p < safe_end) { - c = *p; - if (!is_digit(c)) goto end_digits; - p++; - mantissa = mantissa * 10 + (uint64_t)(c - '0'); + else if (MS_UNLIKELY(type->types & MS_TYPE_DATETIME)) { + return ms_decode_datetime(view, size, type, path); } - if (MS_UNLIKELY(remaining > 19)) { - /* Reading a 20th digit may or may not cause overflow. Any additional - * digits definitely will. Read the 20th digit (and check for a 21st), - * erroring upon overflow. */ - c = *p; - if (MS_UNLIKELY(is_digit(c))) { - p++; - uint64_t mantissa2 = mantissa * 10 + (uint64_t)(c - '0'); - bool out_of_range = ( - (mantissa2 < mantissa) || - ((mantissa2 - (uint64_t)(c - '0')) / 10) != mantissa || - (p != end) - ); - if (out_of_range) goto out_of_range; - mantissa = mantissa2; - } + else if (MS_UNLIKELY(type->types & MS_TYPE_DATE)) { + return ms_decode_date(view, size, path); } - -end_digits: - /* There must be at least one digit */ - if (MS_UNLIKELY(mantissa == 0)) goto invalid; - - /* Check for trailing characters */ - if (p != end) goto invalid; - -done: - if (MS_UNLIKELY(is_negative)) { - if (MS_UNLIKELY(mantissa > 1ull << 63)) { - goto out_of_range; - } - if (MS_LIKELY(type->types & MS_TYPE_INT)) { - *out = ms_decode_int(-1 * (int64_t)mantissa, type, path); - return true; - } - *out = ms_decode_int_enum_or_literal_int64(-1 * (int64_t)mantissa, type, path); - return true; + else if (MS_UNLIKELY(type->types & MS_TYPE_TIME)) { + return ms_decode_time(view, size, type, path); } - if (MS_LIKELY(type->types & MS_TYPE_INT)) { - *out = ms_decode_uint(mantissa, type, path); - return true; + else if (MS_UNLIKELY(type->types & MS_TYPE_UUID)) { + return ms_decode_uuid(view, size, path); } - *out = ms_decode_int_enum_or_literal_uint64(mantissa, type, path); - return true; - -out_of_range: - *out = NULL; - ms_error_with_path("Integer value out of range%U", path); - return true; - -invalid: - /* An `invalid` error occurs when the string is not a valid integer. When - * parsing a union of types from a string (in `convert` with - * `strict=False`) we want to avoid raising an `invalid` error here, so - * other types in the union can be tried. If the string is a valid integer, - * but fails for other reasons (out of range, constraint issues, ...) then - * an error is still raised in this function. - */ - if (err_invalid) { - *out = NULL; - ms_error_with_path("Invalid integer string%U", path); - return true; + else if (MS_UNLIKELY(type->types & MS_TYPE_DECIMAL)) { + return ms_decode_decimal(view, size, is_ascii, path); } - /* `false` indicates no return value and no error raised */ - return false; -} - -static PyObject * -json_decode_int_from_str( - const char *p, Py_ssize_t size, TypeNode *type, PathNode *path -) { - PyObject *out; - json_decode_int_from_str_inner(p, size, true, type, path, &out); - return out; -} - -static PyObject * -json_decode_string(JSONDecoderState *self, TypeNode *type, PathNode *path) { - if ( - MS_LIKELY( - type->types & ( - MS_TYPE_ANY | MS_TYPE_STR | MS_TYPE_ENUM | MS_TYPE_STRLITERAL | - MS_TYPE_BYTES | MS_TYPE_BYTEARRAY | - MS_TYPE_DATETIME | MS_TYPE_DATE | MS_TYPE_TIME | - MS_TYPE_UUID | MS_TYPE_DECIMAL - ) - ) - ) { - char *view = NULL; - bool is_ascii = true; - Py_ssize_t size = json_decode_string_view(self, &view, &is_ascii); - if (size < 0) return NULL; - if (MS_LIKELY(type->types & (MS_TYPE_STR | MS_TYPE_ANY))) { - PyObject *out; - if (MS_LIKELY(is_ascii)) { - out = PyUnicode_New(size, 127); - memcpy(ascii_get_buffer(out), view, size); - } - else { - out = PyUnicode_DecodeUTF8(view, size, NULL); - } - return ms_check_str_constraints(out, type, path); - } - else if (MS_UNLIKELY(type->types & MS_TYPE_DATETIME)) { - return ms_decode_datetime(view, size, type, path); - } - else if (MS_UNLIKELY(type->types & MS_TYPE_DATE)) { - return ms_decode_date(view, size, path); - } - else if (MS_UNLIKELY(type->types & MS_TYPE_TIME)) { - return ms_decode_time(view, size, type, path); - } - else if (MS_UNLIKELY(type->types & MS_TYPE_UUID)) { - return ms_decode_uuid(view, size, path); - } - else if (MS_UNLIKELY(type->types & MS_TYPE_DECIMAL)) { - return ms_decode_decimal(view, size, is_ascii, path); - } - else if (MS_UNLIKELY(type->types & (MS_TYPE_BYTES | MS_TYPE_BYTEARRAY))) { - return json_decode_binary(view, size, type, path); - } - else { - return ms_decode_str_enum_or_literal(view, size, type, path); - } + else if (MS_UNLIKELY(type->types & (MS_TYPE_BYTES | MS_TYPE_BYTEARRAY))) { + return json_decode_binary(view, size, type, path); + } + else if (MS_UNLIKELY(type->types & (MS_TYPE_ENUM | MS_TYPE_STRLITERAL))) { + return ms_decode_str_enum_or_literal(view, size, type, path); } return ms_validation_error("str", type, path); } @@ -14483,7 +14578,7 @@ json_decode_dict_key_fallback( return ms_check_str_constraints(out, type, path); } if (type->types & (MS_TYPE_INT | MS_TYPE_INTENUM | MS_TYPE_INTLITERAL)) { - return json_decode_int_from_str(view, size, type, path); + return ms_decode_int_from_str(view, size, type, path); } else if (type->types & (MS_TYPE_ENUM | MS_TYPE_STRLITERAL)) { return ms_decode_str_enum_or_literal(view, size, type, path); @@ -16581,6 +16676,7 @@ JSONDecoder_decode(JSONDecoder *self, PyObject *const *args, Py_ssize_t nargs) JSONDecoderState state = { .type = self->type, + .strict = self->strict, .dec_hook = self->dec_hook, .scratch = NULL, .scratch_capacity = 0, @@ -16624,6 +16720,7 @@ static struct PyMethodDef JSONDecoder_methods[] = { static PyMemberDef JSONDecoder_members[] = { {"type", T_OBJECT_EX, offsetof(JSONDecoder, orig_type), READONLY, "The Decoder type"}, + {"strict", T_BOOL, offsetof(JSONDecoder, strict), READONLY, "The Decoder strict setting"}, {"dec_hook", T_OBJECT, offsetof(JSONDecoder, dec_hook), READONLY, "The Decoder dec_hook"}, {NULL}, }; @@ -16644,7 +16741,7 @@ static PyTypeObject JSONDecoder_Type = { }; PyDoc_STRVAR(msgspec_json_decode__doc__, -"json_decode(buf, *, type='Any', dec_hook=None)\n" +"json_decode(buf, *, type='Any', strict=True, dec_hook=None)\n" "--\n" "\n" "Deserialize an object from bytes.\n" @@ -16658,6 +16755,10 @@ PyDoc_STRVAR(msgspec_json_decode__doc__, " provided, the message will be type checked and decoded as the specified\n" " type. Defaults to `Any`, in which case the message will be decoded using\n" " the default JSON types.\n" +"strict : bool, optional\n" +" Whether type coercion rules should be strict. Setting to False enables a\n" +" wider set of coercion rules from string to non-string types for all values.\n" +" Default is True.\n" "dec_hook : callable, optional\n" " An optional callback for handling decoding custom types. Should have the\n" " signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` is the\n" @@ -16677,7 +16778,8 @@ PyDoc_STRVAR(msgspec_json_decode__doc__, static PyObject* msgspec_json_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - PyObject *res = NULL, *buf = NULL, *type = NULL, *dec_hook = NULL; + PyObject *res = NULL, *buf = NULL, *type = NULL, *dec_hook = NULL, *strict_obj = NULL; + int strict = 1; MsgspecState *mod = msgspec_get_global_state(); /* Parse arguments */ @@ -16686,6 +16788,7 @@ msgspec_json_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyO if (kwnames != NULL) { Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames); if ((type = find_keyword(kwnames, args + nargs, mod->str_type)) != NULL) nkwargs--; + if ((strict_obj = find_keyword(kwnames, args + nargs, mod->str_strict)) != NULL) nkwargs--; if ((dec_hook = find_keyword(kwnames, args + nargs, mod->str_dec_hook)) != NULL) nkwargs--; if (nkwargs > 0) { PyErr_SetString( @@ -16707,7 +16810,14 @@ msgspec_json_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyO } } + /* Handle strict */ + if (strict_obj != NULL) { + strict = PyObject_IsTrue(strict_obj); + if (strict < 0) return NULL; + } + JSONDecoderState state = { + .strict = strict, .dec_hook = dec_hook, .scratch = NULL, .scratch_capacity = 0, @@ -17404,7 +17514,8 @@ convert_float( ConvertState *self, PyObject *obj, TypeNode *type, PathNode *path ) { if (type->types & (MS_TYPE_ANY | MS_TYPE_FLOAT)) { - return ms_decode_pyfloat(obj, type, path); + Py_INCREF(obj); + return ms_check_float_constraints(obj, type, path); } return ms_validation_error("float", type, path); } @@ -17484,7 +17595,7 @@ convert_str_uncommon( else if ( is_key && self->str_keys && (type->types & (MS_TYPE_INT | MS_TYPE_INTENUM | MS_TYPE_INTLITERAL)) ) { - return json_decode_int_from_str(view, size, type, path); + return ms_decode_int_from_str(view, size, type, path); } return ms_validation_error("str", type, path); } @@ -17512,64 +17623,9 @@ convert_str_lax( Py_ssize_t size; const char* view = unicode_str_and_size(obj, &size); if (view == NULL) return NULL; - - if (type->types & (MS_TYPE_INT | MS_TYPE_INTENUM | MS_TYPE_INTLITERAL)) { - PyObject *out = NULL; - if (json_decode_int_from_str_inner(view, size, false, type, path, &out)) { - return out; - } - } - - if (type->types & MS_TYPE_FLOAT) { - PyObject *out = PyFloat_FromString(obj); - if (out != NULL) { - return ms_decode_pyfloat(out, type, path); - } - PyErr_Clear(); - } - - if (type->types & MS_TYPE_BOOL) { - if (size == 1) { - if (*view == '0') { - Py_RETURN_FALSE; - } - else if (*view == '1') { - Py_RETURN_TRUE; - } - } - else if (size == 4) { - if ( - (view[0] == 't' || view[0] == 'T') && - (view[1] == 'r' || view[1] == 'R') && - (view[2] == 'u' || view[2] == 'U') && - (view[3] == 'e' || view[3] == 'E') - ) { - Py_RETURN_TRUE; - } - } - else if (size == 5) { - if ( - (view[0] == 'f' || view[0] == 'F') && - (view[1] == 'a' || view[1] == 'A') && - (view[2] == 'l' || view[2] == 'L') && - (view[3] == 's' || view[3] == 'S') && - (view[4] == 'e' || view[4] == 'E') - ) { - Py_RETURN_FALSE; - } - } - } - - if (type->types & MS_TYPE_NONE) { - if (size == 4 && - (view[0] == 'n' || view[0] == 'N') && - (view[1] == 'u' || view[1] == 'U') && - (view[2] == 'l' || view[2] == 'L') && - (view[3] == 'l' || view[3] == 'L') - ) { - Py_RETURN_NONE; - } - } + bool invalid = false; + PyObject *out = ms_decode_str_lax(view, size, type, path, &invalid); + if (!invalid) return out; if (type->types & (MS_TYPE_ANY | MS_TYPE_STR)) { Py_INCREF(obj); @@ -17578,7 +17634,6 @@ convert_str_lax( return convert_str_uncommon(self, obj, view, size, false, type, path); } - static PyObject * convert_bytes( ConvertState *self, PyObject *obj, TypeNode *type, PathNode *path @@ -18936,6 +18991,7 @@ msgspec_clear(PyObject *m) Py_CLEAR(st->str_enc_hook); Py_CLEAR(st->str_dec_hook); Py_CLEAR(st->str_ext_hook); + Py_CLEAR(st->str_strict); Py_CLEAR(st->str_utcoffset); Py_CLEAR(st->str___origin__); Py_CLEAR(st->str___args__); @@ -19325,6 +19381,7 @@ PyInit__core(void) CACHED_STRING(str_enc_hook, "enc_hook"); CACHED_STRING(str_dec_hook, "dec_hook"); CACHED_STRING(str_ext_hook, "ext_hook"); + CACHED_STRING(str_strict, "strict"); CACHED_STRING(str_utcoffset, "utcoffset"); CACHED_STRING(str___origin__, "__origin__"); CACHED_STRING(str___args__, "__args__"); diff --git a/msgspec/json.pyi b/msgspec/json.pyi index 614f8a13..07c6f9f4 100644 --- a/msgspec/json.pyi +++ b/msgspec/json.pyi @@ -33,12 +33,14 @@ class Encoder: class Decoder(Generic[T]): type: Type[T] + strict: bool dec_hook: dec_hook_sig @overload def __init__( self: Decoder[Any], *, + strict: bool = True, dec_hook: dec_hook_sig = None, ) -> None: ... @overload @@ -46,6 +48,7 @@ class Decoder(Generic[T]): self: Decoder[T], type: Type[T] = ..., *, + strict: bool = True, dec_hook: dec_hook_sig = None, ) -> None: ... @overload @@ -53,6 +56,7 @@ class Decoder(Generic[T]): self: Decoder[Any], type: Any = ..., *, + strict: bool = True, dec_hook: dec_hook_sig = None, ) -> None: ... def decode(self, data: Union[bytes, str]) -> T: ... @@ -61,6 +65,7 @@ class Decoder(Generic[T]): def decode( buf: Union[bytes, str], *, + strict: bool = True, dec_hook: dec_hook_sig = None, ) -> Any: ... @overload @@ -68,6 +73,7 @@ def decode( buf: Union[bytes, str], *, type: Type[T] = ..., + strict: bool = True, dec_hook: dec_hook_sig = None, ) -> T: ... @overload @@ -75,6 +81,7 @@ def decode( buf: Union[bytes, str], *, type: Any = ..., + strict: bool = True, dec_hook: dec_hook_sig = None, ) -> Any: ... def encode(obj: Any, *, enc_hook: enc_hook_sig = None) -> bytes: ... diff --git a/msgspec/msgpack.pyi b/msgspec/msgpack.pyi index c19a6340..17341a12 100644 --- a/msgspec/msgpack.pyi +++ b/msgspec/msgpack.pyi @@ -24,12 +24,14 @@ class Ext: class Decoder(Generic[T]): type: Type[T] + strict: bool dec_hook: dec_hook_sig ext_hook: ext_hook_sig @overload def __init__( self: Decoder[Any], *, + strict: bool = True, dec_hook: dec_hook_sig = None, ext_hook: ext_hook_sig = None, ) -> None: ... @@ -38,6 +40,7 @@ class Decoder(Generic[T]): self: Decoder[T], type: Type[T] = ..., *, + strict: bool = True, dec_hook: dec_hook_sig = None, ext_hook: ext_hook_sig = None, ) -> None: ... @@ -46,6 +49,7 @@ class Decoder(Generic[T]): self: Decoder[Any], type: Any = ..., *, + strict: bool = True, dec_hook: dec_hook_sig = None, ext_hook: ext_hook_sig = None, ) -> None: ... @@ -69,6 +73,7 @@ class Encoder: def decode( buf: bytes, *, + strict: bool = True, dec_hook: dec_hook_sig = None, ext_hook: ext_hook_sig = None, ) -> Any: ... @@ -77,6 +82,7 @@ def decode( buf: bytes, *, type: Type[T] = ..., + strict: bool = True, dec_hook: dec_hook_sig = None, ext_hook: ext_hook_sig = None, ) -> T: ... @@ -85,6 +91,7 @@ def decode( buf: bytes, *, type: Any = ..., + strict: bool = True, dec_hook: dec_hook_sig = None, ext_hook: ext_hook_sig = None, ) -> Any: ... diff --git a/msgspec/toml.py b/msgspec/toml.py index 0c9af165..86002823 100644 --- a/msgspec/toml.py +++ b/msgspec/toml.py @@ -88,6 +88,7 @@ def encode(obj: Any, *, enc_hook: Optional[Callable[[Any], Any]] = None) -> byte def decode( buf: Union[bytes, str], *, + strict: bool = True, dec_hook: Optional[Callable[[type, Any], Any]] = None, ) -> Any: pass @@ -98,6 +99,7 @@ def decode( buf: Union[bytes, str], *, type: Type[T] = ..., + strict: bool = True, dec_hook: Optional[Callable[[type, Any], Any]] = None, ) -> T: pass @@ -108,12 +110,13 @@ def decode( buf: Union[bytes, str], *, type: Any = ..., + strict: bool = True, dec_hook: Optional[Callable[[type, Any], Any]] = None, ) -> Any: pass -def decode(buf, *, type=Any, dec_hook=None): +def decode(buf, *, type=Any, strict=True, dec_hook=None): """Deserialize an object from TOML. Parameters @@ -125,6 +128,10 @@ def decode(buf, *, type=Any, dec_hook=None): provided, the message will be type checked and decoded as the specified type. Defaults to `Any`, in which case the message will be decoded using the default TOML types. + strict : bool, optional + Whether type coercion rules should be strict. Setting to False enables + a wider set of coercion rules from string to non-string types for all + values. Default is True. dec_hook : callable, optional An optional callback for handling decoding custom types. Should have the signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` @@ -161,5 +168,6 @@ def decode(buf, *, type=Any, dec_hook=None): type, builtin_types=(_datetime.datetime, _datetime.date, _datetime.time), str_keys=True, + strict=strict, dec_hook=dec_hook, ) diff --git a/msgspec/yaml.py b/msgspec/yaml.py index ad46f6d9..6295990c 100644 --- a/msgspec/yaml.py +++ b/msgspec/yaml.py @@ -78,7 +78,10 @@ def encode(obj: Any, *, enc_hook: Optional[Callable[[Any], Any]] = None) -> byte @overload def decode( - buf: Union[bytes, str], *, dec_hook: Optional[Callable[[type, Any], Any]] = None + buf: Union[bytes, str], + *, + strict: bool = True, + dec_hook: Optional[Callable[[type, Any], Any]] = None, ) -> Any: pass @@ -88,6 +91,7 @@ def decode( buf: Union[bytes, str], *, type: Type[T] = ..., + strict: bool = True, dec_hook: Optional[Callable[[type, Any], Any]] = None, ) -> T: pass @@ -98,12 +102,13 @@ def decode( buf: Union[bytes, str], *, type: Any = ..., + strict: bool = True, dec_hook: Optional[Callable[[type, Any], Any]] = None, ) -> Any: pass -def decode(buf, *, type=Any, dec_hook=None): +def decode(buf, *, type=Any, strict=True, dec_hook=None): """Deserialize an object from YAML. Parameters @@ -115,6 +120,10 @@ def decode(buf, *, type=Any, dec_hook=None): provided, the message will be type checked and decoded as the specified type. Defaults to `Any`, in which case the message will be decoded using the default YAML types. + strict : bool, optional + Whether type coercion rules should be strict. Setting to False enables + a wider set of coercion rules from string to non-string types for all + values. Default is True. dec_hook : callable, optional An optional callback for handling decoding custom types. Should have the signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` @@ -150,5 +159,9 @@ def decode(buf, *, type=Any, dec_hook=None): if type is Any: return obj return _convert( - obj, type, builtin_types=(_datetime.datetime, _datetime.date), dec_hook=dec_hook + obj, + type, + builtin_types=(_datetime.datetime, _datetime.date), + strict=strict, + dec_hook=dec_hook, ) diff --git a/tests/basic_typing_examples.py b/tests/basic_typing_examples.py index de8aed05..2181f904 100644 --- a/tests/basic_typing_examples.py +++ b/tests/basic_typing_examples.py @@ -654,6 +654,16 @@ def ext_hook(code: int, data: memoryview) -> Any: msgspec.msgpack.Decoder(ext_hook=ext_hook) +def check_msgpack_Decoder_strict() -> None: + dec = msgspec.msgpack.Decoder(List[int], strict=False) + reveal_type(dec.strict) # assert "bool" in typ + + +def check_msgpack_decode_strict() -> None: + out = msgspec.msgpack.decode(b'', type=List[int], strict=False) + reveal_type(out) # assert "list" in typ.lower() + + def check_msgpack_Ext() -> None: ext = msgspec.msgpack.Ext(1, b"test") reveal_type(ext.code) # assert "int" in typ @@ -765,6 +775,16 @@ def dec_hook(typ: Type, obj: Any) -> Any: msgspec.json.Decoder(dec_hook=dec_hook) +def check_json_Decoder_strict() -> None: + dec = msgspec.json.Decoder(List[int], strict=False) + reveal_type(dec.strict) # assert "bool" in typ + + +def check_json_decode_strict() -> None: + out = msgspec.json.decode(b'', type=List[int], strict=False) + reveal_type(out) # assert "list" in typ.lower() + + def check_json_format() -> None: reveal_type(msgspec.json.format(b"test")) # assert "bytes" in typ reveal_type(msgspec.json.format(b"test", indent=4)) # assert "bytes" in typ @@ -812,6 +832,12 @@ def dec_hook(typ: Type, obj: Any) -> Any: msgspec.yaml.decode(b"test", dec_hook=dec_hook) + +def check_yaml_decode_strict() -> None: + out = msgspec.yaml.decode(b'', type=List[int], strict=False) + reveal_type(out) # assert "list" in typ.lower() + + ########################################################## # TOML # ########################################################## @@ -849,6 +875,11 @@ def dec_hook(typ: Type, obj: Any) -> Any: msgspec.toml.decode(b"a = 1", dec_hook=dec_hook) +def check_toml_decode_strict() -> None: + out = msgspec.toml.decode(b'', type=List[int], strict=False) + reveal_type(out) # assert "list" in typ.lower() + + ########################################################## # msgspec.inspect # ########################################################## diff --git a/tests/test_common.py b/tests/test_common.py index 60cb21f0..ee8b74c9 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -38,6 +38,7 @@ attrs = None import msgspec +from msgspec import Meta, ValidationError UTC = datetime.timezone.utc @@ -52,6 +53,11 @@ T = TypeVar("T") +def assert_eq(x, y): + assert x == y + assert type(x) is type(y) + + @pytest.fixture(params=["json", "msgpack"]) def proto(request): if request.param == "json": @@ -215,7 +221,7 @@ class Test(base_cls): assert dec.decode(proto.encode(1)) is Test.A assert dec.decode(proto.encode(2)) is Test.B - with pytest.raises(msgspec.ValidationError, match="Invalid enum value 3"): + with pytest.raises(ValidationError, match="Invalid enum value 3"): dec.decode(proto.encode(3)) def test_decode_nested(self, proto): @@ -227,7 +233,7 @@ class Test(msgspec.Struct): dec.decode(proto.encode({"fruit": 1})) == Test(FruitInt.APPLE) with pytest.raises( - msgspec.ValidationError, match=r"Invalid enum value 3 - at `\$.fruit`" + ValidationError, match=r"Invalid enum value 3 - at `\$.fruit`" ): dec.decode(proto.encode({"fruit": 3})) @@ -312,7 +318,7 @@ def test_compact(self, values): assert val == val2 for bad in [-1000, min(values) - 1, max(values) + 1, 1000]: - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(msgspec.msgpack.encode(bad)) @pytest.mark.parametrize( @@ -335,7 +341,7 @@ def test_hashtable(self, values): assert val == val2 for bad in [-2000, -1, 1, 2000]: - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(msgspec.msgpack.encode(bad)) @pytest.mark.parametrize( @@ -355,7 +361,7 @@ def test_hashtable_collisions(self, values): assert val == val2 for bad in [0, 7, 9, 56, -min(values), -max(values), 2**64 - 1, -(2**63)]: - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(msgspec.msgpack.encode(bad)) @@ -416,9 +422,7 @@ class Test(base_cls): assert dec.decode(proto.encode("apple")) is Test.A assert dec.decode(proto.encode("banana")) is Test.B - with pytest.raises( - msgspec.ValidationError, match="Invalid enum value 'cherry'" - ): + with pytest.raises(ValidationError, match="Invalid enum value 'cherry'"): dec.decode(proto.encode("cherry")) def test_decode_nested(self, proto): @@ -430,7 +434,7 @@ class Test(msgspec.Struct): dec.decode(proto.encode({"fruit": "apple"})) == Test(FruitStr.APPLE) with pytest.raises( - msgspec.ValidationError, + ValidationError, match=r"Invalid enum value 'cherry' - at `\$.fruit`", ): dec.decode(proto.encode({"fruit": "cherry"})) @@ -503,14 +507,14 @@ def strgen(length): for _ in range(10): key = unique_str() - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(msgspec.msgpack.encode(key)) # Try bad of different lengths for bad_length in [1, 7, 15, 30]: assert bad_length != length key = rand.str(bad_length) - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(msgspec.msgpack.encode(key)) @pytest.mark.parametrize("nitems", [1, 3, 6, 12, 24, 48]) @@ -539,7 +543,7 @@ def strgen(): for _ in range(10): key = unique_str() - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(msgspec.msgpack.encode(key)) @@ -625,12 +629,10 @@ def test_multiple_literals(self): for val in [-1, -2, -3, "apple", "banana"]: assert dec.decode(msgspec.msgpack.encode(val)) == val - with pytest.raises(msgspec.ValidationError, match="Invalid enum value 4"): + with pytest.raises(ValidationError, match="Invalid enum value 4"): dec.decode(msgspec.msgpack.encode(4)) - with pytest.raises( - msgspec.ValidationError, match="Invalid enum value 'carrot'" - ): + with pytest.raises(ValidationError, match="Invalid enum value 'carrot'"): dec.decode(msgspec.msgpack.encode("carrot")) def test_nested_literals(self): @@ -647,12 +649,10 @@ def test_nested_literals(self): for val in [-1, -2, -3, "apple", "banana"]: assert dec.decode(msgspec.msgpack.encode(val)) == val - with pytest.raises(msgspec.ValidationError, match="Invalid enum value 4"): + with pytest.raises(ValidationError, match="Invalid enum value 4"): dec.decode(msgspec.msgpack.encode(4)) - with pytest.raises( - msgspec.ValidationError, match="Invalid enum value 'carrot'" - ): + with pytest.raises(ValidationError, match="Invalid enum value 'carrot'"): dec.decode(msgspec.msgpack.encode("carrot")) def test_mix_int_and_int_literal(self): @@ -769,7 +769,7 @@ def test_310_union_types(self, proto): dec = proto.Decoder(int | str | None) for msg in [1, "abc", None]: assert dec.decode(proto.encode(msg)) == msg - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): assert dec.decode(proto.encode(1.5)) @@ -935,18 +935,18 @@ class Test2(msgspec.Struct, tag=tag2): ) # Tag missing - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(enc.encode({"a": 1, "b": 2})) assert "missing required field `type`" in str(rec.value) # Tag wrong type - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(enc.encode({"type": 123.456, "a": 1, "b": 2})) assert f"Expected `{type(tag1).__name__}`" in str(rec.value) assert "`$.type`" in str(rec.value) # Tag unknown - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(enc.encode({"type": unknown, "a": 1, "b": 2})) assert f"Invalid value {unknown!r} - at `$.type`" == str(rec.value) @@ -984,28 +984,28 @@ class Test3(msgspec.Struct, tag=tag3, array_like=True): assert dec.decode(enc.encode([tag1, 1, 2, 3, 4])) == Test1(1, 2, 3) # Missing required field - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(enc.encode([tag1, 1])) assert "Expected `array` of at least length 3, got 2" in str(rec.value) # Type error has correct field index - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(enc.encode([tag1, 1, "bad", 2])) assert "Expected `int`, got `str` - at `$[2]`" == str(rec.value) # Tag missing - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(enc.encode([])) assert "Expected `array` of at least length 1, got 0" == str(rec.value) # Tag wrong type - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(enc.encode([123.456, 2, 3, 4])) assert f"Expected `{type(tag1).__name__}`" in str(rec.value) assert "`$[0]`" in str(rec.value) # Tag unknown - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(enc.encode([unknown, 1, 2, 3])) assert f"Invalid value {unknown!r} - at `$[0]`" == str(rec.value) @@ -1025,7 +1025,7 @@ class Test2(msgspec.Struct, tag=True, array_like=array_like): for msg in [Test1(1, 2), Test2(3, 4), None, 5, 6]: assert dec.decode(enc.encode(msg)) == msg - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(enc.encode(True)) typ = "array" if array_like else "object" @@ -1159,7 +1159,7 @@ class Ex(msgspec.Struct, Generic[T], array_like=array_like): res = proto.decode(msg, type=Ex[float]) assert type(res.x) is float - with pytest.raises(msgspec.ValidationError, match="Expected `str`, got `int`"): + with pytest.raises(ValidationError, match="Expected `str`, got `int`"): proto.decode(msg, type=Ex[str]) @pytest.mark.parametrize("array_like", [False, True]) @@ -1183,7 +1183,7 @@ class Ex(Struct, Generic[T], array_like={array_like}): assert proto.decode(proto.encode(msg2), type=mod.Ex) == msg2 assert proto.decode(proto.encode(msg), type=mod.Ex[int]) == msg - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: proto.decode(proto.encode(msg2), type=mod.Ex[int]) if array_like: assert "`$[1][0]`" in str(rec.value) @@ -1219,13 +1219,13 @@ class Test2(msgspec.Struct, Generic[T], tag=True, array_like=array_like): assert proto.decode(s2, type=typ[str]) == msg2 assert proto.decode(s3, type=typ[str]) == msg3 - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: proto.decode(s1, type=typ[str]) assert "Expected `str | null`, got `int`" in str(rec.value) loc = "$[1]" if array_like else "$.a" assert loc in str(rec.value) - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: proto.decode(s2, type=typ[int]) assert "Expected `int`, got `str`" in str(rec.value) loc = "$[1]" if array_like else "$.x" @@ -1241,7 +1241,7 @@ def test_unbound_typevars_use_bound_if_set(self, proto): bad = proto.encode([1, {}]) with pytest.raises( - msgspec.ValidationError, + ValidationError, match=r"Expected `int \| str`, got `object` - at `\$\[1\]`", ): dec.decode(bad) @@ -1329,7 +1329,7 @@ class Ex(Generic[T]): res = proto.decode(msg, type=Ex[float]) assert type(res.x) is float - with pytest.raises(msgspec.ValidationError, match="Expected `str`, got `int`"): + with pytest.raises(ValidationError, match="Expected `str`, got `int`"): proto.decode(msg, type=Ex[str]) @pytest.mark.parametrize("module", ["dataclasses", "attrs"]) @@ -1361,7 +1361,7 @@ class Ex(Generic[T]): assert proto.decode(proto.encode(msg2), type=mod.Ex) == msg2 assert proto.decode(proto.encode(msg), type=mod.Ex[int]) == msg - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: proto.decode(proto.encode(msg2), type=mod.Ex[int]) assert "`$.b.a`" in str(rec.value) assert "Expected `int`, got `str`" in str(rec.value) @@ -1376,7 +1376,7 @@ def test_unbound_typevars_use_bound_if_set(self, proto): bad = proto.encode([1, {}]) with pytest.raises( - msgspec.ValidationError, + ValidationError, match=r"Expected `int \| str`, got `object` - at `\$\[1\]`", ): dec.decode(bad) @@ -1472,9 +1472,7 @@ class Test(msgspec.Struct, forbid_unknown_fields=True): assert proto.decode(proto.encode(good), type=Test) == good bad = proto.encode({"x": 1, "y": 2, "z": 3}) - with pytest.raises( - msgspec.ValidationError, match="Object contains unknown field `z`" - ): + with pytest.raises(ValidationError, match="Object contains unknown field `z`"): proto.decode(bad, type=Test) def test_forbid_unknown_fields_array_like(self, proto): @@ -1487,7 +1485,7 @@ class Test(msgspec.Struct, forbid_unknown_fields=True, array_like=True): bad = proto.encode([1, 2, 3]) with pytest.raises( - msgspec.ValidationError, match="Expected `array` of at most length 2" + ValidationError, match="Expected `array` of at most length 2" ): proto.decode(bad, type=Test) @@ -1510,13 +1508,13 @@ def test_rename_decode_struct(self, proto): def test_rename_decode_struct_wrong_type(self, proto): msg = proto.encode({"X": 1, "Y": "bad"}) - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: proto.decode(msg, type=PointUpper) assert "Expected `int`, got `str` - at `$.Y`" == str(rec.value) def test_rename_decode_struct_missing_field(self, proto): msg = proto.encode({"X": 1}) - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: proto.decode(msg, type=PointUpper) assert "Object missing required field `Y`" == str(rec.value) @@ -1545,14 +1543,14 @@ class Test(msgspec.Struct, kw_only=True): msg = proto.encode({"a": 1, "b": 2}) with pytest.raises( - msgspec.ValidationError, + ValidationError, match="missing required field `c`", ): proto.decode(msg, type=Test) msg = proto.encode({"c": 1, "b": 2}) with pytest.raises( - msgspec.ValidationError, + ValidationError, match="missing required field `a`", ): proto.decode(msg, type=Test) @@ -1574,14 +1572,14 @@ class Test(msgspec.Struct, kw_only=True, array_like=True): msg = proto.encode([5, 6]) with pytest.raises( - msgspec.ValidationError, + ValidationError, match="Expected `array` of at least length 3, got 2", ): proto.decode(msg, type=Test) msg = proto.encode([]) with pytest.raises( - msgspec.ValidationError, + ValidationError, match="Expected `array` of at least length 3, got 0", ): proto.decode(msg, type=Test) @@ -1675,7 +1673,7 @@ class Ex(TypedDict): dec = proto.Decoder(mod.Ex) assert dec.decode(proto.encode(msg)) == msg - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(proto.encode({"a": 1, "b": {"a": "bad"}})) assert "`$.b.a`" in str(rec.value) assert "Expected `int`, got `str`" in str(rec.value) @@ -1693,11 +1691,11 @@ class Ex(TypedDict): x2 = {"a": 1, "b": "two", "c": "extra"} assert dec.decode(proto.encode(x2)) == x - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(proto.encode({"b": "two"})) assert "Object missing required field `a`" == str(rec.value) - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(proto.encode({"a": 1, "b": 2})) assert "Expected `str`, got `int` - at `$.b`" == str(rec.value) @@ -1718,7 +1716,7 @@ class Ex(TypedDict): assert dec.decode(msg) == {"a": 2, "b": "two"} msg = temp.replace(b"x", b"a").replace(b"b", b"c") - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(msg) assert "Object missing required field `b`" == str(rec.value) @@ -1768,7 +1766,7 @@ class Ex(Base, total=False): x2 = {"a": 1, "b": "two"} assert dec.decode(proto.encode(x2)) == x2 - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(proto.encode({"b": "two"})) assert "Object missing required field `a`" == str(rec.value) @@ -1810,11 +1808,11 @@ class Ex(Base, total=False): x2 = {"a": 1, "d": False} assert dec.decode(proto.encode(x2)) == x2 - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(proto.encode({"d": False})) assert "Object missing required field `a`" == str(rec.value) - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(proto.encode({"a": 2})) assert "Object missing required field `d`" == str(rec.value) @@ -1887,7 +1885,7 @@ class Ex(TypedDict, Generic[T]): res = proto.decode(msg, type=Ex[float]) assert type(res["x"]) is float - with pytest.raises(msgspec.ValidationError, match="Expected `str`, got `int`"): + with pytest.raises(ValidationError, match="Expected `str`, got `int`"): proto.decode(msg, type=Ex[str]) def test_recursive_generic_typeddict(self, proto): @@ -1912,7 +1910,7 @@ class Ex(TypedDict, Generic[T]): assert proto.decode(proto.encode(msg2), type=mod.Ex) == msg2 assert proto.decode(proto.encode(msg), type=mod.Ex[int]) == msg - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: proto.decode(proto.encode(msg2), type=mod.Ex[int]) assert "`$.b.a`" in str(rec.value) assert "Expected `int`, got `str`" in str(rec.value) @@ -1978,7 +1976,7 @@ class Ex(NamedTuple): dec = proto.Decoder(mod.Ex) assert dec.decode(proto.encode(msg)) == msg - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(proto.encode(mod.Ex(1, ("bad", "two")))) assert "`$[1][0]`" in str(rec.value) assert "Expected `int`, got `str`" in str(rec.value) @@ -2001,11 +1999,11 @@ class Example(NamedTuple): assert res == msg suffix = ", got 1" if proto is msgspec.msgpack else "" - with pytest.raises(msgspec.ValidationError, match=f"length 3{suffix}"): + with pytest.raises(ValidationError, match=f"length 3{suffix}"): dec.decode(proto.encode((1,))) suffix = ", got 6" if proto is msgspec.msgpack else "" - with pytest.raises(msgspec.ValidationError, match=f"length 3{suffix}"): + with pytest.raises(ValidationError, match=f"length 3{suffix}"): dec.decode(proto.encode((1, 2, 3, 4, 5, 6))) @pytest.mark.parametrize("use_typing", [True, False]) @@ -2029,27 +2027,25 @@ class Example(NamedTuple): assert res == msg suffix = ", got 1" if proto is msgspec.msgpack else "" - with pytest.raises(msgspec.ValidationError, match=f"length 2 to 5{suffix}"): + with pytest.raises(ValidationError, match=f"length 2 to 5{suffix}"): dec.decode(proto.encode((1,))) suffix = ", got 6" if proto is msgspec.msgpack else "" - with pytest.raises(msgspec.ValidationError, match=f"length 2 to 5{suffix}"): + with pytest.raises(ValidationError, match=f"length 2 to 5{suffix}"): dec.decode(proto.encode((1, 2, 3, 4, 5, 6))) def test_decode_namedtuple_field_wrong_type(self, proto): dec = proto.Decoder(PersonTuple) msg = proto.encode((1, "bad", 2)) with pytest.raises( - msgspec.ValidationError, match=r"Expected `str`, got `int` - at `\$\[0\]`" + ValidationError, match=r"Expected `str`, got `int` - at `\$\[0\]`" ): dec.decode(msg) def test_decode_namedtuple_not_array(self, proto): dec = proto.Decoder(PersonTuple) msg = proto.encode({}) - with pytest.raises( - msgspec.ValidationError, match="Expected `array`, got `object`" - ): + with pytest.raises(ValidationError, match="Expected `array`, got `object`"): dec.decode(msg) def test_generic_namedtuple_info_cached(self, proto): @@ -2108,7 +2104,7 @@ class Ex(NamedTuple, Generic[T]): res = proto.decode(msg, type=Ex[float]) assert type(res.x) is float - with pytest.raises(msgspec.ValidationError, match="Expected `str`, got `int`"): + with pytest.raises(ValidationError, match="Expected `str`, got `int`"): proto.decode(msg, type=Ex[str]) def test_recursive_generic_namedtuple(self, proto): @@ -2133,7 +2129,7 @@ class Ex(NamedTuple, Generic[T]): assert proto.decode(proto.encode(msg2), type=mod.Ex) == msg2 assert proto.decode(proto.encode(msg), type=mod.Ex[int]) == msg - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: proto.decode(proto.encode(msg2), type=mod.Ex[int]) assert "`$[1][0]`" in str(rec.value) assert "Expected `int`, got `str`" in str(rec.value) @@ -2285,7 +2281,7 @@ class Ex: dec = proto.Decoder(mod.Ex) assert dec.decode(proto.encode(msg)) == msg - with pytest.raises(msgspec.ValidationError) as rec: + with pytest.raises(ValidationError) as rec: dec.decode(proto.encode({"a": 1, "b": {"a": "bad"}})) assert "`$.b.a`" in str(rec.value) assert "Expected `int`, got `str`" in str(rec.value) @@ -2349,12 +2345,12 @@ class Example: assert res == msg # Missing fields error - with pytest.raises(msgspec.ValidationError, match="missing required field `b`"): + with pytest.raises(ValidationError, match="missing required field `b`"): dec.decode(proto.encode({"a": 1})) # Incorrect field types error with pytest.raises( - msgspec.ValidationError, match=r"Expected `int`, got `str` - at `\$.a`" + ValidationError, match=r"Expected `int`, got `str` - at `\$.a`" ): dec.decode(proto.encode({"a": "bad"})) @@ -2383,7 +2379,7 @@ class Example: assert res == sol # Missing fields error - with pytest.raises(msgspec.ValidationError, match="missing required field `a`"): + with pytest.raises(ValidationError, match="missing required field `a`"): dec.decode(proto.encode({"c": 1, "d": 2, "e": 3})) def test_decode_dataclass_default_factory_errors(self, proto): @@ -2441,9 +2437,7 @@ class Example: dec = proto.Decoder(Example) msg = proto.encode([]) - with pytest.raises( - msgspec.ValidationError, match="Expected `object`, got `array`" - ): + with pytest.raises(ValidationError, match="Expected `object`, got `array`"): dec.decode(msg) @@ -2553,12 +2547,12 @@ class Example: assert res == msg # Missing fields error - with pytest.raises(msgspec.ValidationError, match="missing required field `b`"): + with pytest.raises(ValidationError, match="missing required field `b`"): dec.decode(proto.encode({"a": 1})) # Incorrect field types error with pytest.raises( - msgspec.ValidationError, match=r"Expected `int`, got `str` - at `\$.a`" + ValidationError, match=r"Expected `int`, got `str` - at `\$.a`" ): dec.decode(proto.encode({"a": "bad"})) @@ -2579,7 +2573,7 @@ class Example: assert res == msg # Missing fields error - with pytest.raises(msgspec.ValidationError, match="missing required field `a`"): + with pytest.raises(ValidationError, match="missing required field `a`"): dec.decode(proto.encode({"c": 1, "d": 2, "e": 3})) def test_decode_attrs_default_factory_errors(self, proto): @@ -2663,9 +2657,7 @@ class Example: dec = proto.Decoder(Example) msg = proto.encode([]) - with pytest.raises( - msgspec.ValidationError, match="Expected `object`, got `array`" - ): + with pytest.raises(ValidationError, match="Expected `object`, got `array`"): dec.decode(msg) @@ -2698,9 +2690,7 @@ def test_decode_date(self, proto, s): def test_decode_date_wrong_type(self, proto): msg = proto.encode([]) - with pytest.raises( - msgspec.ValidationError, match="Expected `date`, got `array`" - ): + with pytest.raises(ValidationError, match="Expected `date`, got `array`"): proto.decode(msg, type=datetime.date) @pytest.mark.parametrize( @@ -2731,7 +2721,7 @@ def test_decode_date_wrong_type(self, proto): ) def test_decode_date_malformed(self, proto, s): msg = proto.encode(s) - with pytest.raises(msgspec.ValidationError, match="Invalid RFC3339"): + with pytest.raises(ValidationError, match="Invalid RFC3339"): proto.decode(msg, type=datetime.date) @@ -2774,9 +2764,7 @@ def test_decode_time_naive(self, proto, t): def test_decode_time_wrong_type(self, proto): msg = proto.encode([]) - with pytest.raises( - msgspec.ValidationError, match="Expected `time`, got `array`" - ): + with pytest.raises(ValidationError, match="Expected `time`, got `array`"): proto.decode(msg, type=datetime.time) @pytest.mark.parametrize( @@ -2940,7 +2928,7 @@ def test_decode_time_nanos(self, proto, t, sol): ) def test_decode_time_malformed(self, proto, s): msg = proto.encode(s) - with pytest.raises(msgspec.ValidationError, match="Invalid RFC3339"): + with pytest.raises(ValidationError, match="Invalid RFC3339"): proto.decode(msg, type=datetime.time) @@ -3017,7 +3005,7 @@ def test_decode_uuid(self, proto, upper, hyphens): ) def test_decode_uuid_malformed(self, proto, uuid_str): msg = proto.encode(uuid_str) - with pytest.raises(msgspec.ValidationError, match="Invalid UUID"): + with pytest.raises(ValidationError, match="Invalid UUID"): proto.decode(msg, type=uuid.UUID) @@ -3026,14 +3014,14 @@ def test_decode_newtype(self, proto): UserId = NewType("UserId", int) assert proto.decode(proto.encode(1), type=UserId) == 1 - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): proto.decode(proto.encode("bad"), type=UserId) # Nested NewId works UserId2 = NewType("UserId2", UserId) assert proto.decode(proto.encode(1), type=UserId2) == 1 - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): proto.decode(proto.encode("bad"), type=UserId2) def test_decode_annotated_newtype(self, proto, Annotated): @@ -3041,7 +3029,7 @@ def test_decode_annotated_newtype(self, proto, Annotated): dec = proto.Decoder(Annotated[UserId, msgspec.Meta(ge=0)]) assert dec.decode(proto.encode(1)) == 1 - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(proto.encode(-1)) def test_decode_newtype_annotated(self, proto, Annotated): @@ -3049,7 +3037,7 @@ def test_decode_newtype_annotated(self, proto, Annotated): dec = proto.Decoder(UserId) assert dec.decode(proto.encode(1)) == 1 - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(proto.encode(-1)) def test_decode_annotated_newtype_annotated(self, proto, Annotated): @@ -3060,7 +3048,7 @@ def test_decode_annotated_newtype_annotated(self, proto, Annotated): assert dec.decode(proto.encode(1)) == 1 for bad in [-1, 11]: - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(proto.encode(bad)) @@ -3079,7 +3067,7 @@ def test_decode_decimal(self, proto): def test_decode_decimal_invalid(self, proto): msg = proto.encode("1..5") - with pytest.raises(msgspec.ValidationError, match="Invalid decimal string"): + with pytest.raises(ValidationError, match="Invalid decimal string"): proto.decode(msg, type=decimal.Decimal) @@ -3107,16 +3095,12 @@ def test_abstract_sequence(self, proto, typ): sol = [1, 2] msg = proto.encode(sol) assert proto.decode(msg, type=typ) == sol - with pytest.raises( - msgspec.ValidationError, match="Expected `array`, got `str`" - ): + with pytest.raises(ValidationError, match="Expected `array`, got `str`"): proto.decode(proto.encode("a"), type=typ) if PY39 or type(typ) is not abc.ABCMeta: assert proto.decode(msg, type=typ[int]) == sol - with pytest.raises( - msgspec.ValidationError, match="Expected `int`, got `str`" - ): + with pytest.raises(ValidationError, match="Expected `int`, got `str`"): proto.decode(proto.encode(["a"]), type=typ[int]) @pytest.mark.parametrize( @@ -3132,16 +3116,12 @@ def test_abstract_mapping(self, proto, typ): sol = {"x": 1, "y": 2} msg = proto.encode(sol) assert proto.decode(msg, type=typ) == sol - with pytest.raises( - msgspec.ValidationError, match="Expected `object`, got `str`" - ): + with pytest.raises(ValidationError, match="Expected `object`, got `str`"): proto.decode(proto.encode("a"), type=typ) if PY39 or type(typ) is not abc.ABCMeta: assert proto.decode(msg, type=typ[str, int]) == sol - with pytest.raises( - msgspec.ValidationError, match="Expected `int`, got `str`" - ): + with pytest.raises(ValidationError, match="Expected `int`, got `str`"): proto.decode(proto.encode({"a": "b"}), type=typ[str, int]) @@ -3210,14 +3190,14 @@ def test_decode_final(self, proto): dec = proto.Decoder(Final[int]) assert dec.decode(proto.encode(1)) == 1 - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(proto.encode("bad")) def test_decode_final_annotated(self, proto, Annotated): dec = proto.Decoder(Final[Annotated[int, msgspec.Meta(ge=0)]]) assert dec.decode(proto.encode(1)) == 1 - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(proto.encode(-1)) def test_decode_final_newtype(self, proto): @@ -3225,5 +3205,199 @@ def test_decode_final_newtype(self, proto): dec = proto.Decoder(Final[UserId]) assert dec.decode(proto.encode(1)) == 1 - with pytest.raises(msgspec.ValidationError): + with pytest.raises(ValidationError): dec.decode(proto.encode("bad")) + + +class TestLax: + @pytest.mark.parametrize("strict", [True, False]) + def test_strict_lax_decoder(self, proto, strict): + dec = proto.Decoder(List[int], strict=strict) + + assert dec.strict is strict + + msg = proto.encode(["1", "2"]) + + if strict: + with pytest.raises(ValidationError): + dec.decode(msg) + else: + assert dec.decode(msg) == [1, 2] + + def test_lax_none(self, proto): + for x in ["null", "Null", "nUll", "nuLl", "nulL"]: + msg = proto.encode(x) + assert proto.decode(msg, type=None, strict=False) is None + + for x in ["xull", "nxll", "nuxl", "nulx"]: + msg = proto.encode(x) + with pytest.raises(ValidationError, match="Expected `null`, got `str`"): + proto.decode(msg, type=None, strict=False) + + def test_lax_bool_true(self, proto): + for x in ["1", "true", "True", "tRue", "trUe", "truE"]: + msg = proto.encode(x) + assert proto.decode(msg, type=bool, strict=False) is True + + for x in ["x", "xx", "xrue", "txue", "trxe", "trux"]: + msg = proto.encode(x) + with pytest.raises(ValidationError, match="Expected `bool`, got `str`"): + assert proto.decode(msg, type=bool, strict=False) + + def test_lax_bool_false(self, proto): + for x in ["0", "false", "False", "fAlse", "faLse", "falSe", "falsE"]: + msg = proto.encode(x) + assert proto.decode(msg, type=bool, strict=False) is False + + for x in ["x", "xx", "xalse", "fxlse", "faxse", "falxe", "falsx"]: + msg = proto.encode(x) + with pytest.raises(ValidationError, match="Expected `bool`, got `str`"): + assert proto.decode(msg, type=bool, strict=False) + + def test_lax_int(self, proto): + for x in ["1", "-1", "123456"]: + msg = proto.encode(x) + assert proto.decode(msg, type=int, strict=False) == int(x) + + for x in ["a", "1a", "1.0", "1.."]: + msg = proto.encode(x) + with pytest.raises(ValidationError, match="Expected `int`, got `str`"): + proto.decode(msg, type=int, strict=False) + + def test_lax_int_constr(self, proto, Annotated): + typ = Annotated[int, Meta(ge=0)] + msg = proto.encode("1") + assert proto.decode(msg, type=typ, strict=False) == 1 + + msg = proto.encode("-1") + with pytest.raises(ValidationError): + proto.decode(msg, type=typ, strict=False) + + def test_lax_int_enum(self, proto): + class Ex(enum.IntEnum): + x = 1 + y = -2 + + def roundtrip(msg): + return proto.decode(proto.encode(msg), type=Ex, strict=False) + + assert roundtrip("1") is Ex.x + assert roundtrip("-2") is Ex.y + with pytest.raises(ValidationError, match="Invalid enum value 3"): + roundtrip("3") + with pytest.raises(ValidationError, match="Expected `int`, got `str`"): + roundtrip("A") + + def test_lax_int_literal(self, proto): + typ = Literal[1, -2] + + def roundtrip(msg): + return proto.decode(proto.encode(msg), type=typ, strict=False) + + assert roundtrip("1") == 1 + assert roundtrip("-2") == -2 + with pytest.raises(ValidationError, match="Invalid enum value 3"): + roundtrip("3") + with pytest.raises(ValidationError, match="Expected `int`, got `str`"): + roundtrip("A") + + def test_lax_float(self, proto): + for x in ["1", "-1", "123456", "1.5", "-1.5", "inf"]: + msg = proto.encode(x) + assert proto.decode(msg, type=float, strict=False) == float(x) + + for x in ["a", "1a", "1.0.0", "1.."]: + msg = proto.encode(x) + with pytest.raises(ValidationError, match="Expected `float`, got `str`"): + proto.decode(msg, type=float, strict=False) + + def test_lax_float_constr(self, proto, Annotated): + msg = proto.encode("1.5") + assert proto.decode(msg, type=Annotated[float, Meta(ge=0)], strict=False) == 1.5 + + msg = proto.encode("-1.0") + with pytest.raises(ValidationError): + proto.decode(msg, type=Annotated[float, Meta(ge=0)], strict=False) + + def test_lax_str(self, proto): + for x in ["1", "1.5", "false", "null"]: + msg = proto.encode(x) + assert proto.decode(msg, type=str, strict=False) == x + + def test_lax_str_constr(self, proto, Annotated): + typ = Annotated[str, Meta(max_length=10)] + msg = proto.encode("xxx") + assert proto.decode(msg, type=typ, strict=False) == "xxx" + + msg = proto.encode("x" * 20) + with pytest.raises(ValidationError): + proto.decode(msg, type=typ, strict=False) + + @pytest.mark.parametrize( + "x, sol", + [ + ("1", 1), + ("0", 0), + ("-1", -1), + ("12.5", 12.5), + ("inf", float("inf")), + ("true", True), + ("false", False), + ("null", None), + ("1a", "1a"), + ("falsx", "falsx"), + ("nulx", "nulx"), + ], + ) + def test_lax_union_valid(self, x, sol, proto): + typ = Union[int, float, bool, None, str] + msg = proto.encode(x) + assert_eq(proto.decode(msg, type=typ, strict=False), sol) + + @pytest.mark.parametrize("x", ["1a", "1.5a", "falsx", "trux", "nulx"]) + def test_lax_union_invalid(self, x, proto): + typ = Union[int, float, bool, None] + msg = proto.encode(x) + with pytest.raises( + ValidationError, match="Expected `int | float | bool | null`" + ): + proto.decode(msg, type=typ, strict=False) + + @pytest.mark.parametrize( + "x, err", + [ + ("-1", "`int` >= 0"), + ("184467440737095516100", "out of range"), + ("18446744073709551617", "out of range"), + ("-9223372036854775809", "out of range"), + ("100.5", "`float` <= 100.0"), + ("x" * 11, "length <= 10"), + ], + ) + def test_lax_union_invalid_constr(self, x, err, proto, Annotated): + """Ensure that values that parse properly but don't meet the specified + constraints error with a specific constraint error""" + msg = proto.encode(x) + typ = Union[ + Annotated[int, Meta(ge=0)], + Annotated[float, Meta(le=100)], + Annotated[str, Meta(max_length=10)], + ] + with pytest.raises(ValidationError, match=err): + proto.decode(msg, type=typ, strict=False) + + @pytest.mark.parametrize( + "x, sol", + [ + ("1", 1), + ("1.5", 1.5), + ("false", False), + ("true", True), + ("null", None), + ("2022-05-02", datetime.date(2022, 5, 2)), + ], + ) + def test_lax_union_extended(self, proto, x, sol): + typ = Union[int, float, bool, None, datetime.date] + msg = proto.encode(x) + assert_eq(proto.decode(msg, type=typ, strict=False), sol) diff --git a/tests/test_toml.py b/tests/test_toml.py index 3a2a31a5..a51dda76 100644 --- a/tests/test_toml.py +++ b/tests/test_toml.py @@ -179,6 +179,20 @@ def test_decode_validation_error(): msgspec.toml.decode(b"a = [1, 2, 3]", type=Dict[str, List[str]]) +@needs_decode +@pytest.mark.parametrize("strict", [True, False]) +def test_decode_strict_or_lax(strict): + msg = b"a = ['1', '2']" + typ = Dict[str, List[int]] + + if strict: + with pytest.raises(msgspec.ValidationError, match="Expected `int`"): + msgspec.toml.decode(msg, type=typ, strict=strict) + else: + res = msgspec.toml.decode(msg, type=typ, strict=strict) + assert res == {"a": [1, 2]} + + @needs_decode def test_decode_dec_hook(): def dec_hook(typ, val): diff --git a/tests/test_yaml.py b/tests/test_yaml.py index 213e09fd..78cca981 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -145,6 +145,19 @@ def test_decode_validation_error(): msgspec.yaml.decode(b"[1, 2, 3]", type=List[str]) +@pytest.mark.parametrize("strict", [True, False]) +def test_decode_strict_or_lax(strict): + msg = b"a: ['1', '2']" + typ = Dict[str, List[int]] + + if strict: + with pytest.raises(msgspec.ValidationError, match="Expected `int`"): + msgspec.yaml.decode(msg, type=typ, strict=strict) + else: + res = msgspec.yaml.decode(msg, type=typ, strict=strict) + assert res == {"a": [1, 2]} + + def test_decode_dec_hook(): def dec_hook(typ, val): if typ is Decimal: