From 32a1f073ec30f799a182d4eabbfbc3b318801a6d Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Thu, 31 Mar 2022 01:47:06 -0500 Subject: [PATCH 1/5] WIP - skip encoding defaults --- msgspec/_core.c | 132 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 114 insertions(+), 18 deletions(-) diff --git a/msgspec/_core.c b/msgspec/_core.c index 931ffa6d..35fb741a 100644 --- a/msgspec/_core.c +++ b/msgspec/_core.c @@ -1262,6 +1262,7 @@ typedef struct { char frozen; char array_like; char nogc; + char skip_encode_defaults; } StructMetaObject; static PyTypeObject StructMetaType; @@ -2775,16 +2776,18 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int arg_frozen = -1, frozen = -1; int arg_array_like = -1, array_like = -1; int arg_nogc = -1, nogc = -1; + int arg_skip_encode_defaults = -1, skip_encode_defaults = -1; static char *kwlist[] = { - "name", "bases", "dict", "tag_field", "tag", "frozen", "array_like", "nogc", NULL + "name", "bases", "dict", "tag_field", "tag", "frozen", "array_like", "nogc", "skip_encode_defaults", NULL }; /* Parse arguments: (name, bases, dict) */ if (!PyArg_ParseTupleAndKeywords( - args, kwargs, "UO!O!|$OOppp:StructMeta.__new__", kwlist, + args, kwargs, "UO!O!|$OOpppp:StructMeta.__new__", kwlist, &name, &PyTuple_Type, &bases, &PyDict_Type, &orig_dict, - &arg_tag_field, &arg_tag, &arg_frozen, &arg_array_like, &arg_nogc) + &arg_tag_field, &arg_tag, &arg_frozen, &arg_array_like, + &arg_nogc, &arg_skip_encode_defaults) ) return NULL; @@ -2836,6 +2839,9 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) frozen = STRUCT_MERGE_OPTIONS(frozen, ((StructMetaObject *)base)->frozen); array_like = STRUCT_MERGE_OPTIONS(array_like, ((StructMetaObject *)base)->array_like); nogc = STRUCT_MERGE_OPTIONS(nogc, ((StructMetaObject *)base)->nogc); + skip_encode_defaults = STRUCT_MERGE_OPTIONS( + skip_encode_defaults, ((StructMetaObject *)base)->skip_encode_defaults + ); base_fields = StructMeta_GET_FIELDS(base); base_defaults = StructMeta_GET_DEFAULTS(base); base_offsets = StructMeta_GET_OFFSETS(base); @@ -2869,6 +2875,7 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) frozen = STRUCT_MERGE_OPTIONS(frozen, arg_frozen); array_like = STRUCT_MERGE_OPTIONS(array_like, arg_array_like); nogc = STRUCT_MERGE_OPTIONS(nogc, arg_nogc); + skip_encode_defaults = STRUCT_MERGE_OPTIONS(skip_encode_defaults, arg_skip_encode_defaults); new_dict = PyDict_Copy(orig_dict); if (new_dict == NULL) @@ -3080,6 +3087,7 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) cls->frozen = frozen; cls->array_like = array_like; cls->nogc = nogc; + cls->skip_encode_defaults = skip_encode_defaults; return (PyObject *) cls; error: Py_XDECREF(arg_fields); @@ -3413,6 +3421,29 @@ maybe_deepcopy_default(PyObject *obj) { return CALL_ONE_ARG(mod->deepcopy, obj); } +static MS_INLINE bool +is_default(PyObject *x, PyObject *d) { + if (x == d) return true; + PyTypeObject *x_type = Py_TYPE(x); + PyTypeObject *d_type = Py_TYPE(d); + if (x_type != d_type) return false; + if ( + d_type == &PyList_Type + && PyList_GET_SIZE(x) == 0 + && PyList_GET_SIZE(d) == 0 + ) return true; + if ( + d_type == &PyDict_Type + && PyDict_Size(x) == 0 + && PyDict_Size(d) == 0 + ) return true; + if ( + d_type == &PySet_Type + && PySet_GET_SIZE(x) == 0 + && PySet_GET_SIZE(d) == 0 + ) return true; + return false; +} /* Set field #index on obj. Steals a reference to val */ static inline void @@ -4923,16 +4954,10 @@ mpack_encode_struct(EncoderState *self, PyObject *obj) nfields = PyTuple_GET_SIZE(fields); len = nfields + tagged; + if (Py_EnterRecursiveCall(" while serializing an object")) return -1; + if (array_like) { if (mpack_encode_array_header(self, len, "structs") < 0) return -1; - } - else { - if (mpack_encode_map_header(self, len, "structs") < 0) return -1; - } - if (len == 0) return 0; - if (Py_EnterRecursiveCall(" while serializing an object")) - return -1; - if (array_like) { if (tagged) { if (mpack_encode_str(self, tag_value) < 0) goto cleanup; } @@ -4943,7 +4968,53 @@ mpack_encode_struct(EncoderState *self, PyObject *obj) } } } + else if (struct_type->skip_encode_defaults == OPT_TRUE) { + Py_ssize_t nunchecked = nfields - PyTuple_GET_SIZE(struct_type->struct_defaults); + Py_ssize_t actual_len = len; + Py_ssize_t header_offset = self->output_len; + + if (mpack_encode_map_header(self, len, "structs") < 0) return -1; + if (tagged) { + if (mpack_encode_str(self, tag_field) < 0) goto cleanup; + if (mpack_encode_str(self, tag_value) < 0) goto cleanup; + } + + for (i = 0; i < nunchecked; i++) { + key = PyTuple_GET_ITEM(fields, i); + val = Struct_get_index(obj, i); + if (val == NULL || mpack_encode_str(self, key) < 0 || mpack_encode(self, val) < 0) { + goto cleanup; + } + } + for (i = nunchecked; i < nfields; i++) { + key = PyTuple_GET_ITEM(fields, i); + val = Struct_get_index(obj, i); + if (val == NULL) goto cleanup; + PyObject *default_val = PyTuple_GET_ITEM( + struct_type->struct_defaults, i - nunchecked + ); + if (!is_default(val, default_val)) { + if (mpack_encode_str(self, key) < 0 || mpack_encode(self, val) < 0) goto cleanup; + } + else { + actual_len--; + } + } + if (actual_len != len) { + char *header_loc = self->output_buffer_raw + header_offset; + if (len < 16) { + *header_loc = MP_FIXMAP | actual_len; + } else if (len < (1 << 16)) { + *header_loc++ = MP_MAP16; + _msgspec_store16(header_loc, (uint16_t)actual_len); + } else { + *header_loc++ = MP_MAP32; + _msgspec_store32(header_loc, (uint32_t)actual_len); + } + } + } else { + if (mpack_encode_map_header(self, len, "structs") < 0) return -1; if (tagged) { if (mpack_encode_str(self, tag_field) < 0) goto cleanup; if (mpack_encode_str(self, tag_value) < 0) goto cleanup; @@ -5794,7 +5865,7 @@ static int json_encode_struct(EncoderState *self, PyObject *obj) { PyObject *key, *val, *fields, *tag_field, *tag_value; - Py_ssize_t i, len; + Py_ssize_t i, nfields; int tagged, status = -1; StructMetaObject *struct_type = (StructMetaObject *)Py_TYPE(obj); @@ -5802,17 +5873,17 @@ json_encode_struct(EncoderState *self, PyObject *obj) tag_value = struct_type->struct_tag_value; tagged = tag_value != NULL; fields = struct_type->struct_fields; - len = PyTuple_GET_SIZE(fields); + nfields = PyTuple_GET_SIZE(fields); if (struct_type->array_like == OPT_TRUE) { - if ((len + tagged) == 0) return ms_write(self, "[]", 2); + if ((nfields + tagged) == 0) return ms_write(self, "[]", 2); if (ms_write(self, "[", 1) < 0) return -1; if (Py_EnterRecursiveCall(" while serializing an object")) return -1; if (tag_value != NULL) { if (json_encode_str(self, tag_value) < 0) goto cleanup; if (ms_write(self, ",", 1) < 0) goto cleanup; } - for (i = 0; i < len; i++) { + for (i = 0; i < nfields; i++) { val = Struct_get_index(obj, i); if (val == NULL) goto cleanup; if (json_encode(self, val) < 0) goto cleanup; @@ -5822,8 +5893,8 @@ json_encode_struct(EncoderState *self, PyObject *obj) *(self->output_buffer_raw + self->output_len - 1) = ']'; } else { - if ((len + tagged) == 0) return ms_write(self, "{}", 2); if (ms_write(self, "{", 1) < 0) return -1; + Py_ssize_t start_len = self->output_len; if (Py_EnterRecursiveCall(" while serializing an object")) return -1; if (tag_value != NULL) { if (json_encode_str(self, tag_field) < 0) goto cleanup; @@ -5831,7 +5902,13 @@ json_encode_struct(EncoderState *self, PyObject *obj) if (json_encode_str(self, tag_value) < 0) goto cleanup; if (ms_write(self, ",", 1) < 0) goto cleanup; } - for (i = 0; i < len; i++) { + + Py_ssize_t nunchecked = nfields; + if (struct_type->skip_encode_defaults == OPT_TRUE) { + nunchecked = nfields - PyTuple_GET_SIZE(struct_type->struct_defaults); + } + + for (i = 0; i < nunchecked; i++) { key = PyTuple_GET_ITEM(fields, i); val = Struct_get_index(obj, i); if (val == NULL) goto cleanup; @@ -5840,8 +5917,27 @@ json_encode_struct(EncoderState *self, PyObject *obj) if (json_encode(self, val) < 0) goto cleanup; if (ms_write(self, ",", 1) < 0) goto cleanup; } + for (i = nunchecked; i < nfields; i++) { + key = PyTuple_GET_ITEM(fields, i); + val = Struct_get_index(obj, i); + if (val == NULL) goto cleanup; + PyObject *default_val = PyTuple_GET_ITEM( + struct_type->struct_defaults, i - nunchecked + ); + if (!is_default(val, default_val)) { + if (json_encode_str_nocheck(self, key) < 0) goto cleanup; + if (ms_write(self, ":", 1) < 0) goto cleanup; + if (json_encode(self, val) < 0) goto cleanup; + if (ms_write(self, ",", 1) < 0) goto cleanup; + } + } /* Overwrite trailing comma with } */ - *(self->output_buffer_raw + self->output_len - 1) = '}'; + if (MS_UNLIKELY(start_len == self->output_len)) { + if (ms_write(self, "}", 1) < 0) goto cleanup; + } + else { + *(self->output_buffer_raw + self->output_len - 1) = '}'; + } } status = 0; cleanup: From 7289db551c81bbc0ab73f90c01a5fc84945b07b2 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Thu, 31 Mar 2022 12:10:25 -0500 Subject: [PATCH 2/5] Rename kwarg to `omit_defaults` --- msgspec/_core.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/msgspec/_core.c b/msgspec/_core.c index 35fb741a..5ff13ed5 100644 --- a/msgspec/_core.c +++ b/msgspec/_core.c @@ -1262,7 +1262,7 @@ typedef struct { char frozen; char array_like; char nogc; - char skip_encode_defaults; + char omit_defaults; } StructMetaObject; static PyTypeObject StructMetaType; @@ -2776,10 +2776,10 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int arg_frozen = -1, frozen = -1; int arg_array_like = -1, array_like = -1; int arg_nogc = -1, nogc = -1; - int arg_skip_encode_defaults = -1, skip_encode_defaults = -1; + int arg_omit_defaults = -1, omit_defaults = -1; static char *kwlist[] = { - "name", "bases", "dict", "tag_field", "tag", "frozen", "array_like", "nogc", "skip_encode_defaults", NULL + "name", "bases", "dict", "tag_field", "tag", "frozen", "array_like", "nogc", "omit_defaults", NULL }; /* Parse arguments: (name, bases, dict) */ @@ -2787,7 +2787,7 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) args, kwargs, "UO!O!|$OOpppp:StructMeta.__new__", kwlist, &name, &PyTuple_Type, &bases, &PyDict_Type, &orig_dict, &arg_tag_field, &arg_tag, &arg_frozen, &arg_array_like, - &arg_nogc, &arg_skip_encode_defaults) + &arg_nogc, &arg_omit_defaults) ) return NULL; @@ -2839,8 +2839,8 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) frozen = STRUCT_MERGE_OPTIONS(frozen, ((StructMetaObject *)base)->frozen); array_like = STRUCT_MERGE_OPTIONS(array_like, ((StructMetaObject *)base)->array_like); nogc = STRUCT_MERGE_OPTIONS(nogc, ((StructMetaObject *)base)->nogc); - skip_encode_defaults = STRUCT_MERGE_OPTIONS( - skip_encode_defaults, ((StructMetaObject *)base)->skip_encode_defaults + omit_defaults = STRUCT_MERGE_OPTIONS( + omit_defaults, ((StructMetaObject *)base)->omit_defaults ); base_fields = StructMeta_GET_FIELDS(base); base_defaults = StructMeta_GET_DEFAULTS(base); @@ -2875,7 +2875,7 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) frozen = STRUCT_MERGE_OPTIONS(frozen, arg_frozen); array_like = STRUCT_MERGE_OPTIONS(array_like, arg_array_like); nogc = STRUCT_MERGE_OPTIONS(nogc, arg_nogc); - skip_encode_defaults = STRUCT_MERGE_OPTIONS(skip_encode_defaults, arg_skip_encode_defaults); + omit_defaults = STRUCT_MERGE_OPTIONS(omit_defaults, arg_omit_defaults); new_dict = PyDict_Copy(orig_dict); if (new_dict == NULL) @@ -3087,7 +3087,7 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) cls->frozen = frozen; cls->array_like = array_like; cls->nogc = nogc; - cls->skip_encode_defaults = skip_encode_defaults; + cls->omit_defaults = omit_defaults; return (PyObject *) cls; error: Py_XDECREF(arg_fields); @@ -4968,7 +4968,7 @@ mpack_encode_struct(EncoderState *self, PyObject *obj) } } } - else if (struct_type->skip_encode_defaults == OPT_TRUE) { + else if (struct_type->omit_defaults == OPT_TRUE) { Py_ssize_t nunchecked = nfields - PyTuple_GET_SIZE(struct_type->struct_defaults); Py_ssize_t actual_len = len; Py_ssize_t header_offset = self->output_len; @@ -5904,7 +5904,7 @@ json_encode_struct(EncoderState *self, PyObject *obj) } Py_ssize_t nunchecked = nfields; - if (struct_type->skip_encode_defaults == OPT_TRUE) { + if (struct_type->omit_defaults == OPT_TRUE) { nunchecked = nfields - PyTuple_GET_SIZE(struct_type->struct_defaults); } From 9a80798340b6961093703fbdf297752818c27494 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Thu, 31 Mar 2022 13:21:46 -0500 Subject: [PATCH 3/5] Minor performance improvements The extra branching in `json_encode_struct` led to a minor slowdown, splitting things out a bit reduces the cost here to something I'm more comfortable with. --- msgspec/_core.c | 272 +++++++++++++++++++++++++++++------------------- 1 file changed, 165 insertions(+), 107 deletions(-) diff --git a/msgspec/_core.c b/msgspec/_core.c index 5ff13ed5..886856f0 100644 --- a/msgspec/_core.c +++ b/msgspec/_core.c @@ -4945,7 +4945,6 @@ mpack_encode_struct(EncoderState *self, PyObject *obj) Py_ssize_t i, nfields, len; int tagged, status = -1; StructMetaObject *struct_type = (StructMetaObject *)Py_TYPE(obj); - bool array_like = struct_type->array_like == OPT_TRUE; tag_field = struct_type->struct_tag_field; tag_value = struct_type->struct_tag_value; @@ -4956,7 +4955,7 @@ mpack_encode_struct(EncoderState *self, PyObject *obj) if (Py_EnterRecursiveCall(" while serializing an object")) return -1; - if (array_like) { + if (struct_type->array_like == OPT_TRUE) { if (mpack_encode_array_header(self, len, "structs") < 0) return -1; if (tagged) { if (mpack_encode_str(self, tag_value) < 0) goto cleanup; @@ -4968,62 +4967,62 @@ mpack_encode_struct(EncoderState *self, PyObject *obj) } } } - else if (struct_type->omit_defaults == OPT_TRUE) { - Py_ssize_t nunchecked = nfields - PyTuple_GET_SIZE(struct_type->struct_defaults); - Py_ssize_t actual_len = len; + else { Py_ssize_t header_offset = self->output_len; - if (mpack_encode_map_header(self, len, "structs") < 0) return -1; + if (tagged) { if (mpack_encode_str(self, tag_field) < 0) goto cleanup; if (mpack_encode_str(self, tag_value) < 0) goto cleanup; } - for (i = 0; i < nunchecked; i++) { - key = PyTuple_GET_ITEM(fields, i); - val = Struct_get_index(obj, i); - if (val == NULL || mpack_encode_str(self, key) < 0 || mpack_encode(self, val) < 0) { - goto cleanup; - } - } - for (i = nunchecked; i < nfields; i++) { - key = PyTuple_GET_ITEM(fields, i); - val = Struct_get_index(obj, i); - if (val == NULL) goto cleanup; - PyObject *default_val = PyTuple_GET_ITEM( - struct_type->struct_defaults, i - nunchecked - ); - if (!is_default(val, default_val)) { - if (mpack_encode_str(self, key) < 0 || mpack_encode(self, val) < 0) goto cleanup; + if (struct_type->omit_defaults == OPT_TRUE) { + Py_ssize_t nunchecked = nfields - PyTuple_GET_SIZE(struct_type->struct_defaults); + Py_ssize_t actual_len = len; + + for (i = 0; i < nunchecked; i++) { + key = PyTuple_GET_ITEM(fields, i); + val = Struct_get_index(obj, i); + if (val == NULL || mpack_encode_str(self, key) < 0 || mpack_encode(self, val) < 0) { + goto cleanup; + } } - else { - actual_len--; + for (i = nunchecked; i < nfields; i++) { + key = PyTuple_GET_ITEM(fields, i); + val = Struct_get_index(obj, i); + if (val == NULL) goto cleanup; + PyObject *default_val = PyTuple_GET_ITEM( + struct_type->struct_defaults, i - nunchecked + ); + if (!is_default(val, default_val)) { + if (mpack_encode_str(self, key) < 0 || mpack_encode(self, val) < 0) goto cleanup; + } + else { + actual_len--; + } } - } - if (actual_len != len) { - char *header_loc = self->output_buffer_raw + header_offset; - if (len < 16) { - *header_loc = MP_FIXMAP | actual_len; - } else if (len < (1 << 16)) { - *header_loc++ = MP_MAP16; - _msgspec_store16(header_loc, (uint16_t)actual_len); - } else { - *header_loc++ = MP_MAP32; - _msgspec_store32(header_loc, (uint32_t)actual_len); + if (actual_len != len) { + /* Fixup the header length after we know how many fields were + * actually written */ + char *header_loc = self->output_buffer_raw + header_offset; + if (len < 16) { + *header_loc = MP_FIXMAP | actual_len; + } else if (len < (1 << 16)) { + *header_loc++ = MP_MAP16; + _msgspec_store16(header_loc, (uint16_t)actual_len); + } else { + *header_loc++ = MP_MAP32; + _msgspec_store32(header_loc, (uint32_t)actual_len); + } } } - } - else { - if (mpack_encode_map_header(self, len, "structs") < 0) return -1; - if (tagged) { - if (mpack_encode_str(self, tag_field) < 0) goto cleanup; - if (mpack_encode_str(self, tag_value) < 0) goto cleanup; - } - for (i = 0; i < nfields; i++) { - key = PyTuple_GET_ITEM(fields, i); - val = Struct_get_index(obj, i); - if (val == NULL || mpack_encode_str(self, key) < 0 || mpack_encode(self, val) < 0) { - goto cleanup; + else { + for (i = 0; i < nfields; i++) { + key = PyTuple_GET_ITEM(fields, i); + val = Struct_get_index(obj, i); + if (val == NULL || mpack_encode_str(self, key) < 0 || mpack_encode(self, val) < 0) { + goto cleanup; + } } } } @@ -5862,82 +5861,95 @@ json_encode_dict(EncoderState *self, PyObject *obj) } static int -json_encode_struct(EncoderState *self, PyObject *obj) -{ +json_encode_struct_default( + EncoderState *self, StructMetaObject *struct_type, PyObject *obj +) { PyObject *key, *val, *fields, *tag_field, *tag_value; Py_ssize_t i, nfields; - int tagged, status = -1; - StructMetaObject *struct_type = (StructMetaObject *)Py_TYPE(obj); - + int status = -1; tag_field = struct_type->struct_tag_field; tag_value = struct_type->struct_tag_value; - tagged = tag_value != NULL; fields = struct_type->struct_fields; nfields = PyTuple_GET_SIZE(fields); - if (struct_type->array_like == OPT_TRUE) { - if ((nfields + tagged) == 0) return ms_write(self, "[]", 2); - if (ms_write(self, "[", 1) < 0) return -1; - if (Py_EnterRecursiveCall(" while serializing an object")) return -1; - if (tag_value != NULL) { - if (json_encode_str(self, tag_value) < 0) goto cleanup; - if (ms_write(self, ",", 1) < 0) goto cleanup; - } - for (i = 0; i < nfields; i++) { - val = Struct_get_index(obj, i); - if (val == NULL) goto cleanup; - if (json_encode(self, val) < 0) goto cleanup; - if (ms_write(self, ",", 1) < 0) goto cleanup; - } - /* Overwrite trailing comma with ] */ - *(self->output_buffer_raw + self->output_len - 1) = ']'; + if (nfields == 0 && tag_value == NULL) return ms_write(self, "{}", 2); + if (ms_write(self, "{", 1) < 0) return -1; + if (Py_EnterRecursiveCall(" while serializing an object")) return -1; + if (tag_value != NULL) { + if (json_encode_str(self, tag_field) < 0) goto cleanup; + if (ms_write(self, ":", 1) < 0) goto cleanup; + if (json_encode_str(self, tag_value) < 0) goto cleanup; + if (ms_write(self, ",", 1) < 0) goto cleanup; } - else { - if (ms_write(self, "{", 1) < 0) return -1; - Py_ssize_t start_len = self->output_len; - if (Py_EnterRecursiveCall(" while serializing an object")) return -1; - if (tag_value != NULL) { - if (json_encode_str(self, tag_field) < 0) goto cleanup; - if (ms_write(self, ":", 1) < 0) goto cleanup; - if (json_encode_str(self, tag_value) < 0) goto cleanup; - if (ms_write(self, ",", 1) < 0) goto cleanup; - } + for (i = 0; i < nfields; i++) { + key = PyTuple_GET_ITEM(fields, i); + val = Struct_get_index(obj, i); + if (val == NULL) goto cleanup; + if (json_encode_str_nocheck(self, key) < 0) goto cleanup; + if (ms_write(self, ":", 1) < 0) goto cleanup; + if (json_encode(self, val) < 0) goto cleanup; + if (ms_write(self, ",", 1) < 0) goto cleanup; + } + /* Overwrite trailing comma with } */ + *(self->output_buffer_raw + self->output_len - 1) = '}'; + status = 0; +cleanup: + Py_LeaveRecursiveCall(); + return status; +} - Py_ssize_t nunchecked = nfields; - if (struct_type->omit_defaults == OPT_TRUE) { - nunchecked = nfields - PyTuple_GET_SIZE(struct_type->struct_defaults); - } +static int +json_encode_struct_omit_defaults( + EncoderState *self, StructMetaObject *struct_type, PyObject *obj +) { + PyObject *key, *val, *fields, *defaults, *tag_field, *tag_value; + Py_ssize_t i, nfields, nunchecked; + int status = -1; + tag_field = struct_type->struct_tag_field; + tag_value = struct_type->struct_tag_value; + fields = struct_type->struct_fields; + defaults = struct_type->struct_defaults; + nfields = PyTuple_GET_SIZE(fields); + nunchecked = nfields - PyTuple_GET_SIZE(defaults); - for (i = 0; i < nunchecked; i++) { - key = PyTuple_GET_ITEM(fields, i); - val = Struct_get_index(obj, i); - if (val == NULL) goto cleanup; + if (ms_write(self, "{", 1) < 0) return -1; + Py_ssize_t start_len = self->output_len; + if (Py_EnterRecursiveCall(" while serializing an object")) return -1; + if (tag_value != NULL) { + if (json_encode_str(self, tag_field) < 0) goto cleanup; + if (ms_write(self, ":", 1) < 0) goto cleanup; + if (json_encode_str(self, tag_value) < 0) goto cleanup; + if (ms_write(self, ",", 1) < 0) goto cleanup; + } + + for (i = 0; i < nunchecked; i++) { + key = PyTuple_GET_ITEM(fields, i); + val = Struct_get_index(obj, i); + if (val == NULL) goto cleanup; + if (json_encode_str_nocheck(self, key) < 0) goto cleanup; + if (ms_write(self, ":", 1) < 0) goto cleanup; + if (json_encode(self, val) < 0) goto cleanup; + if (ms_write(self, ",", 1) < 0) goto cleanup; + } + for (i = nunchecked; i < nfields; i++) { + key = PyTuple_GET_ITEM(fields, i); + val = Struct_get_index(obj, i); + if (val == NULL) goto cleanup; + PyObject *default_val = PyTuple_GET_ITEM(defaults, i - nunchecked); + if (!is_default(val, default_val)) { if (json_encode_str_nocheck(self, key) < 0) goto cleanup; if (ms_write(self, ":", 1) < 0) goto cleanup; if (json_encode(self, val) < 0) goto cleanup; if (ms_write(self, ",", 1) < 0) goto cleanup; } - for (i = nunchecked; i < nfields; i++) { - key = PyTuple_GET_ITEM(fields, i); - val = Struct_get_index(obj, i); - if (val == NULL) goto cleanup; - PyObject *default_val = PyTuple_GET_ITEM( - struct_type->struct_defaults, i - nunchecked - ); - if (!is_default(val, default_val)) { - if (json_encode_str_nocheck(self, key) < 0) goto cleanup; - if (ms_write(self, ":", 1) < 0) goto cleanup; - if (json_encode(self, val) < 0) goto cleanup; - if (ms_write(self, ",", 1) < 0) goto cleanup; - } - } + } + if (MS_UNLIKELY(start_len == self->output_len)) { + /* Empty struct, append "}" */ + if (ms_write(self, "}", 1) < 0) goto cleanup; + } + else { /* Overwrite trailing comma with } */ - if (MS_UNLIKELY(start_len == self->output_len)) { - if (ms_write(self, "}", 1) < 0) goto cleanup; - } - else { - *(self->output_buffer_raw + self->output_len - 1) = '}'; - } + *(self->output_buffer_raw + self->output_len - 1) = '}'; } status = 0; cleanup: @@ -5945,6 +5957,52 @@ json_encode_struct(EncoderState *self, PyObject *obj) return status; } +static int +json_encode_struct_array_like( + EncoderState *self, StructMetaObject *struct_type, PyObject *obj +) { + int status = -1; + PyObject *tag_value = struct_type->struct_tag_value; + PyObject *fields = struct_type->struct_fields; + Py_ssize_t nfields = PyTuple_GET_SIZE(fields); + + if (nfields == 0 && tag_value == NULL) return ms_write(self, "[]", 2); + if (ms_write(self, "[", 1) < 0) return -1; + if (Py_EnterRecursiveCall(" while serializing an object")) return -1; + if (tag_value != NULL) { + if (json_encode_str(self, tag_value) < 0) goto cleanup; + if (ms_write(self, ",", 1) < 0) goto cleanup; + } + for (Py_ssize_t i = 0; i < nfields; i++) { + PyObject *val = Struct_get_index(obj, i); + if (val == NULL) goto cleanup; + if (json_encode(self, val) < 0) goto cleanup; + if (ms_write(self, ",", 1) < 0) goto cleanup; + } + /* Overwrite trailing comma with ] */ + *(self->output_buffer_raw + self->output_len - 1) = ']'; + status = 0; +cleanup: + Py_LeaveRecursiveCall(); + return status; +} + +static int +json_encode_struct(EncoderState *self, PyObject *obj) +{ + StructMetaObject *struct_type = (StructMetaObject *)Py_TYPE(obj); + + if (struct_type->array_like == OPT_TRUE) { + return json_encode_struct_array_like(self, struct_type, obj); + } + else if (struct_type->omit_defaults == OPT_TRUE) { + return json_encode_struct_omit_defaults(self, struct_type, obj); + } + else { + return json_encode_struct_default(self, struct_type, obj); + } +} + static int json_encode(EncoderState *self, PyObject *obj) { From afeeb9bfab25ef3b3d5e10a552cbfd8f8e6d73e7 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Thu, 31 Mar 2022 13:58:29 -0500 Subject: [PATCH 4/5] Add tests --- msgspec/__init__.pyi | 1 + msgspec/_core.c | 8 +++++ tests/mypy_examples.py | 11 +++++++ tests/test_common.py | 75 +++++++++++++++++++++++++++++++++++++++++- tests/test_struct.py | 2 +- 5 files changed, 95 insertions(+), 2 deletions(-) diff --git a/msgspec/__init__.pyi b/msgspec/__init__.pyi index 842738da..39ec6a53 100644 --- a/msgspec/__init__.pyi +++ b/msgspec/__init__.pyi @@ -28,6 +28,7 @@ class Struct(metaclass=__StructMeta): cls, tag: Union[None, bool, str, Callable[[str], str]] = None, tag_field: Union[None, str] = None, + omit_defaults: bool = False, frozen: bool = False, array_like: bool = False, nogc: bool = False, diff --git a/msgspec/_core.c b/msgspec/_core.c index 886856f0..600f0b06 100644 --- a/msgspec/_core.c +++ b/msgspec/_core.c @@ -3248,6 +3248,13 @@ StructMeta_nogc(StructMetaObject *self, void *closure) else { Py_RETURN_FALSE; } } +static PyObject* +StructMeta_omit_defaults(StructMetaObject *self, void *closure) +{ + if (self->omit_defaults == OPT_TRUE) { Py_RETURN_TRUE; } + else { Py_RETURN_FALSE; } +} + static PyObject* StructMeta_signature(StructMetaObject *self, void *closure) { @@ -3353,6 +3360,7 @@ static PyGetSetDef StructMeta_getset[] = { {"frozen", (getter) StructMeta_frozen, NULL, NULL, NULL}, {"array_like", (getter) StructMeta_array_like, NULL, NULL, NULL}, {"nogc", (getter) StructMeta_nogc, NULL, NULL, NULL}, + {"omit_defaults", (getter) StructMeta_omit_defaults, NULL, NULL, NULL}, {NULL}, }; diff --git a/tests/mypy_examples.py b/tests/mypy_examples.py index 5ad1a350..ae08b338 100644 --- a/tests/mypy_examples.py +++ b/tests/mypy_examples.py @@ -28,6 +28,17 @@ class Test(msgspec.Struct): reveal_type(t.y) # assert "str" in typ +def check_struct_omit_defaults() -> None: + class Test(msgspec.Struct, omit_defaults=True): + x: int + y: str + + t = Test(1, "foo") + reveal_type(t) # assert "Test" in typ + reveal_type(t.x) # assert "int" in typ + reveal_type(t.y) # assert "str" in typ + + def check_struct_array_like() -> None: class Test(msgspec.Struct, array_like=True): x: int diff --git a/tests/test_common.py b/tests/test_common.py index 47606081..b9577181 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -6,7 +6,7 @@ import random import string import weakref -from typing import Literal, List, Union, Deque, NamedTuple, Dict, Tuple +from typing import Literal, List, Union, Deque, NamedTuple, Dict, Tuple, Optional import pytest @@ -830,3 +830,76 @@ class Test2(msgspec.Struct, tag=True): assert len(cache) == MAX_CACHE_SIZE assert first not in cache assert frozenset(new) in cache + + +class TestStructOmitDefaults: + def test_omit_defaults(self, proto): + class Test(msgspec.Struct, omit_defaults=True): + a: int = 0 + b: bool = False + c: Optional[str] = None + d: list = [] + e: Union[list, set] = set() + f: dict = {} + + cases = [ + (Test(), {}), + (Test(1), {"a": 1}), + (Test(1, False), {"a": 1}), + (Test(1, True), {"a": 1, "b": True}), + (Test(1, c=None), {"a": 1}), + (Test(1, c="test"), {"a": 1, "c": "test"}), + (Test(1, d=[1]), {"a": 1, "d": [1]}), + (Test(1, e={1}), {"a": 1, "e": [1]}), + (Test(1, e=[]), {"a": 1, "e": []}), + (Test(1, f={"a": 1}), {"a": 1, "f": {"a": 1}}), + ] + + for obj, sol in cases: + res = proto.decode(proto.encode(obj)) + assert res == sol + + def test_omit_defaults_positional(self, proto): + class Test(msgspec.Struct, omit_defaults=True): + a: int + b: bool = False + + cases = [ + (Test(1), {"a": 1}), + (Test(1, False), {"a": 1}), + (Test(1, True), {"a": 1, "b": True}), + ] + + for obj, sol in cases: + res = proto.decode(proto.encode(obj)) + assert res == sol + + def test_omit_defaults_tagged(self, proto): + class Test(msgspec.Struct, omit_defaults=True, tag=True): + a: int + b: bool = False + + cases = [ + (Test(1), {"type": "Test", "a": 1}), + (Test(1, False), {"type": "Test", "a": 1}), + (Test(1, True), {"type": "Test", "a": 1, "b": True}), + ] + + for obj, sol in cases: + res = proto.decode(proto.encode(obj)) + assert res == sol + + def test_omit_defaults_ignored_for_array_like(self, proto): + class Test(msgspec.Struct, omit_defaults=True, array_like=True): + a: int + b: bool = False + + cases = [ + (Test(1), [1, False]), + (Test(1, False), [1, False]), + (Test(1, True), [1, True]), + ] + + for obj, sol in cases: + res = proto.decode(proto.encode(obj)) + assert res == sol diff --git a/tests/test_struct.py b/tests/test_struct.py index a237ace4..0740540c 100644 --- a/tests/test_struct.py +++ b/tests/test_struct.py @@ -746,7 +746,7 @@ def test_struct_handles_missing_attributes(): pickle.dumps(t) -@pytest.mark.parametrize("option", ["frozen", "array_like", "nogc"]) +@pytest.mark.parametrize("option", ["frozen", "array_like", "nogc", "omit_defaults"]) def test_struct_option_precedence(option): def get(cls): return getattr(cls, option) From 55f0dfc505f948d5c4ce909ff2d99ccd4dcb0dfe Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Thu, 31 Mar 2022 14:52:03 -0500 Subject: [PATCH 5/5] Add docs --- docs/source/perf-tips.rst | 14 ++++++++ docs/source/structs.rst | 69 +++++++++++++++++++++++++++++++++++++++ msgspec/_core.c | 3 ++ 3 files changed, 86 insertions(+) diff --git a/docs/source/perf-tips.rst b/docs/source/perf-tips.rst index 35c35261..e31bb555 100644 --- a/docs/source/perf-tips.rst +++ b/docs/source/perf-tips.rst @@ -48,6 +48,20 @@ defining a `msgspec.Struct` type (or types) for your schema and preferring that over other types like `dict`/`dataclasses`/... +Avoid Encoding Default Values +----------------------------- + +By default, ``msgspec`` encodes all fields in a Struct type, including optional +fields (those configured with a default value). If the default values are known +on the decoding end (making serializing them redundant), it may be beneficial +to omit default values from the encoded message. This can be done by +configuring ``omit_defaults=True`` as part of the Struct definition Omitting +defaults reduces the size of the encoded message, and often also improves +encoding and decoding performance (since there's less work to do). + +For more information, see :ref:`omit_defaults`. + + Avoid Decoding Unused Fields ---------------------------- diff --git a/docs/source/structs.rst b/docs/source/structs.rst index b7b2f7f5..285a0bf5 100644 --- a/docs/source/structs.rst +++ b/docs/source/structs.rst @@ -336,6 +336,75 @@ for all struct types you wish to tag. Get(key='my key') +.. _omit_defaults: + +Omitting Default Values +----------------------- + +By default, ``msgspec`` encodes all fields in a Struct type, including optional +fields (those configured with a default value). + +.. code-block:: python + + >>> import msgspec + + >>> class User(msgspec.Struct): + ... name : str + ... email : Optional[str] = None + ... groups : Set[str] = set() + + >>> alice = User("alice") + + >>> alice # email & groups are using the default values + User(name='alice', email=None, groups=set()) + + >>> msgspec.json.encode(alice) # default values are present in encoded message + b'{"name":"alice","email":null,"groups":[]}' + +If the default values are known on the decoding end (making serializing them +redundant), it may be beneficial and desired to omit default values from the +encoded message. This can be done by configuring ``omit_defaults=True`` as part +of the Struct definition: + +.. code-block:: python + + >>> import msgspec + + >>> class User(msgspec.Struct, omit_defaults=True): + ... name : str + ... email : Optional[str] = None + ... groups : Set[str] = set() + + >>> alice = User("alice") + + >>> msgspec.json.encode(alice) # default values are omitted + b'{"name":"alice"}' + + >>> bob = User("bob", email="bob@company.com") + + >>> msgspec.json.encode(bob) + b'{"name":"bob","email":"bob@company.com"}' + +Omitting defaults reduces the size of the encoded message, and often also +improves encoding and decoding performance (since there's less work to do). + +Note that detection of default values is optimized for performance; in certain +situations a default value may still be encoded. For the curious, the current +detection logic is as follows: + +.. code-block:: python + + >>> def matches_default(value: Any, default: Any) -> bool: + ... """Whether a value matches the default for a field""" + ... if value is default: + ... return True + ... if type(value) != type(default): + ... return False + ... if type(value) in (list, set, dict) and (len(value) == len(default) == 0): + ... return True + ... return False + + Encoding/Decoding as Arrays --------------------------- diff --git a/msgspec/_core.c b/msgspec/_core.c index 600f0b06..7e20beb4 100644 --- a/msgspec/_core.c +++ b/msgspec/_core.c @@ -3870,6 +3870,9 @@ PyDoc_STRVAR(Struct__doc__, " that takes a string (the class name) and returns a new string to use for the\n" " tag value (for example, to automatically lowercase class names). See the docs\n" " for more information.\n" +"- ``omit_defaults``: whether fields should be omitted from encoding if the field\n" +" contains the default value for that field. Enabling this may reduce message\n" +" size, and often also improve encoding & decoding performance.\n" "- ``array_like``: whether this type should be treated as an array-like type\n" " (rather than a dict-like type, the default) when encoding/decoding.\n" "- ``nogc``: whether garbage collection is disabled for this class. Enabling,\n"