Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,8 @@ msgspec
|github| |pypi| |conda|

``msgspec`` is a *fast* and *friendly* serialization library for Python,
supporting both `JSON <https://json.org>`__ and `MessagePack
<https://msgpack.org>`__. It integrates well with Python's `type annotations
<https://docs.python.org/3/library/typing.html>`__, providing ergonomic (and
performant!) schema validation.
supporting both JSON_ and MessagePack_. It integrates well with Python's `type
annotations`_, providing ergonomic (and performant!) schema validation.

**Define** your message schemas using standard Python type annotations.

Expand Down Expand Up @@ -50,10 +48,9 @@ performant!) schema validation.
msgspec.DecodingError: Expected `str`, got `int` - at `$.groups[0]`

``msgspec`` is designed to be as performant as possible, while retaining some
of the nicities of validation libraries like `pydantic
<https://pydantic-docs.helpmanual.io/>`__. For supported types,
encoding/decoding a message with ``msgspec`` can be *~2-40x faster*
than alternative libraries.
of the nicities of validation libraries like pydantic_. For supported types,
encoding/decoding a message with ``msgspec`` can be *~2-40x faster* than
alternative libraries.

.. image:: https://github.com/jcrist/msgspec/raw/master/docs/source/_static/bench-1.png
:target: https://jcristharif.com/msgspec/benchmarks.html
Expand All @@ -67,6 +64,11 @@ LICENSE
New BSD. See the
`License File <https://github.com/jcrist/msgspec/blob/master/LICENSE>`_.

.. _type annotations: https://docs.python.org/3/library/typing.html
.. _JSON: https://json.org
.. _MessagePack: https://msgpack.org
.. _pydantic: https://pydantic-docs.helpmanual.io/

.. |github| image:: https://github.com/jcrist/msgspec/actions/workflows/ci.yml/badge.svg
:target: https://github.com/jcrist/msgspec/actions/workflows/ci.yml
.. |pypi| image:: https://img.shields.io/pypi/v/msgspec.svg
Expand Down
16 changes: 8 additions & 8 deletions docs/source/structs.rst
Original file line number Diff line number Diff line change
Expand Up @@ -89,30 +89,30 @@ for converting a struct to a dict.
>>> p.to_dict()
{"x": 1.0, "y": 2.0}

Immutability
------------
Frozen Instances
----------------

A struct type can optionally be marked as immutable by specifying
``immutable=True``. This disables modifying field values after initialization,
A struct type can optionally be marked as "frozen" by specifying
``frozen=True``. This disables modifying attributes after initialization,
and adds a ``__hash__`` method to the class definition.

.. code-block:: python

>>> class Point(msgspec.Struct, immutable=True):
>>> class Point(msgspec.Struct, frozen=True):
... """This struct is immutable & hashable"""
... x: float
... y: float
...

>>> p = Point(1.0, 2.0)

>>> {p: 1} # immutable structs are hashable, and can be keys in dicts
>>> {p: 1} # frozen structs are hashable, and can be keys in dicts
{Point(1.0, 2.0): 1}

>>> p.x = 2.0 # immutable structs cannot be modified after creation
>>> p.x = 2.0 # frozen structs cannot be modified after creation
Traceback (most recent call last):
...
TypeError: immutable type: 'Point'
AttributeError: immutable type: 'Point'

Encoding/Decoding as Arrays
---------------------------
Expand Down
35 changes: 35 additions & 0 deletions msgspec/__init__.pyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
from typing import Type
from typing import Callable, Tuple, Any, Union, TypeVar

# Use `__dataclass_transform__` to catch more errors under pyright. Since we don't expose
# the underlying metaclass, hide it under an underscore name. See
# https://github.com/microsoft/pyright/blob/main/specs/dataclass_transforms.md
# for more information.

_T = TypeVar("_T")

def __dataclass_transform__(
*,
eq_default: bool = True,
order_default: bool = False,
kw_only_default: bool = False,
field_descriptors: Tuple[Union[type, Callable[..., Any]], ...] = (()),
) -> Callable[[_T], _T]: ...
@__dataclass_transform__()
class __StructMeta(type):
def __new__(
cls: Type[type], name: str, bases: tuple, classdict: dict
) -> "__StructMeta": ...

class Struct(metaclass=__StructMeta):
def __init__(self, *args: Any, **kwargs: Any) -> None: ...
def __init_subclass__(cls, asarray: bool = False, frozen: bool = False) -> None: ...

class MsgspecError(Exception): ...
class DecodingError(MsgspecError): ...
class EncodingError(MsgspecError): ...

from . import msgpack
from . import json

__version__: str
46 changes: 25 additions & 21 deletions msgspec/_core.c
Original file line number Diff line number Diff line change
Expand Up @@ -526,7 +526,7 @@ typedef struct {
TypeNode **struct_types;
bool json_compatible;
bool traversing;
char immutable;
char frozen;
char asarray;
} StructMetaObject;

Expand Down Expand Up @@ -1475,15 +1475,15 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
PyObject *default_val, *field;
Py_ssize_t nfields, ndefaults, i, j, k;
Py_ssize_t *offsets = NULL, *base_offsets;
int arg_immutable = -1, arg_asarray = -1, immutable = -1, asarray = -1;
int arg_frozen = -1, arg_asarray = -1, frozen = -1, asarray = -1;

static char *kwlist[] = {"name", "bases", "dict", "immutable", "asarray", NULL};
static char *kwlist[] = {"name", "bases", "dict", "frozen", "asarray", NULL};

/* Parse arguments: (name, bases, dict) */
if (!PyArg_ParseTupleAndKeywords(
args, kwargs, "UO!O!|$pp:StructMeta.__new__", kwlist,
&name, &PyTuple_Type, &bases, &PyDict_Type, &orig_dict,
&arg_immutable, &arg_asarray))
&arg_frozen, &arg_asarray))
return NULL;

if (PyDict_GetItemString(orig_dict, "__init__") != NULL) {
Expand Down Expand Up @@ -1522,7 +1522,7 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
);
goto error;
}
immutable = STRUCT_MERGE_OPTIONS(immutable, ((StructMetaObject *)base)->immutable);
frozen = STRUCT_MERGE_OPTIONS(frozen, ((StructMetaObject *)base)->frozen);
asarray = STRUCT_MERGE_OPTIONS(asarray, ((StructMetaObject *)base)->asarray);
base_fields = StructMeta_GET_FIELDS(base);
base_defaults = StructMeta_GET_DEFAULTS(base);
Expand Down Expand Up @@ -1552,7 +1552,7 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
Py_DECREF(offset);
}
}
immutable = STRUCT_MERGE_OPTIONS(immutable, arg_immutable);
frozen = STRUCT_MERGE_OPTIONS(frozen, arg_frozen);
asarray = STRUCT_MERGE_OPTIONS(asarray, arg_asarray);

new_dict = PyDict_Copy(orig_dict);
Expand Down Expand Up @@ -1680,7 +1680,7 @@ StructMeta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
cls->struct_fields = fields;
cls->struct_defaults = defaults;
cls->struct_offsets = offsets;
cls->immutable = immutable;
cls->frozen = frozen;
cls->asarray = asarray;
return (PyObject *) cls;
error:
Expand Down Expand Up @@ -1816,9 +1816,9 @@ StructMeta_dealloc(StructMetaObject *self)
}

static PyObject*
StructMeta_immutable(StructMetaObject *self, void *closure)
StructMeta_frozen(StructMetaObject *self, void *closure)
{
if (self->immutable == OPT_TRUE) { Py_RETURN_TRUE; }
if (self->frozen == OPT_TRUE) { Py_RETURN_TRUE; }
else { Py_RETURN_FALSE; }
}

Expand Down Expand Up @@ -1929,7 +1929,7 @@ static PyMemberDef StructMeta_members[] = {

static PyGetSetDef StructMeta_getset[] = {
{"__signature__", (getter) StructMeta_signature, NULL, NULL, NULL},
{"immutable", (getter) StructMeta_immutable, NULL, NULL, NULL},
{"frozen", (getter) StructMeta_frozen, NULL, NULL, NULL},
{"asarray", (getter) StructMeta_asarray, NULL, NULL, NULL},
{NULL},
};
Expand Down Expand Up @@ -2167,8 +2167,12 @@ Struct_vectorcall(PyTypeObject *cls, PyObject *const *args, size_t nargsf, PyObj

static int
Struct_setattro(PyObject *self, PyObject *key, PyObject *value) {
if (((StructMetaObject *)Py_TYPE(self))->immutable == OPT_TRUE) {
PyErr_Format(PyExc_TypeError, "immutable type: '%s'", Py_TYPE(self)->tp_name);
if (((StructMetaObject *)Py_TYPE(self))->frozen == OPT_TRUE) {
PyErr_Format(
PyExc_AttributeError,
"immutable type: '%s'",
Py_TYPE(self)->tp_name
);
return -1;
}
if (PyObject_GenericSetAttr(self, key, value) < 0)
Expand Down Expand Up @@ -2255,7 +2259,7 @@ Struct_hash(PyObject *self) {
Py_ssize_t i, nfields;
Py_uhash_t acc = MS_HASH_XXPRIME_5;

if (((StructMetaObject *)Py_TYPE(self))->immutable != OPT_TRUE) {
if (((StructMetaObject *)Py_TYPE(self))->frozen != OPT_TRUE) {
PyErr_Format(PyExc_TypeError, "unhashable type: '%s'", Py_TYPE(self)->tp_name);
return -1;
}
Expand Down Expand Up @@ -2423,10 +2427,10 @@ PyDoc_STRVAR(Struct__doc__,
"Additional class options can be enabled by passing keywords to the class\n"
"definition (see example below). The following options exist:\n"
"\n"
"- ``immutable``: whether instances of the class are immutable. If true,\n"
"- ``frozen``: whether instances of the class are pseudo-immutable. If true,\n"
" attribute assignment is disabled and a corresponding ``__hash__`` is defined.\n"
"- ``asarray``: whether instances of the class should be serialized as\n"
" MessagePack arrays, rather than dicts (the default).\n"
" arrays rather than dicts (the default).\n"
"\n"
"Structs automatically define ``__init__``, ``__eq__``, ``__repr__``, and\n"
"``__copy__`` methods. Additional methods can be defined on the class as\n"
Expand All @@ -2448,13 +2452,13 @@ PyDoc_STRVAR(Struct__doc__,
"Dog(name='snickers', breed='corgi', is_good_boy=True)\n"
"\n"
"Additional struct options can be set as part of the class definition. Here\n"
"we define a new `Struct` type for an immutable `Point` object.\n"
"we define a new `Struct` type for a frozen `Point` object.\n"
"\n"
">>> class Point(Struct, immutable=True):\n"
">>> class Point(Struct, frozen=True):\n"
"... x: float\n"
"... y: float\n"
"...\n"
">>> {Point(1.5, 2.0): 1} # immutable structs are hashable\n"
">>> {Point(1.5, 2.0): 1} # frozen structs are hashable\n"
"{Point(1.5, 2.0): 1}"
);

Expand Down Expand Up @@ -4711,7 +4715,7 @@ mpack_has_trailing_characters(DecoderState *self)
if (self->input_pos != self->input_end) {
PyErr_Format(
msgspec_get_global_state()->DecodingError,
"MsgPack data is malformed: trailing characters (byte %zd)",
"MessagePack data is malformed: trailing characters (byte %zd)",
(Py_ssize_t)(self->input_pos - self->input_start)
);
return true;
Expand Down Expand Up @@ -5881,8 +5885,8 @@ PyDoc_STRVAR(JSONDecoder__doc__,
" An optional callback for handling decoding custom types. Should have the\n"
" signature ``dec_hook(type: Type, obj: Any) -> Any``, where ``type`` is the\n"
" expected message type, and ``obj`` is the decoded representation composed\n"
" of only basic MessagePack types. This hook should transform ``obj`` into\n"
" type ``type``, or raise a ``TypeError`` if unsupported."
" of only basic JSON types. This hook should transform ``obj`` into type\n"
" ``type``, or raise a ``TypeError`` if unsupported."
);
static int
JSONDecoder_init(JSONDecoder *self, PyObject *args, PyObject *kwds)
Expand Down
5 changes: 0 additions & 5 deletions msgspec/_core.pyi

This file was deleted.

4 changes: 2 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@
"Source": "https://github.com/jcrist/msgspec/",
"Issue Tracker": "https://github.com/jcrist/msgspec/issues",
},
description="Fast and friendly msgpack (de)serialization, with type validation",
keywords="msgpack Messagepack serialization",
description="A fast and friendly JSON/MessagePack library, with optional schema validation",
keywords="JSON msgpack Messagepack serialization schema",
classifiers=[
"License :: OSI Approved :: BSD License",
"Development Status :: 4 - Beta",
Expand Down
14 changes: 12 additions & 2 deletions tests/mypy_examples.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,16 @@
# Structs #
##########################################################

def check___version__() -> None:
reveal_type(msgspec.__version__) # assert "str" in typ


def check_exceptions() -> None:
reveal_type(msgspec.DecodingError) # assert "Any" not in typ
reveal_type(msgspec.EncodingError) # assert "Any" not in typ
reveal_type(msgspec.MsgspecError) # assert "Any" not in typ


def check_struct() -> None:
class Test(msgspec.Struct):
x: int
Expand All @@ -30,8 +40,8 @@ class Test(msgspec.Struct, asarray=True):
reveal_type(t.y) # assert "str" in typ


def check_struct_immutable() -> None:
class Test(msgspec.Struct, immutable=True):
def check_struct_frozen() -> None:
class Test(msgspec.Struct, frozen=True):
x: int
y: str

Expand Down
24 changes: 12 additions & 12 deletions tests/test_struct.py
Original file line number Diff line number Diff line change
Expand Up @@ -657,7 +657,7 @@ def test_struct_handles_missing_attributes():
pickle.dumps(t)


@pytest.mark.parametrize("option", ["immutable", "asarray"])
@pytest.mark.parametrize("option", ["frozen", "asarray"])
def test_struct_option_precedence(option):
def get(cls):
return getattr(cls, option)
Expand Down Expand Up @@ -710,28 +710,28 @@ class Foo(Struct, invalid=True):
pass


class ImmutablePoint(Struct, immutable=True):
class FrozenPoint(Struct, frozen=True):
x: int
y: int


class TestImmutable:
def test_immutable_objects_no_setattr(self):
p = ImmutablePoint(1, 2)
with pytest.raises(TypeError, match="immutable type: 'ImmutablePoint'"):
class TestFrozen:
def test_frozen_objects_no_setattr(self):
p = FrozenPoint(1, 2)
with pytest.raises(AttributeError, match="immutable type: 'FrozenPoint'"):
p.x = 3

def test_immutable_objects_hashable(self):
p1 = ImmutablePoint(1, 2)
p2 = ImmutablePoint(1, 2)
p3 = ImmutablePoint(1, 3)
def test_frozen_objects_hashable(self):
p1 = FrozenPoint(1, 2)
p2 = FrozenPoint(1, 2)
p3 = FrozenPoint(1, 3)
assert hash(p1) == hash(p2)
assert hash(p1) != hash(p3)
assert p1 == p2
assert p1 != p3

def test_immutable_objects_hash_errors_if_field_unhashable(self):
p = ImmutablePoint(1, [2])
def test_frozen_objects_hash_errors_if_field_unhashable(self):
p = FrozenPoint(1, [2])
with pytest.raises(TypeError):
hash(p)

Expand Down