Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .cspell.dict/cpython.txt
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ prec
preinitialized
pybuilddir
pycore
pyinner
pydecimal
Pyfunc
pylifecycle
Expand Down
4 changes: 2 additions & 2 deletions crates/vm/src/builtins/complex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ impl PyPayload for PyComplex {
}

#[inline]
unsafe fn freelist_push(obj: *mut PyObject) -> bool {
unsafe fn freelist_push(obj: *mut PyObject, _hint: usize, _typ: &Py<PyType>) -> bool {
COMPLEX_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand All @@ -58,7 +58,7 @@ impl PyPayload for PyComplex {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
COMPLEX_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
4 changes: 2 additions & 2 deletions crates/vm/src/builtins/dict.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ impl PyPayload for PyDict {
}

#[inline]
unsafe fn freelist_push(obj: *mut PyObject) -> bool {
unsafe fn freelist_push(obj: *mut PyObject, _hint: usize, _typ: &Py<PyType>) -> bool {
DICT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand All @@ -93,7 +93,7 @@ impl PyPayload for PyDict {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
DICT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
4 changes: 2 additions & 2 deletions crates/vm/src/builtins/float.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ impl PyPayload for PyFloat {
}

#[inline]
unsafe fn freelist_push(obj: *mut PyObject) -> bool {
unsafe fn freelist_push(obj: *mut PyObject, _hint: usize, _typ: &Py<PyType>) -> bool {
FLOAT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand All @@ -65,7 +65,7 @@ impl PyPayload for PyFloat {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
FLOAT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
4 changes: 2 additions & 2 deletions crates/vm/src/builtins/int.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ impl PyPayload for PyInt {
}

#[inline]
unsafe fn freelist_push(obj: *mut PyObject) -> bool {
unsafe fn freelist_push(obj: *mut PyObject, _hint: usize, _typ: &Py<PyType>) -> bool {
INT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand All @@ -86,7 +86,7 @@ impl PyPayload for PyInt {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
INT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
4 changes: 2 additions & 2 deletions crates/vm/src/builtins/list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ impl PyPayload for PyList {
}

#[inline]
unsafe fn freelist_push(obj: *mut PyObject) -> bool {
unsafe fn freelist_push(obj: *mut PyObject, _hint: usize, _typ: &Py<PyType>) -> bool {
LIST_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand All @@ -105,7 +105,7 @@ impl PyPayload for PyList {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
LIST_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
4 changes: 2 additions & 2 deletions crates/vm/src/builtins/range.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ impl PyPayload for PyRange {
}

#[inline]
unsafe fn freelist_push(obj: *mut PyObject) -> bool {
unsafe fn freelist_push(obj: *mut PyObject, _hint: usize, _typ: &Py<PyType>) -> bool {
RANGE_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand All @@ -101,7 +101,7 @@ impl PyPayload for PyRange {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
RANGE_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
4 changes: 2 additions & 2 deletions crates/vm/src/builtins/slice.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ impl PyPayload for PySlice {
}

#[inline]
unsafe fn freelist_push(obj: *mut PyObject) -> bool {
unsafe fn freelist_push(obj: *mut PyObject, _hint: usize, _typ: &Py<PyType>) -> bool {
SLICE_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand All @@ -76,7 +76,7 @@ impl PyPayload for PySlice {
}

#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
SLICE_FREELIST
.try_with(|fl| {
let mut list = fl.take();
Expand Down
99 changes: 96 additions & 3 deletions crates/vm/src/builtins/tuple.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ use crate::{
vm::VirtualMachine,
};
use alloc::fmt;
use core::cell::Cell;
use core::ptr::NonNull;

#[pyclass(module = false, name = "tuple", traverse = "manual")]
pub struct PyTuple<R = PyObjectRef> {
Expand All @@ -53,14 +55,105 @@ unsafe impl Traverse for PyTuple {
}
}

// No freelist for PyTuple: structseq types (stat_result, struct_time, etc.)
// are static subtypes sharing the same Rust payload, making type-safe reuse
// impractical without a type-pointer comparison at push time.
// spell-checker:ignore MAXFREELIST MAXSAVESIZE
/// Per-size freelist storage for tuples, matching tuples[PyTuple_MAXSAVESIZE].
/// Each bucket caches tuples of a specific element count (index = len - 1).
struct TupleFreeList {
buckets: [Vec<NonNull<PyObject>>; Self::MAXSAVESIZE],
}

impl TupleFreeList {
/// Largest tuple size to cache on the freelist (sizes 1..=20).
const MAXSAVESIZE: usize = 20;
const fn new() -> Self {
Self {
buckets: [const { Vec::new() }; Self::MAXSAVESIZE],
}
}
}

impl Default for TupleFreeList {
fn default() -> Self {
Self::new()
}
}

impl Drop for TupleFreeList {
fn drop(&mut self) {
// Same safety pattern as FreeList<T>::drop — free raw allocation
// without running payload destructors to avoid TLS-after-destruction panics.
let layout = crate::object::pyinner_layout::<PyTuple>();
for bucket in &mut self.buckets {
for ptr in bucket.drain(..) {
unsafe {
alloc::alloc::dealloc(ptr.as_ptr() as *mut u8, layout);
}
}
}
}
}

thread_local! {
static TUPLE_FREELIST: Cell<TupleFreeList> = const { Cell::new(TupleFreeList::new()) };
}

impl PyPayload for PyTuple {
const MAX_FREELIST: usize = 2000;
const HAS_FREELIST: bool = true;

#[inline]
fn class(ctx: &Context) -> &'static Py<PyType> {
ctx.types.tuple_type
}

#[inline]
unsafe fn freelist_hint(obj: *mut PyObject) -> usize {
let py_tuple = unsafe { &*(obj as *const crate::Py<PyTuple>) };
py_tuple.elements.len()
}

#[inline]
unsafe fn freelist_push(obj: *mut PyObject, hint: usize, typ: &Py<PyType>) -> bool {
// Py_IS_TYPE(op, &PyTuple_Type): reject structseq and other subtypes
if !core::ptr::eq(typ, Self::class(crate::vm::Context::genesis())) {
return false;
}
let len = hint;
if len == 0 || len > TupleFreeList::MAXSAVESIZE {
return false;
}
TUPLE_FREELIST
.try_with(|fl| {
let mut list = fl.take();
let bucket = &mut list.buckets[len - 1];
let stored = if bucket.len() < Self::MAX_FREELIST {
bucket.push(unsafe { NonNull::new_unchecked(obj) });
true
} else {
false
};
fl.set(list);
stored
})
.unwrap_or(false)
}

#[inline]
unsafe fn freelist_pop(payload: &Self) -> Option<NonNull<PyObject>> {
let len = payload.elements.len();
if len == 0 || len > TupleFreeList::MAXSAVESIZE {
return None;
}
TUPLE_FREELIST
.try_with(|fl| {
let mut list = fl.take();
let result = list.buckets[len - 1].pop();
fl.set(list);
result
})
.ok()
.flatten()
}
}

pub trait IntoPyTuple {
Expand Down
54 changes: 39 additions & 15 deletions crates/vm/src/object/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -188,17 +188,28 @@ pub(super) unsafe fn default_dealloc<T: PyPayload>(obj: *mut PyObject) {
);
}

// Capture freelist bucket hint before tp_clear empties the payload.
// Size-based freelists (e.g. PyTuple) need the element count for bucket selection,
// but clear() replaces elements with an empty slice.
let freelist_hint = if T::HAS_FREELIST {
unsafe { T::freelist_hint(obj) }
} else {
0
};

// Extract child references before deallocation to break circular refs (tp_clear)
let mut edges = Vec::new();
if let Some(clear_fn) = vtable.clear {
unsafe { clear_fn(obj, &mut edges) };
}

// Try to store in freelist for reuse; otherwise deallocate.
// Only exact types (not heaptype subclasses) go into the freelist,
// because the pop site assumes the cached typ matches the base type.
let pushed = if T::HAS_FREELIST && obj_ref.class().heaptype_ext.is_none() {
unsafe { T::freelist_push(obj) }
// Only exact types (not heaptype subclasses) go into the freelist.
// The runtime type is passed to freelist_push for Py_IS_TYPE-style
// exact type checking (e.g. reject structseq subtypes of PyTuple).
let typ = obj_ref.class();
let pushed = if T::HAS_FREELIST && typ.heaptype_ext.is_none() {
unsafe { T::freelist_push(obj, freelist_hint, typ) }
} else {
false
};
Expand Down Expand Up @@ -1008,6 +1019,11 @@ impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
}
}

/// Returns the allocation layout for `PyInner<T>`, for use in freelist Drop impls.
pub(crate) const fn pyinner_layout<T: PyPayload>() -> core::alloc::Layout {
core::alloc::Layout::new::<PyInner<T>>()
}

/// Thread-local freelist storage for reusing object allocations.
///
/// Wraps a `Vec<*mut PyObject>`. On thread teardown, `Drop` frees raw
Expand Down Expand Up @@ -2076,24 +2092,32 @@ impl<T: PyPayload + crate::object::MaybeTraverse + core::fmt::Debug> PyRef<T> {

// Try to reuse from freelist (exact type only, no dict, no heaptype)
let cached = if !has_dict && !is_heaptype {
unsafe { T::freelist_pop() }
unsafe { T::freelist_pop(&payload) }
} else {
None
};

let ptr = if let Some(cached) = cached {
let inner = cached.as_ptr() as *mut PyInner<T>;
unsafe {
core::ptr::write(&mut (*inner).ref_count, RefCount::new());
(*inner).gc_bits.store(0, Ordering::Relaxed);
core::ptr::drop_in_place(&mut (*inner).payload);
core::ptr::write(&mut (*inner).payload, payload);
// typ, vtable, slots are preserved; dict is None, weak_list was
// cleared by drop_slow_inner before freelist push
// Push-side exact type check filters subtypes, but subtypes
// sharing the same Rust payload (e.g. structseq) may still
// call freelist_pop. Verify typ matches; if not, deallocate.
let cached_typ = unsafe { (*inner).typ.load_raw() };
if !core::ptr::eq(cached_typ, &*typ as *const Py<PyType>) {
unsafe { PyInner::dealloc(inner) };
let inner = PyInner::new(payload, typ, dict);
unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }
} else {
unsafe {
core::ptr::write(&mut (*inner).ref_count, RefCount::new());
(*inner).gc_bits.store(0, Ordering::Relaxed);
core::ptr::drop_in_place(&mut (*inner).payload);
core::ptr::write(&mut (*inner).payload, payload);
}
// Drop the caller's typ since the cached object already holds one
drop(typ);
unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }
}
// Drop the caller's typ since the cached object already holds one
drop(typ);
unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }
} else {
let inner = PyInner::new(payload, typ, dict);
unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }
Expand Down
17 changes: 15 additions & 2 deletions crates/vm/src/object/payload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,27 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static {
/// Maximum number of objects to keep in the freelist.
const MAX_FREELIST: usize = 0;

/// Capture a hint value from the payload before tp_clear runs.
/// Used by size-based freelists (e.g. PyTuple) to remember the element
/// count before clear empties the payload.
///
/// # Safety
/// `obj` must be a valid pointer to a `PyInner<Self>` with the payload still intact.
#[inline]
unsafe fn freelist_hint(_obj: *mut PyObject) -> usize {
0
}

/// Try to push a dead object onto this type's freelist for reuse.
/// Returns true if the object was stored (caller must NOT free the memory).
/// `hint` is the value returned by `freelist_hint` before tp_clear.
/// `typ` is the runtime type of the object, for exact-type filtering.
///
/// # Safety
/// `obj` must be a valid pointer to a `PyInner<Self>` with refcount 0,
/// after `drop_slow_inner` and `tp_clear` have already run.
#[inline]
unsafe fn freelist_push(_obj: *mut PyObject) -> bool {
unsafe fn freelist_push(_obj: *mut PyObject, _hint: usize, _typ: &Py<PyType>) -> bool {
false
}

Expand All @@ -75,7 +88,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static {
/// whose payload is still initialized from a previous allocation. The caller
/// will drop and overwrite `payload` before reuse.
#[inline]
unsafe fn freelist_pop() -> Option<NonNull<PyObject>> {
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
None
}

Expand Down
Loading