studio/cli
2
0
Fork 0
Code Issues Pull Requests Actions Packages Releases 4 Wiki Activity

chore: checkpoint before Python removal

Browse Source
This commit is contained in:
Sienna Meridian Satterwhite
2026-03-26 22:33:59 +00:00
parent 683cec9307
commit e568ddf82a
29972 changed files with 11269302 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

496
vendor/seize/src/collector.rs vendored Normal file
View File

@@ -0,0 +1,496 @@
use crate::raw;
use crate::tls::Thread;
use std::cell::UnsafeCell;
use std::marker::PhantomData;
use std::num::NonZeroU64;
use std::sync::atomic::{AtomicPtr, Ordering};
use std::{fmt, ptr};
/// Fast, efficient, and robust memory reclamation.
///
/// See the [crate documentation](crate) for details.
pub struct Collector {
raw: raw::Collector,
unique: *mut u8,
}
unsafe impl Send for Collector {}
unsafe impl Sync for Collector {}
impl Collector {
const DEFAULT_RETIRE_TICK: usize = 120;
const DEFAULT_EPOCH_TICK: NonZeroU64 = unsafe { NonZeroU64::new_unchecked(110) };
/// Creates a new collector.
pub fn new() -> Self {
let cpus = std::thread::available_parallelism()
.map(Into::into)
.unwrap_or(1);
Self {
raw: raw::Collector::new(cpus, Self::DEFAULT_EPOCH_TICK, Self::DEFAULT_RETIRE_TICK),
unique: Box::into_raw(Box::new(0)),
}
}
/// Sets the frequency of epoch advancement.
///
/// Seize uses epochs to protect against stalled threads.
/// The more frequently the epoch is advanced, the faster
/// stalled threads can be detected. However, it also means
/// that threads will have to do work to catch up to the
/// current epoch more often.
///
/// The default epoch frequency is `110`, meaning that
/// the epoch will advance after every 110 values are
/// linked to the collector. Benchmarking has shown that
/// this is a good tradeoff between throughput and memory
/// efficiency.
///
/// If `None` is passed epoch tracking, and protection
/// against stalled threads, will be disabled completely.
pub fn epoch_frequency(mut self, n: Option<NonZeroU64>) -> Self {
self.raw.epoch_frequency = n;
self
}
/// Sets the number of values that must be in a batch
/// before reclamation is attempted.
///
/// Retired values are added to thread-local *batches*
/// before starting the reclamation process. After
/// `batch_size` is hit, values are moved to separate
/// *retirement lists*, where reference counting kicks
/// in and batches are eventually reclaimed.
///
/// A larger batch size means that deallocation is done
/// less frequently, but reclamation also becomes more
/// expensive due to longer retirement lists needing
/// to be traversed and freed.
///
/// Note that batch sizes should generally be larger
/// than the number of threads accessing objects.
///
/// The default batch size is `120`. Tests have shown that
/// this makes a good tradeoff between throughput and memory
/// efficiency.
pub fn batch_size(mut self, n: usize) -> Self {
self.raw.batch_size = n;
self
}
/// Marks the current thread as active, returning a guard
/// that allows protecting loads of atomic pointers. The thread
/// will be marked as inactive when the guard is dropped.
///
/// See [the guide](crate#starting-operations) for an introduction
/// to using guards.
///
/// # Examples
///
/// ```rust
/// # use std::sync::atomic::{AtomicPtr, Ordering};
/// # let collector = seize::Collector::new();
/// use seize::{reclaim, Linked};
///
/// let ptr = AtomicPtr::new(collector.link_boxed(1_usize));
///
/// let guard = collector.enter();
/// let value = guard.protect(&ptr, Ordering::Acquire);
/// unsafe { assert_eq!(**value, 1) }
/// # unsafe { guard.defer_retire(value, reclaim::boxed::<Linked<usize>>) };
/// ```
///
/// Note that `enter` is reentrant, and it is legal to create
/// multiple guards on the same thread. The thread will stay
/// marked as active until the last guard is dropped:
///
/// ```rust
/// # use std::sync::atomic::{AtomicPtr, Ordering};
/// # let collector = seize::Collector::new();
/// use seize::{reclaim, Linked};
///
/// let ptr = AtomicPtr::new(collector.link_boxed(1_usize));
///
/// let guard1 = collector.enter();
/// let guard2 = collector.enter();
///
/// let value = guard2.protect(&ptr, Ordering::Acquire);
/// drop(guard1);
/// // the first guard is dropped, but `value`
/// // is still safe to access as a guard still
/// // exists
/// unsafe { assert_eq!(**value, 1) }
/// # unsafe { guard2.defer_retire(value, reclaim::boxed::<Linked<usize>>) };
/// drop(guard2) // _now_, the thread is marked as inactive
/// ```
pub fn enter(&self) -> Guard<'_> {
let thread = Thread::current();
unsafe { self.raw.enter(thread) };
Guard {
thread,
collector: Some(self),
_unsend: PhantomData,
}
}
/// Link a value to the collector.
///
/// See [the guide](crate#allocating-objects) for details.
pub fn link(&self) -> Link {
Link {
node: UnsafeCell::new(self.raw.node()),
}
}
/// Creates a new `Linked` object with the given value.
///
/// This is equivalent to:
///
/// ```ignore
/// Linked {
/// value,
/// link: collector.link()
/// }
/// ```
pub fn link_value<T>(&self, value: T) -> Linked<T> {
Linked {
link: self.link(),
value,
}
}
/// Links a value to the collector and allocates it with `Box`.
///
/// This is equivalent to:
///
/// ```ignore
/// Box::into_raw(Box::new(Linked {
/// value,
/// link: collector.link()
/// }))
/// ```
pub fn link_boxed<T>(&self, value: T) -> *mut Linked<T> {
Box::into_raw(Box::new(Linked {
link: self.link(),
value,
}))
}
/// Retires a value, running `reclaim` when no threads hold a reference to it.
///
/// Note that this method is disconnected from any guards on the current thread,
/// so the pointer may be reclaimed immediately. See [`Guard::defer_retire`] if
/// the pointer may still be accessed by the current thread.
///
/// See [the guide](crate#retiring-objects) for details.
///
/// # Safety
///
/// The retired object must no longer be accessible to any thread that enters
/// after it is removed. It also cannot be accessed by the current thread
/// after `retire` is called.
///
/// Additionally, he reclaimer passed to `retire` must correctly deallocate values of type `T`.
#[allow(clippy::missing_safety_doc)] // in guide
pub unsafe fn retire<T: AsLink>(&self, ptr: *mut T, reclaim: unsafe fn(*mut Link)) {
debug_assert!(!ptr.is_null(), "attempted to retire null pointer");
// note that `add` doesn't actually reclaim the pointer immediately if the
// current thread is active, it instead adds it to it's reclamation list,
// but we don't guarantee that publicly.
unsafe { self.raw.add(ptr, reclaim, Thread::current()) }
}
/// Returns true if both references point to the same collector.
pub fn ptr_eq(this: &Collector, other: &Collector) -> bool {
ptr::eq(this.unique, other.unique)
}
}
impl Drop for Collector {
fn drop(&mut self) {
unsafe {
let _ = Box::from_raw(self.unique);
}
}
}
impl Clone for Collector {
fn clone(&self) -> Self {
Collector::new()
.batch_size(self.raw.batch_size)
.epoch_frequency(self.raw.epoch_frequency)
}
}
impl Default for Collector {
fn default() -> Self {
Self::new()
}
}
impl fmt::Debug for Collector {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut strukt = f.debug_struct("Collector");
if self.raw.epoch_frequency.is_some() {
strukt.field("epoch", &self.raw.epoch.load(Ordering::Acquire));
}
strukt
.field("batch_size", &self.raw.batch_size)
.field("epoch_frequency", &self.raw.epoch_frequency)
.finish()
}
}
/// A guard that keeps the current thread marked as active,
/// enabling protected loads of atomic pointers.
///
/// See [`Collector::enter`] for details.
pub struct Guard<'a> {
collector: Option<&'a Collector>,
thread: Thread,
// must not be Send or Sync as we are tied to the current threads state in
// the collector
_unsend: PhantomData<*mut ()>,
}
impl Guard<'_> {
/// Returns a dummy guard.
///
/// Calling [`protect`](Guard::protect) on an unprotected guard will
/// load the pointer directly, and [`retire`](Guard::defer_retire) will
/// reclaim objects immediately.
///
/// Unprotected guards are useful when calling guarded functions
/// on a data structure that has just been created or is about
/// to be destroyed, because you know that no other thread holds
/// a reference to it.
///
/// # Safety
///
/// You must ensure that code used with this guard is sound with
/// the unprotected behavior described above.
pub const unsafe fn unprotected() -> Guard<'static> {
Guard {
thread: Thread::EMPTY,
collector: None,
_unsend: PhantomData,
}
}
/// Protects the load of an atomic pointer.
///
/// See [the guide](crate#protecting-pointers) for details.
#[inline]
pub fn protect<T: AsLink>(&self, ptr: &AtomicPtr<T>, ordering: Ordering) -> *mut T {
match self.collector {
Some(collector) => unsafe { collector.raw.protect(ptr, ordering, self.thread) },
// unprotected guard
None => ptr.load(ordering),
}
}
/// Retires a value, running `reclaim` when no threads hold a reference to it.
///
/// This method delays reclamation until the guard is dropped as opposed to
/// [`Collector::retire`], which may reclaim objects immediately.
///
/// See [the guide](crate#retiring-objects) for details.
#[allow(clippy::missing_safety_doc)] // in guide
pub unsafe fn defer_retire<T: AsLink>(&self, ptr: *mut T, reclaim: unsafe fn(*mut Link)) {
debug_assert!(!ptr.is_null(), "attempted to retire null pointer");
match self.collector {
Some(collector) => unsafe { collector.raw.add(ptr, reclaim, self.thread) },
// unprotected guard
None => unsafe { (reclaim)(ptr.cast::<Link>()) },
}
}
/// Get a reference to the collector this guard we created from.
///
/// This method is useful when you need to ensure that all guards
/// used with a data structure come from the same collector.
///
/// If this is an [`unprotected`](Guard::unprotected) guard
/// this method will return `None`.
pub fn collector(&self) -> Option<&Collector> {
self.collector
}
/// Refreshes the guard.
///
/// Refreshing a guard is similar to dropping and immediately
/// creating a new guard. The curent thread remains active, but any
/// pointers that were previously protected may be reclaimed.
///
/// # Safety
///
/// This method is not marked as `unsafe`, but will affect
/// the validity of pointers returned by [`protect`](Guard::protect),
/// similar to dropping a guard. It is intended to be used safely
/// by users of concurrent data structures, as references will
/// be tied to the guard and this method takes `&mut self`.
///
/// If this is an [`unprotected`](Guard::unprotected) guard
/// this method will be a no-op.
pub fn refresh(&mut self) {
match self.collector {
None => {}
Some(collector) => unsafe { collector.raw.refresh(self.thread) },
}
}
/// Flush any retired values in the local batch.
///
/// This method flushes any values from the current thread's local
/// batch, starting the reclamation process. Note that no memory
/// can be reclaimed while this guard is active, but calling `flush`
/// may allow memory to be reclaimed more quickly after the guard is
/// dropped.
///
/// See [`Collector::batch_size`] for details about batching.
pub fn flush(&self) {
if let Some(collector) = self.collector {
unsafe { collector.raw.try_retire_batch(self.thread) }
}
}
/// Returns a numeric identifier for the current thread.
///
/// Guards rely on thread-local state, including thread IDs. If you already
/// have a guard you can use this method to get a cheap identifier for the
/// current thread, avoiding TLS overhead. Note that thread IDs may be reused,
/// so the value returned is only unique for the lifetime of this thread.
pub fn thread_id(&self) -> usize {
self.thread.id
}
}
impl Drop for Guard<'_> {
fn drop(&mut self) {
if let Some(collector) = self.collector {
unsafe { collector.raw.leave(self.thread) }
}
}
}
impl fmt::Debug for Guard<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_tuple("Guard").finish()
}
}
/// A link to the collector.
///
/// See [the guide](crate#custom-reclaimers) for details.
#[repr(transparent)]
pub struct Link {
#[allow(dead_code)]
pub(crate) node: UnsafeCell<raw::Node>,
}
impl Link {
/// Cast this `link` to it's underlying type.
///
/// Note that while this function is safe, using the returned
/// pointer is only sound if the link is in fact a type-erased `T`.
/// This means that when casting a link in a reclaimer, the value
/// that was retired must be of type `T`.
pub fn cast<T: AsLink>(link: *mut Link) -> *mut T {
link.cast()
}
}
/// A type that can be pointer-cast to and from a [`Link`].
///
/// Most reclamation use cases can avoid this trait and work instead
/// with the [`Linked`] wrapper type. However, if you want more control
/// over the layout of your type (i.e. are working with a DST),
/// you may need to implement this trait directly.
///
/// # Safety
///
/// Types implementing this trait must be marked `#[repr(C)]`
/// and have a [`Link`] as their **first** field.
///
/// # Examples
///
/// ```rust
/// use seize::{AsLink, Collector, Link};
///
/// #[repr(C)]
/// struct Bytes {
/// // safety invariant: Link must be the first field
/// link: Link,
/// values: [*mut u8; 0],
/// }
///
/// // Safety: Bytes is repr(C) and has Link as it's first field
/// unsafe impl AsLink for Bytes {}
///
/// // Deallocate an `Bytes`.
/// unsafe fn dealloc(ptr: *mut Bytes, collector: &Collector) {
/// collector.retire(ptr, |link| {
/// // safety `ptr` is of type *mut Bytes
/// let link: *mut Bytes = Link::cast(link);
/// // ..
/// });
/// }
/// ```
pub unsafe trait AsLink {}
/// A value [linked](Collector::link) to a collector.
///
/// This type implements `Deref` and `DerefMut` to the
/// inner value, so you can access methods on fields
/// on it as normal. An extra `*` may be needed when
/// `T` needs to be accessed directly.
///
/// See [the guide](crate#allocating-objects) for details.
#[repr(C)]
pub struct Linked<T> {
pub link: Link, // Safety Invariant: this field must come first
pub value: T,
}
unsafe impl<T> AsLink for Linked<T> {}
impl<T: PartialEq> PartialEq for Linked<T> {
fn eq(&self, other: &Self) -> bool {
self.value == other.value
}
}
impl<T: Eq> Eq for Linked<T> {}
impl<T: fmt::Debug> fmt::Debug for Linked<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self.value)
}
}
impl<T: fmt::Display> fmt::Display for Linked<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.value)
}
}
impl<T> std::ops::Deref for Linked<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.value
}
}
impl<T> std::ops::DerefMut for Linked<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.value
}
}

11
vendor/seize/src/lib.rs vendored Normal file
View File

@@ -0,0 +1,11 @@
#![deny(unsafe_op_in_unsafe_fn)]
#![doc = include_str!("../README.md")]
mod collector;
mod raw;
mod tls;
mod utils;
pub mod reclaim;
pub use collector::{AsLink, Collector, Guard, Link, Linked};

571
vendor/seize/src/raw.rs vendored Normal file
View File

@@ -0,0 +1,571 @@
use crate::tls::{Thread, ThreadLocal};
use crate::utils::CachePadded;
use crate::{AsLink, Link};
use std::cell::{Cell, UnsafeCell};
use std::mem::ManuallyDrop;
use std::num::NonZeroU64;
use std::ptr::{self, NonNull};
use std::sync::atomic::{self, AtomicPtr, AtomicU64, AtomicUsize, Ordering};
// Fast, lock-free, robust concurrent memory reclamation.
//
// The core algorithm is described [in this paper](https://arxiv.org/pdf/2108.02763.pdf).
pub struct Collector {
// Per-thread reservations lists
reservations: ThreadLocal<CachePadded<Reservation>>,
// Per-thread batches of retired nodes
batches: ThreadLocal<CachePadded<UnsafeCell<LocalBatch>>>,
// The number of nodes allocated per-thread
node_count: ThreadLocal<UnsafeCell<u64>>,
// The global epoch value
pub(crate) epoch: AtomicU64,
// The number of node allocations before advancing the global epoch
pub(crate) epoch_frequency: Option<NonZeroU64>,
// The number of nodes in a batch before we free
pub(crate) batch_size: usize,
}
impl Collector {
// Create a collector with the provided configuration.
pub fn new(threads: usize, epoch_frequency: NonZeroU64, batch_size: usize) -> Self {
Self {
epoch: AtomicU64::new(1),
reservations: ThreadLocal::with_capacity(threads),
batches: ThreadLocal::with_capacity(threads),
node_count: ThreadLocal::with_capacity(threads),
epoch_frequency: Some(epoch_frequency),
batch_size,
}
}
// Create a new node.
pub fn node(&self) -> Node {
// safety: node counts are only accessed by the current thread
let count = unsafe { &mut *self.node_count.load(Thread::current()).get() };
*count += 1;
// record the current epoch value
//
// note that it's fine if we see older epoch values here, which just means more
// threads will be counted as active than might actually be
let birth_epoch = match self.epoch_frequency {
// advance the global epoch
Some(ref freq) if *count % freq.get() == 0 => {
self.epoch.fetch_add(1, Ordering::Relaxed) + 1
}
Some(_) => self.epoch.load(Ordering::Relaxed),
// we aren't tracking epochs
None => 0,
};
Node { birth_epoch }
}
// Mark the current thread as active.
//
// # Safety
//
// `thread` must be the current thread.
pub unsafe fn enter(&self, thread: Thread) {
let reservation = self.reservations.load(thread);
// calls to `enter` may be reentrant, so we need to keep track of the number
// of active guards for the current thread
let guards = reservation.guards.get();
reservation.guards.set(guards + 1);
// avoid clearing already active reservation lists
if guards == 0 {
// mark the thread as active
//
// seqcst: establish a total order between this store and the fence in `retire`
// - if our store comes first, the thread retiring will see that we are active
// - if the fence comes first, we will see the new values of any objects being
// retired by that thread (all pointer loads are also seqcst and thus participate
// in the total order)
reservation.head.store(ptr::null_mut(), Ordering::SeqCst);
}
}
// Load an atomic pointer.
//
// # Safety
//
// `thread` must be the current thread.
#[inline]
pub unsafe fn protect<T>(
&self,
ptr: &AtomicPtr<T>,
_ordering: Ordering,
thread: Thread,
) -> *mut T {
if self.epoch_frequency.is_none() {
// epoch tracking is disabled, but pointer loads still need to be seqcst to participate
// in the total order. see `enter` for details
return ptr.load(Ordering::SeqCst);
}
let reservation = self.reservations.load(thread);
// load the last epoch we recorded
//
// relaxed: the reservation epoch is only modified by the current thread
let mut prev_epoch = reservation.epoch.load(Ordering::Relaxed);
loop {
// seqcst:
// - ensure that this load participates in the total order. see the store
// to reservation.head and reservation.epoch for details
// - acquire the birth epoch of the pointer. we need to record at least
// that epoch below to let other threads know we have access to this pointer
// (TOOD: this requires objects to be stored with release ordering, which is
// not documented)
let ptr = ptr.load(Ordering::SeqCst);
// relaxed: we acquired at least the pointer's birth epoch above
let current_epoch = self.epoch.load(Ordering::Relaxed);
// we were already marked as active in the birth epoch, so we are safe
if prev_epoch == current_epoch {
return ptr;
}
// our epoch is out of date, record the new one and try again
//
// seqcst: establish a total order between this store and the fence in `retire`
// - if our store comes first, the thread retiring will see that we are active in
// the current epoch
// - if the fence comes first, we will see the new values of any objects being
// retired by that thread (all pointer loads are also seqcst and thus participate
// in the total order)
reservation.epoch.store(current_epoch, Ordering::SeqCst);
prev_epoch = current_epoch;
}
}
// Mark the current thread as inactive.
//
// # Safety
//
// `thread` must be the current thread.
pub unsafe fn leave(&self, thread: Thread) {
let reservation = self.reservations.load(thread);
// decrement the active guard count
let guards = reservation.guards.get();
reservation.guards.set(guards - 1);
// we can only decrement reference counts after all guards for the current thread
// are dropped
if guards == 1 {
// release: exit the critical section
// acquire: acquire any new entries
let head = reservation.head.swap(Entry::INACTIVE, Ordering::AcqRel);
if head != Entry::INACTIVE {
// decrement the reference counts of any entries that were added
unsafe { Collector::traverse(head) }
}
}
}
// Decrement any reference counts, keeping the thread marked as active.
//
// # Safety
//
// `thread` must be the current thread.
pub unsafe fn refresh(&self, thread: Thread) {
let reservation = self.reservations.load(thread);
let guards = reservation.guards.get();
// we can only decrement reference counts after all guards for the current
// thread are dropped
if guards == 1 {
// release: exit the critical section
// acquire: acquire any new entries and the values of any objects
// that were retired
let head = reservation.head.swap(ptr::null_mut(), Ordering::AcqRel);
if head != Entry::INACTIVE {
// decrement the reference counts of any entries that were added
unsafe { Collector::traverse(head) }
}
}
}
// Add a node to the retirement batch, retiring the batch if `batch_size` is reached.
//
// # Safety
//
// `ptr` is a valid pointer, and `thread` must be the current thread.
pub unsafe fn add<T>(&self, ptr: *mut T, reclaim: unsafe fn(*mut Link), thread: Thread)
where
T: AsLink,
{
// safety: local batches are only accessed by the current thread until retirement
let local_batch = unsafe {
&mut *self
.batches
.load_or(|| LocalBatch::new(self.batch_size), thread)
.get()
};
// safety: local batch pointers are always valid until reclamation
let batch = unsafe { local_batch.0.as_mut() };
// `ptr` is guaranteed to be a valid pointer that can be cast to a node (`T: AsLink`)
//
// any other thread with a reference to the pointer only has a shared
// reference to the UnsafeCell<Node>, which is allowed to alias. the caller
// guarantees that the same pointer is not retired twice, so we can safely write
// to the node through this pointer.
let node = UnsafeCell::raw_get(ptr.cast::<UnsafeCell<Node>>());
// if a thread is active in the minimum birth era, it has access to at least one
// of the nodes in the batch and must be tracked.
//
// if epoch tracking is disabled this will always be false (0 > 0).
let birth_epoch = unsafe { (*node).birth_epoch };
if batch.min_epoch > birth_epoch {
batch.min_epoch = birth_epoch;
}
// create an entry for this node
batch.entries.push(Entry {
node,
reclaim,
batch: local_batch.0.as_ptr(),
});
// attempt to retire the batch if we have enough entries
if batch.entries.len() % self.batch_size == 0 {
unsafe { self.try_retire(local_batch, thread) }
}
}
// Attempt to retire nodes in the current thread's batch.
//
// # Safety
//
// `thread` must be the current thread.
pub unsafe fn try_retire_batch(&self, thread: Thread) {
let local_batch = self
.batches
.load_or(|| LocalBatch::new(self.batch_size), thread);
// safety: batches are only accessed by the current thread
unsafe { self.try_retire(&mut *local_batch.get(), thread) }
}
// Attempt to retire nodes in this batch.
//
// Note that if a guard on the current thread is active, the batch will also be added to it's
// reservation list for deferred reclamation.
//
// # Safety
//
// `thread` must be the current thread.
pub unsafe fn try_retire(&self, local_batch: &mut LocalBatch, thread: Thread) {
// establish a total order between the retirement of nodes in this batch and stores
// marking a thread as active (or active in an epoch):
// - if the store comes first, we will see that the thread is active
// - if this fence comes first, the thread will see the new values of any objects
// in this batch.
//
// this fence also establishes synchronizes with the fence run when a thread is created:
// - if our fence comes first, they will see the new values of any objects in this batch
// - if their fence comes first, we will see the new thread
atomic::fence(Ordering::SeqCst);
// safety: local batch pointers are always valid until reclamation.
// if the batch ends up being retired then this pointer is stable
let batch_entries = unsafe { local_batch.0.as_mut().entries.as_mut_ptr() };
let batch = unsafe { local_batch.0.as_ref() };
// if there are not enough entries in this batch for active threads, we have to try again later
//
// relaxed: the fence above already ensures that we see any threads that might
// have access to any objects in this batch. any other threads that were created
// after it will see their new values.
if batch.entries.len() <= self.reservations.threads.load(Ordering::Relaxed) {
return;
}
let current_reservation = self.reservations.load(thread);
let mut marked = 0;
// record all active threads
//
// we need to do this in a separate step before actually retiring to
// make sure we have enough entries, as the number of threads can grow
for reservation in self.reservations.iter() {
// if we don't have enough entries to insert into the reservation lists
// of all active threads, try again later
let Some(entry) = batch.entries.get(marked) else {
return;
};
// if this thread is inactive, we can skip it
//
// relaxed: see the acquire fence below
if reservation.head.load(Ordering::Relaxed) == Entry::INACTIVE {
continue;
}
// if this thread's epoch is behind the earliest birth epoch in this batch
// we can skip it, as there is no way it could have accessed any of the objects
// in this batch. we make sure never to skip the current thread even if it's epoch
// is behind because it may still have access to the pointer (because it's the
// thread that allocated it). the current thread is only skipped if there is no
// active guard.
//
// relaxed: if the epoch is behind there is nothing to synchronize with, and
// we already ensured we will see it's relevant epoch with the seqcst fence
// above
//
// if epoch tracking is disabled this is always false (0 < 0)
if !ptr::eq(reservation, current_reservation)
&& reservation.epoch.load(Ordering::Relaxed) < batch.min_epoch
{
continue;
}
// temporarily store this thread's list in a node in our batch
//
// safety: all nodes in a batch are valid, and this batch has not been
// shared yet to other threads
unsafe { (*entry.node).head = &reservation.head }
marked += 1;
}
// for any inactive threads we skipped above, synchronize with `leave` to ensure
// any accesses happen-before we retire. we ensured with the seqcst fence above
// that the next time the thread becomes active it will see the new values of any
// objects in this batch.
atomic::fence(Ordering::Acquire);
// add the batch to all active thread's reservation lists
let mut active = 0;
for i in 0..marked {
let curr = &batch.entries[i];
let curr_ptr = unsafe { batch_entries.add(i) };
// safety: all nodes in the batch are valid, and we just initialized `head`
// for all `marked` nodes in the loop above
let head = unsafe { &*(*curr.node).head };
// acquire:
// - if the thread became inactive, synchronize with `leave` to ensure any accesses
// happen-before we retire
// - if the thread is active, acquire any entries added by a concurrent call
// to `retire`
let mut prev = head.load(Ordering::Acquire);
loop {
// the thread became inactive, skip it
//
// as long as the thread became inactive at some point after we verified it was
// active, it can no longer access any objects in this batch. the next time it
// becomes active it will load the new object values due to the seqcst fence above
if prev == Entry::INACTIVE {
break;
}
// link this node to the reservation list
unsafe { *(*curr.node).next = AtomicPtr::new(prev) }
// release: release the entries in this batch
match head.compare_exchange_weak(
prev,
curr_ptr,
Ordering::Release,
Ordering::Relaxed,
) {
Ok(_) => {
active += 1;
break;
}
// lost the race to another thread, retry
Err(found) => {
// acquire the new entries
atomic::fence(Ordering::Acquire);
prev = found;
continue;
}
}
}
}
// release: if we don't free the list, release the batch to the thread that will
if batch
.active
.fetch_add(active, Ordering::Release)
.wrapping_add(active)
== 0
{
// ensure any access of the data in the list happens-before we free the list
atomic::fence(Ordering::Acquire);
// safety: the reference count is 0, meaning that either no threads were active,
// or they have all already decremented the count
unsafe { Collector::free_batch(local_batch.0.as_ptr()) }
}
// reset the batch
*local_batch = LocalBatch::new(self.batch_size).value.into_inner();
}
// Traverse the reservation list, decrementing the reference count of each batch.
//
// # Safety
//
// `list` must be a valid reservation list
unsafe fn traverse(mut list: *mut Entry) {
loop {
let curr = list;
if curr.is_null() {
break;
}
// safety: `curr` is a valid link in the list
//
// relaxed: any entries were acquired when we loaded `head`
list = unsafe { (*(*curr).node).next.load(Ordering::Relaxed) };
let batch = unsafe { (*curr).batch };
// safety: batch pointers are valid for reads until they are freed
unsafe {
// release: if we don't free the list, release any access of the batch to the thread
// that will
if (*batch).active.fetch_sub(1, Ordering::Release) == 1 {
// ensure any access of the data in the list happens-before we free the list
atomic::fence(Ordering::Acquire);
// safety: we have the last reference to the batch
Collector::free_batch(batch)
}
}
}
}
// Free a reservation list.
//
// # Safety
//
// The batch reference count must be zero.
unsafe fn free_batch(batch: *mut Batch) {
// safety: we are the last reference to the batch
for entry in unsafe { (*batch).entries.iter_mut() } {
unsafe { (entry.reclaim)(entry.node.cast::<Link>()) };
}
unsafe { LocalBatch::free(batch) };
}
}
impl Drop for Collector {
fn drop(&mut self) {
for batch in self.batches.iter() {
// safety: we have &mut self
let batch = unsafe { &mut *batch.get() };
// safety: we have &mut self
unsafe { Collector::free_batch(batch.0.as_ptr()) }
}
}
}
// A node attached to every allocated object.
//
// Nodes keep track of their birth epoch, as well as thread-local
// reservation lists.
#[repr(C)]
pub union Node {
// Before retiring: the epoch this node was created in
birth_epoch: u64,
// While retiring: temporary location for an active reservation list.
head: *const AtomicPtr<Entry>,
// After retiring: next node in the thread's reservation list
next: ManuallyDrop<AtomicPtr<Entry>>,
}
// A per-thread reservation list.
//
// Reservation lists are lists of retired entries, where
// each entry represents a batch.
#[repr(C)]
struct Reservation {
// The head of the list
head: AtomicPtr<Entry>,
// The epoch this thread last accessed a pointer in
epoch: AtomicU64,
// the number of active guards for this thread
guards: Cell<u64>,
}
impl Default for Reservation {
fn default() -> Self {
Reservation {
head: AtomicPtr::new(Entry::INACTIVE),
epoch: AtomicU64::new(0),
guards: Cell::new(0),
}
}
}
// A batch of nodes waiting to be retired
struct Batch {
// Nodes in this batch.
//
// TODO: this allocation can be flattened
entries: Vec<Entry>,
// The minimum epoch of all nodes in this batch.
min_epoch: u64,
// The reference count for active threads.
active: AtomicUsize,
}
// A retired node.
struct Entry {
node: *mut Node,
reclaim: unsafe fn(*mut Link),
// the batch this node is a part of.
batch: *mut Batch,
}
impl Entry {
// Represents an inactive thread.
//
// While null indicates an empty list, INACTIVE indicates the thread has no active
// guards and is not accessing any objects.
pub const INACTIVE: *mut Entry = -1_isize as usize as _;
}
pub struct LocalBatch(NonNull<Batch>);
impl LocalBatch {
// Create a new batch with an initial capacity.
fn new(capacity: usize) -> CachePadded<UnsafeCell<LocalBatch>> {
let ptr = unsafe {
NonNull::new_unchecked(Box::into_raw(Box::new(Batch {
entries: Vec::with_capacity(capacity),
min_epoch: 0,
active: AtomicUsize::new(0),
})))
};
CachePadded::new(UnsafeCell::new(LocalBatch(ptr)))
}
// Free the batch.
unsafe fn free(ptr: *mut Batch) {
unsafe { drop(Box::from_raw(ptr)) }
}
}
// Local batches are only accessed by the current thread.
unsafe impl Send for LocalBatch {}
unsafe impl Sync for LocalBatch {}

41
vendor/seize/src/reclaim.rs vendored Normal file
View File

@@ -0,0 +1,41 @@
//! Common memory reclaimers.
//!
//! Functions in this module can be passed to [`retire`](crate::Collector::retire)
//! to free allocated memory or run drop glue. See [the guide](crate#custom-reclaimers)
//! for details about memory reclamation, and writing custom reclaimers.
use std::ptr;
use crate::{AsLink, Link};
/// Reclaims memory allocated with [`Box`].
///
/// This function calls [`Box::from_raw`] on the linked pointer.
///
/// # Safety
///
/// Ensure that the correct type annotations are used when
/// passing this function to [`retire`](crate::Collector::retire).
/// The value retired must have been of type `T` to be retired through
/// `boxed::<T>`.
pub unsafe fn boxed<T: AsLink>(link: *mut Link) {
unsafe {
let _: Box<T> = Box::from_raw(Link::cast(link));
}
}
/// Reclaims memory by dropping the value in place.
///
/// This function calls [`ptr::drop_in_place`] on the linked pointer.
///
/// # Safety
///
/// Ensure that the correct type annotations are used when
/// passing this function to [`retire`](crate::Collector::retire).
/// The value retired must have been of type `T` to be retired through
/// `in_place::<T>`.
pub unsafe fn in_place<T: AsLink>(link: *mut Link) {
unsafe {
ptr::drop_in_place::<T>(Link::cast(link));
}
}

359
vendor/seize/src/tls/mod.rs vendored Normal file
View File

@@ -0,0 +1,359 @@
// Copyright 2017 Amanieu d'Antras
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.
mod thread_id;
use std::cell::UnsafeCell;
use std::mem::{self, MaybeUninit};
use std::ptr;
use std::sync::atomic::{self, AtomicBool, AtomicPtr, AtomicUsize, Ordering};
pub use thread_id::Thread;
const BUCKETS: usize = (usize::BITS + 1) as usize;
pub struct ThreadLocal<T: Send> {
buckets: [AtomicPtr<Entry<T>>; BUCKETS],
pub threads: AtomicUsize,
}
struct Entry<T> {
present: AtomicBool,
value: UnsafeCell<MaybeUninit<T>>,
}
impl<T> Drop for Entry<T> {
fn drop(&mut self) {
unsafe {
if *self.present.get_mut() {
ptr::drop_in_place((*self.value.get()).as_mut_ptr());
}
}
}
}
unsafe impl<T: Send> Sync for ThreadLocal<T> {}
impl<T> ThreadLocal<T>
where
T: Send,
{
pub fn with_capacity(capacity: usize) -> ThreadLocal<T> {
let allocated_buckets = capacity
.checked_sub(1)
.map(|c| (usize::BITS as usize) - (c.leading_zeros() as usize) + 1)
.unwrap_or(0);
let mut buckets = [ptr::null_mut(); BUCKETS];
let mut bucket_size = 1;
for (i, bucket) in buckets[..allocated_buckets].iter_mut().enumerate() {
*bucket = allocate_bucket::<T>(bucket_size);
if i != 0 {
bucket_size <<= 1;
}
}
ThreadLocal {
// safety: `AtomicPtr` has the same representation as a pointer
buckets: unsafe { mem::transmute(buckets) },
threads: AtomicUsize::new(0),
}
}
pub fn load(&self, thread: Thread) -> &T
where
T: Default,
{
self.load_or(T::default, thread)
}
pub fn load_or(&self, create: impl Fn() -> T, thread: Thread) -> &T {
let bucket = unsafe { self.buckets.get_unchecked(thread.bucket) };
let mut bucket_ptr = bucket.load(Ordering::Acquire);
if bucket_ptr.is_null() {
let new_bucket = allocate_bucket(thread.bucket_size);
match bucket.compare_exchange(
ptr::null_mut(),
new_bucket,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => bucket_ptr = new_bucket,
// if the bucket value changed (from null), that means
// another thread stored a new bucket before we could,
// and we can free our bucket and use that one instead
Err(other) => unsafe {
let _ = Box::from_raw(ptr::slice_from_raw_parts_mut(
new_bucket,
thread.bucket_size,
));
bucket_ptr = other;
},
}
}
unsafe {
let entry = &*bucket_ptr.add(thread.index);
// relaxed: only this thread can set the value
if entry.present.load(Ordering::Relaxed) {
(*entry.value.get()).assume_init_ref()
} else {
// insert the new element into the bucket
entry.value.get().write(MaybeUninit::new(create()));
// release: necessary for iterator
entry.present.store(true, Ordering::Release);
self.threads.fetch_add(1, Ordering::Relaxed);
// seqcst: synchronize with the fence in `retire`:
// - if this fence comes first, the thread retiring will see the new thread count
// and our entry
// - if their fence comes first, we will see the new values of any pointers being
// retired by that thread
atomic::fence(Ordering::SeqCst);
(*entry.value.get()).assume_init_ref()
}
}
}
#[cfg(test)]
fn try_load(&self) -> Option<&T> {
let thread = Thread::current();
let bucket_ptr =
unsafe { self.buckets.get_unchecked(thread.bucket) }.load(Ordering::Acquire);
if bucket_ptr.is_null() {
return None;
}
unsafe {
let entry = &*bucket_ptr.add(thread.index);
// read without atomic operations as only this thread can set the value.
if entry.present.load(Ordering::Relaxed) {
Some((*entry.value.get()).assume_init_ref())
} else {
None
}
}
}
pub fn iter(&self) -> Iter<'_, T> {
Iter {
bucket: 0,
bucket_size: 1,
index: 0,
thread_local: self,
}
}
}
impl<T> Drop for ThreadLocal<T>
where
T: Send,
{
fn drop(&mut self) {
let mut bucket_size = 1;
for (i, bucket) in self.buckets.iter_mut().enumerate() {
let bucket_ptr = *bucket.get_mut();
let this_bucket_size = bucket_size;
if i != 0 {
bucket_size <<= 1;
}
if bucket_ptr.is_null() {
continue;
}
let _ = unsafe {
Box::from_raw(std::slice::from_raw_parts_mut(bucket_ptr, this_bucket_size))
};
}
}
}
pub struct Iter<'a, T>
where
T: Send,
{
thread_local: &'a ThreadLocal<T>,
bucket: usize,
bucket_size: usize,
index: usize,
}
impl<'a, T> Iterator for Iter<'a, T>
where
T: Send,
{
type Item = &'a T;
fn next(&mut self) -> Option<Self::Item> {
// because we reuse thread IDs, a new thread could join and be inserted into the middle of the list,
// so we have to check all the buckets here. yielding extra values is fine, but not yielding all originally
// active threads is not
while self.bucket < BUCKETS {
let bucket = unsafe {
self.thread_local
.buckets
.get_unchecked(self.bucket)
.load(Ordering::Acquire)
};
if !bucket.is_null() {
while self.index < self.bucket_size {
let entry = unsafe { &*bucket.add(self.index) };
self.index += 1;
if entry.present.load(Ordering::Acquire) {
return Some(unsafe { (*entry.value.get()).assume_init_ref() });
}
}
}
if self.bucket != 0 {
self.bucket_size <<= 1;
}
self.bucket += 1;
self.index = 0;
}
None
}
}
fn allocate_bucket<T>(size: usize) -> *mut Entry<T> {
Box::into_raw(
(0..size)
.map(|_| Entry::<T> {
present: AtomicBool::new(false),
value: UnsafeCell::new(MaybeUninit::uninit()),
})
.collect(),
) as *mut _
}
#[cfg(test)]
#[allow(clippy::redundant_closure)]
mod tests {
use super::*;
use std::cell::RefCell;
use std::sync::atomic::Ordering::Relaxed;
use std::sync::Arc;
use std::thread;
fn make_create() -> Arc<dyn Fn() -> usize + Send + Sync> {
let count = AtomicUsize::new(0);
Arc::new(move || count.fetch_add(1, Relaxed))
}
#[test]
fn same_thread() {
let create = make_create();
let tls = ThreadLocal::with_capacity(1);
assert_eq!(None, tls.try_load());
assert_eq!(0, *tls.load_or(|| create(), Thread::current()));
assert_eq!(Some(&0), tls.try_load());
assert_eq!(0, *tls.load_or(|| create(), Thread::current()));
assert_eq!(Some(&0), tls.try_load());
assert_eq!(0, *tls.load_or(|| create(), Thread::current()));
assert_eq!(Some(&0), tls.try_load());
}
#[test]
fn different_thread() {
let create = make_create();
let tls = Arc::new(ThreadLocal::with_capacity(1));
assert_eq!(None, tls.try_load());
assert_eq!(0, *tls.load_or(|| create(), Thread::current()));
assert_eq!(Some(&0), tls.try_load());
let tls2 = tls.clone();
let create2 = create.clone();
thread::spawn(move || {
assert_eq!(None, tls2.try_load());
assert_eq!(1, *tls2.load_or(|| create2(), Thread::current()));
assert_eq!(Some(&1), tls2.try_load());
})
.join()
.unwrap();
assert_eq!(Some(&0), tls.try_load());
assert_eq!(0, *tls.load_or(|| create(), Thread::current()));
}
#[test]
fn iter() {
let tls = Arc::new(ThreadLocal::with_capacity(1));
tls.load_or(|| Box::new(1), Thread::current());
let tls2 = tls.clone();
thread::spawn(move || {
tls2.load_or(|| Box::new(2), Thread::current());
let tls3 = tls2.clone();
thread::spawn(move || {
tls3.load_or(|| Box::new(3), Thread::current());
})
.join()
.unwrap();
drop(tls2);
})
.join()
.unwrap();
let tls = Arc::try_unwrap(tls).unwrap_or_else(|_| panic!("."));
let mut v = tls.iter().map(|x| **x).collect::<Vec<i32>>();
v.sort_unstable();
assert_eq!(vec![1, 2, 3], v);
}
#[test]
fn iter_snapshot() {
let tls = Arc::new(ThreadLocal::with_capacity(1));
tls.load_or(|| Box::new(1), Thread::current());
let iterator = tls.iter();
tls.load_or(|| Box::new(2), Thread::current());
let v = iterator.map(|x| **x).collect::<Vec<i32>>();
assert_eq!(vec![1], v);
}
#[test]
fn test_drop() {
let local = ThreadLocal::with_capacity(1);
struct Dropped(Arc<AtomicUsize>);
impl Drop for Dropped {
fn drop(&mut self) {
self.0.fetch_add(1, Relaxed);
}
}
let dropped = Arc::new(AtomicUsize::new(0));
local.load_or(|| Dropped(dropped.clone()), Thread::current());
assert_eq!(dropped.load(Relaxed), 0);
drop(local);
assert_eq!(dropped.load(Relaxed), 1);
}
#[test]
fn is_sync() {
fn foo<T: Sync>() {}
foo::<ThreadLocal<String>>();
foo::<ThreadLocal<RefCell<String>>>();
}
}

131
vendor/seize/src/tls/thread_id.rs vendored Normal file
View File

@@ -0,0 +1,131 @@
// Copyright 2017 Amanieu d'Antras
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.
use std::cmp::Reverse;
use std::collections::BinaryHeap;
use std::sync::{Mutex, OnceLock};
use std::usize;
/// Thread ID manager which allocates thread IDs. It attempts to aggressively
/// reuse thread IDs where possible to avoid cases where a ThreadLocal grows
/// indefinitely when it is used by many short-lived threads.
#[derive(Default)]
struct ThreadIdManager {
free_from: usize,
free_list: BinaryHeap<Reverse<usize>>,
}
impl ThreadIdManager {
fn alloc(&mut self) -> usize {
if let Some(id) = self.free_list.pop() {
id.0
} else {
let id = self.free_from;
self.free_from = self
.free_from
.checked_add(1)
.expect("Ran out of thread IDs");
id
}
}
fn free(&mut self, id: usize) {
self.free_list.push(Reverse(id));
}
}
fn thread_id_manager() -> &'static Mutex<ThreadIdManager> {
static THREAD_ID_MANAGER: OnceLock<Mutex<ThreadIdManager>> = OnceLock::new();
THREAD_ID_MANAGER.get_or_init(Default::default)
}
/// Data which is unique to the current thread while it is running.
/// A thread ID may be reused after a thread exits.
#[derive(Clone, Copy)]
pub struct Thread {
pub(crate) id: usize,
pub(crate) bucket: usize,
pub(crate) bucket_size: usize,
pub(crate) index: usize,
}
impl Thread {
pub const EMPTY: Thread = Thread {
id: 0,
bucket: 0,
bucket_size: 0,
index: 0,
};
fn new(id: usize) -> Thread {
let bucket = (usize::BITS as usize) - id.leading_zeros() as usize;
let bucket_size = 1 << bucket.saturating_sub(1);
let index = if id != 0 { id ^ bucket_size } else { 0 };
Thread {
id,
bucket,
bucket_size,
index,
}
}
/// Get the current thread.
pub fn current() -> Thread {
THREAD_HOLDER.with(|holder| holder.0)
}
}
/// Wrapper around `Thread` that allocates and deallocates the ID.
struct ThreadHolder(Thread);
impl ThreadHolder {
fn new() -> ThreadHolder {
ThreadHolder(Thread::new(thread_id_manager().lock().unwrap().alloc()))
}
}
impl Drop for ThreadHolder {
fn drop(&mut self) {
thread_id_manager().lock().unwrap().free(self.0.id);
}
}
thread_local!(static THREAD_HOLDER: ThreadHolder = ThreadHolder::new());
#[test]
fn test_thread() {
let thread = Thread::new(0);
assert_eq!(thread.id, 0);
assert_eq!(thread.bucket, 0);
assert_eq!(thread.bucket_size, 1);
assert_eq!(thread.index, 0);
let thread = Thread::new(1);
assert_eq!(thread.id, 1);
assert_eq!(thread.bucket, 1);
assert_eq!(thread.bucket_size, 1);
assert_eq!(thread.index, 0);
let thread = Thread::new(2);
assert_eq!(thread.id, 2);
assert_eq!(thread.bucket, 2);
assert_eq!(thread.bucket_size, 2);
assert_eq!(thread.index, 0);
let thread = Thread::new(3);
assert_eq!(thread.id, 3);
assert_eq!(thread.bucket, 2);
assert_eq!(thread.bucket_size, 2);
assert_eq!(thread.index, 1);
let thread = Thread::new(19);
assert_eq!(thread.id, 19);
assert_eq!(thread.bucket, 5);
assert_eq!(thread.bucket_size, 16);
assert_eq!(thread.index, 3);
}

56
vendor/seize/src/utils.rs vendored Normal file
View File

@@ -0,0 +1,56 @@
/// Pads and aligns a value to the length of a cache line.
#[cfg_attr(
any(
target_arch = "x86_64",
target_arch = "aarch64",
target_arch = "powerpc64",
),
repr(align(128))
)]
#[cfg_attr(
any(
target_arch = "arm",
target_arch = "mips",
target_arch = "mips64",
target_arch = "riscv64",
),
repr(align(32))
)]
#[cfg_attr(target_arch = "s390x", repr(align(256)))]
#[cfg_attr(
not(any(
target_arch = "x86_64",
target_arch = "aarch64",
target_arch = "powerpc64",
target_arch = "arm",
target_arch = "mips",
target_arch = "mips64",
target_arch = "riscv64",
target_arch = "s390x",
)),
repr(align(64))
)]
#[derive(Default)]
pub struct CachePadded<T> {
pub value: T,
}
impl<T> CachePadded<T> {
pub fn new(value: T) -> Self {
Self { value }
}
}
impl<T> std::ops::Deref for CachePadded<T> {
type Target = T;
fn deref(&self) -> &T {
&self.value
}
}
impl<T> std::ops::DerefMut for CachePadded<T> {
fn deref_mut(&mut self) -> &mut T {
&mut self.value
}
}