chore: checkpoint before Python removal

This commit is contained in:
2026-03-26 22:33:59 +00:00
parent 683cec9307
commit e568ddf82a
29972 changed files with 11269302 additions and 2 deletions

38
vendor/icu_provider/src/baked.rs vendored Normal file
View File

@@ -0,0 +1,38 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
//! This module contains scaffolding for baked providers, typically generated using
//! databake.
//!
//! It can be glob-imported, and includes the icu_provider prelude.
//!
//! This needs the `"baked"` feature to be enabled.
pub mod zerotrie;
use crate::prelude::{DataIdentifierBorrowed, DataMarker, DataPayload};
/// A backing store for baked data
pub trait DataStore<M: DataMarker>: private::Sealed {
/// Get the value for a key
fn get(
&self,
req: DataIdentifierBorrowed,
attributes_prefix_match: bool,
) -> Option<DataPayload<M>>;
/// The type returned by the iterator
///
/// ✨ *Enabled with the `alloc` Cargo feature.*
#[cfg(feature = "alloc")]
type IterReturn: Iterator<Item = crate::prelude::DataIdentifierCow<'static>>;
/// Iterate over all data
///
/// ✨ *Enabled with the `alloc` Cargo feature.*
#[cfg(feature = "alloc")]
fn iter(&'static self) -> Self::IterReturn;
}
pub(crate) mod private {
pub trait Sealed {}
}

View File

@@ -0,0 +1,218 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
//! Data stored as as [`ZeroTrieSimpleAscii`]
/// This is a valid separator as `DataLocale` will never produce it.
///
/// Mostly for internal use
pub const ID_SEPARATOR: u8 = 0x1E;
pub use crate::DynamicDataMarker;
use crate::{
prelude::{zerofrom::ZeroFrom, *},
ule::MaybeAsVarULE,
};
pub use zerotrie::ZeroTrieSimpleAscii;
use zerovec::VarZeroSlice;
fn get_index(
trie: ZeroTrieSimpleAscii<&'static [u8]>,
id: DataIdentifierBorrowed,
attributes_prefix_match: bool,
) -> Option<usize> {
use writeable::Writeable;
let mut cursor = trie.cursor();
let _is_ascii = id.locale.write_to(&mut cursor);
if !id.marker_attributes.is_empty() {
cursor.step(ID_SEPARATOR);
id.marker_attributes.write_to(&mut cursor).ok()?;
loop {
if let Some(v) = cursor.take_value() {
break Some(v);
}
if !attributes_prefix_match || cursor.probe(0).is_none() {
break None;
}
}
} else {
cursor.take_value()
}
}
#[cfg(feature = "alloc")]
#[expect(clippy::type_complexity)]
fn iter(
trie: &'static ZeroTrieSimpleAscii<&'static [u8]>,
) -> core::iter::FilterMap<
zerotrie::ZeroTrieStringIterator<'static>,
fn((alloc::string::String, usize)) -> Option<DataIdentifierCow<'static>>,
> {
use alloc::borrow::ToOwned;
trie.iter().filter_map(move |(s, _)| {
if let Some((locale, attrs)) = s.split_once(ID_SEPARATOR as char) {
Some(DataIdentifierCow::from_owned(
DataMarkerAttributes::try_from_str(attrs).ok()?.to_owned(),
locale.parse().ok()?,
))
} else {
s.parse().ok().map(DataIdentifierCow::from_locale)
}
})
}
/// Regular baked data: a trie for lookups and a slice of values
#[derive(Debug)]
pub struct Data<M: DataMarker> {
// Unsafe invariant: actual values contained MUST be valid indices into `values`
trie: ZeroTrieSimpleAscii<&'static [u8]>,
values: &'static [M::DataStruct],
}
impl<M: DataMarker> Data<M> {
/// Construct from a trie and values
///
/// # Safety
/// The actual values contained in the trie must be valid indices into `values`
pub const unsafe fn from_trie_and_values_unchecked(
trie: ZeroTrieSimpleAscii<&'static [u8]>,
values: &'static [M::DataStruct],
) -> Self {
Self { trie, values }
}
}
impl<M: DataMarker> super::private::Sealed for Data<M> {}
impl<M: DataMarker> super::DataStore<M> for Data<M> {
fn get(
&self,
id: DataIdentifierBorrowed,
attributes_prefix_match: bool,
) -> Option<DataPayload<M>> {
get_index(self.trie, id, attributes_prefix_match)
// Safety: Allowed since `i` came from the trie and the field safety invariant
.map(|i| unsafe { self.values.get_unchecked(i) })
.map(DataPayload::from_static_ref)
}
#[cfg(feature = "alloc")]
type IterReturn = core::iter::FilterMap<
zerotrie::ZeroTrieStringIterator<'static>,
fn((alloc::string::String, usize)) -> Option<DataIdentifierCow<'static>>,
>;
#[cfg(feature = "alloc")]
fn iter(&'static self) -> Self::IterReturn {
iter(&self.trie)
}
}
/// Regular baked data: a trie for lookups and a slice of values
#[derive(Debug)]
pub struct DataRef<M: DataMarker> {
// Unsafe invariant: actual values contained MUST be valid indices into `values`
trie: ZeroTrieSimpleAscii<&'static [u8]>,
values: &'static [&'static M::DataStruct],
}
impl<M: DataMarker> DataRef<M> {
/// Construct from a trie and references to values
///
/// # Safety
/// The actual values contained in the trie must be valid indices into `values`
pub const unsafe fn from_trie_and_refs_unchecked(
trie: ZeroTrieSimpleAscii<&'static [u8]>,
values: &'static [&'static M::DataStruct],
) -> Self {
Self { trie, values }
}
}
impl<M: DataMarker> super::private::Sealed for DataRef<M> {}
impl<M: DataMarker> super::DataStore<M> for DataRef<M> {
fn get(
&self,
id: DataIdentifierBorrowed,
attributes_prefix_match: bool,
) -> Option<DataPayload<M>> {
get_index(self.trie, id, attributes_prefix_match)
// Safety: Allowed since `i` came from the trie and the field safety invariant
.map(|i| unsafe { self.values.get_unchecked(i) })
.copied()
.map(DataPayload::from_static_ref)
}
#[cfg(feature = "alloc")]
type IterReturn = core::iter::FilterMap<
zerotrie::ZeroTrieStringIterator<'static>,
fn((alloc::string::String, usize)) -> Option<DataIdentifierCow<'static>>,
>;
#[cfg(feature = "alloc")]
fn iter(&'static self) -> Self::IterReturn {
iter(&self.trie)
}
}
/// Optimized data stored as a single VarZeroSlice to reduce token count
#[allow(missing_debug_implementations)] // Debug on this will not be too useful
pub struct DataForVarULEs<M: DataMarker>
where
M::DataStruct: MaybeAsVarULE,
M::DataStruct: ZeroFrom<'static, <M::DataStruct as MaybeAsVarULE>::EncodedStruct>,
{
// Unsafe invariant: actual values contained MUST be valid indices into `values`
trie: ZeroTrieSimpleAscii<&'static [u8]>,
values: &'static VarZeroSlice<<M::DataStruct as MaybeAsVarULE>::EncodedStruct>,
}
impl<M: DataMarker> super::private::Sealed for DataForVarULEs<M>
where
M::DataStruct: MaybeAsVarULE,
M::DataStruct: ZeroFrom<'static, <M::DataStruct as MaybeAsVarULE>::EncodedStruct>,
{
}
impl<M: DataMarker> DataForVarULEs<M>
where
M::DataStruct: MaybeAsVarULE,
M::DataStruct: ZeroFrom<'static, <M::DataStruct as MaybeAsVarULE>::EncodedStruct>,
{
/// Construct from a trie and values
///
/// # Safety
/// The actual values contained in the trie must be valid indices into `values`
pub const unsafe fn from_trie_and_values_unchecked(
trie: ZeroTrieSimpleAscii<&'static [u8]>,
values: &'static VarZeroSlice<<M::DataStruct as MaybeAsVarULE>::EncodedStruct>,
) -> Self {
Self { trie, values }
}
}
impl<M: DataMarker> super::DataStore<M> for DataForVarULEs<M>
where
M::DataStruct: MaybeAsVarULE,
M::DataStruct: ZeroFrom<'static, <M::DataStruct as MaybeAsVarULE>::EncodedStruct>,
{
fn get(
&self,
id: DataIdentifierBorrowed,
attributes_prefix_match: bool,
) -> Option<DataPayload<M>> {
get_index(self.trie, id, attributes_prefix_match)
// Safety: Allowed since `i` came from the trie and the field safety invariant
.map(|i| unsafe { self.values.get_unchecked(i) })
.map(M::DataStruct::zero_from)
.map(DataPayload::from_owned)
}
#[cfg(feature = "alloc")]
type IterReturn = core::iter::FilterMap<
zerotrie::ZeroTrieStringIterator<'static>,
fn((alloc::string::String, usize)) -> Option<DataIdentifierCow<'static>>,
>;
#[cfg(feature = "alloc")]
fn iter(&'static self) -> Self::IterReturn {
iter(&self.trie)
}
}

133
vendor/icu_provider/src/buf.rs vendored Normal file
View File

@@ -0,0 +1,133 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
//! Traits for data providers that produce opaque buffers.
use crate::prelude::*;
#[cfg(feature = "serde")]
mod serde;
#[cfg(feature = "serde")]
pub use self::serde::*;
/// [`DynamicDataMarker`] for raw buffers. Returned by [`BufferProvider`].
///
/// The data is expected to be deserialized before it can be used; see
/// [`DataPayload::into_deserialized`].
#[non_exhaustive]
#[derive(Debug)]
pub struct BufferMarker;
impl DynamicDataMarker for BufferMarker {
type DataStruct = &'static [u8];
}
/// A data provider that returns opaque bytes.
///
/// Generally, these bytes are expected to be deserializable with Serde. To get an object
/// implementing [`DataProvider`] via Serde, use [`as_deserializing()`].
///
/// Passing a `BufferProvider` to a `*_with_buffer_provider` constructor requires enabling
/// the deserialization Cargo feature for the expected format(s):
/// - `deserialize_json`
/// - `deserialize_postcard_1`
/// - `deserialize_bincode_1`
///
/// Along with [`DataProvider`], this is one of the two foundational traits in this crate.
///
/// [`BufferProvider`] can be made into a trait object. It is used over FFI.
///
/// # Examples
///
/// ```
/// # #[cfg(feature = "deserialize_json")] {
/// use icu_locale_core::langid;
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
/// use std::borrow::Cow;
///
/// let buffer_provider = HelloWorldProvider.into_json_provider();
///
/// // Deserializing manually
/// assert_eq!(
/// serde_json::from_slice::<HelloWorld>(
/// buffer_provider
/// .load_data(
/// HelloWorldV1::INFO,
/// DataRequest {
/// id: DataIdentifierBorrowed::for_locale(
/// &langid!("de").into()
/// ),
/// ..Default::default()
/// }
/// )
/// .expect("load should succeed")
/// .payload
/// .get()
/// )
/// .expect("should deserialize"),
/// HelloWorld {
/// message: Cow::Borrowed("Hallo Welt"),
/// },
/// );
///
/// // Deserialize automatically
/// let deserializing_provider: &dyn DataProvider<HelloWorldV1> =
/// &buffer_provider.as_deserializing();
///
/// assert_eq!(
/// deserializing_provider
/// .load(DataRequest {
/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()),
/// ..Default::default()
/// })
/// .expect("load should succeed")
/// .payload
/// .get(),
/// &HelloWorld {
/// message: Cow::Borrowed("Hallo Welt"),
/// },
/// );
/// # }
/// ```
///
/// [`as_deserializing()`]: AsDeserializingBufferProvider::as_deserializing
pub trait BufferProvider: DynamicDataProvider<BufferMarker> {}
impl<P: DynamicDataProvider<BufferMarker> + ?Sized> BufferProvider for P {}
/// An enum expressing all Serde formats known to ICU4X.
#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)]
#[cfg_attr(feature = "serde", derive(::serde::Serialize, ::serde::Deserialize))]
#[non_exhaustive]
pub enum BufferFormat {
/// Serialize using JavaScript Object Notation (JSON), using the [`serde_json`] crate.
Json,
/// Serialize using the [`bincode`] crate, version 1.
Bincode1,
/// Serialize using the [`postcard`] crate, version 1.
Postcard1,
}
impl BufferFormat {
/// Returns an error if the buffer format is not enabled.
pub fn check_available(&self) -> Result<(), DataError> {
match self {
#[cfg(feature = "deserialize_json")]
BufferFormat::Json => Ok(()),
#[cfg(not(feature = "deserialize_json"))]
BufferFormat::Json => Err(DataErrorKind::Deserialize.with_str_context("deserializing `BufferFormat::Json` requires the `deserialize_json` Cargo feature")),
#[cfg(feature = "deserialize_bincode_1")]
BufferFormat::Bincode1 => Ok(()),
#[cfg(not(feature = "deserialize_bincode_1"))]
BufferFormat::Bincode1 => Err(DataErrorKind::Deserialize.with_str_context("deserializing `BufferFormat::Bincode1` requires the `deserialize_bincode_1` Cargo feature")),
#[cfg(feature = "deserialize_postcard_1")]
BufferFormat::Postcard1 => Ok(()),
#[cfg(not(feature = "deserialize_postcard_1"))]
BufferFormat::Postcard1 => Err(DataErrorKind::Deserialize.with_str_context("deserializing `BufferFormat::Postcard1` requires the `deserialize_postcard_1` Cargo feature")),
}
}
}

246
vendor/icu_provider/src/buf/serde.rs vendored Normal file
View File

@@ -0,0 +1,246 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
//! Provides the [`DeserializingBufferProvider`] wrapper, which deserializes data using Serde.
//!
//! Providers that produce opaque buffers that need to be deserialized into concrete data structs,
//! such as `FsDataProvider`, should implement [`BufferProvider`]. These can be converted into
//! [`DeserializingBufferProvider`] using the [`as_deserializing`](AsDeserializingBufferProvider::as_deserializing)
//! convenience method.
//!
//! [`BufferProvider`]: crate::buf::BufferProvider
use crate::buf::BufferFormat;
use crate::buf::BufferProvider;
use crate::data_provider::DynamicDryDataProvider;
use crate::prelude::*;
use crate::DryDataProvider;
use serde::de::Deserialize;
use yoke::Yokeable;
/// A [`BufferProvider`] that deserializes its data using Serde.
#[derive(Debug)]
pub struct DeserializingBufferProvider<'a, P: ?Sized>(&'a P);
/// Blanket-implemented trait adding the [`Self::as_deserializing()`] function.
pub trait AsDeserializingBufferProvider {
/// Wrap this [`BufferProvider`] in a [`DeserializingBufferProvider`].
///
/// This requires enabling the deserialization Cargo feature
/// for the expected format(s):
///
/// - `deserialize_json`
/// - `deserialize_postcard_1`
/// - `deserialize_bincode_1`
fn as_deserializing(&self) -> DeserializingBufferProvider<'_, Self>;
}
impl<P> AsDeserializingBufferProvider for P
where
P: BufferProvider + ?Sized,
{
/// Wrap this [`BufferProvider`] in a [`DeserializingBufferProvider`].
///
/// This requires enabling the deserialization Cargo feature
/// for the expected format(s):
///
/// - `deserialize_json`
/// - `deserialize_postcard_1`
/// - `deserialize_bincode_1`
fn as_deserializing(&self) -> DeserializingBufferProvider<'_, Self> {
DeserializingBufferProvider(self)
}
}
fn deserialize_impl<'data, M>(
// Allow `bytes` to be unused in case all buffer formats are disabled
#[allow(unused_variables)] bytes: &'data [u8],
buffer_format: BufferFormat,
) -> Result<<M::DataStruct as Yokeable<'data>>::Output, DataError>
where
M: DynamicDataMarker,
for<'de> <M::DataStruct as Yokeable<'de>>::Output: Deserialize<'de>,
{
match buffer_format {
#[cfg(feature = "deserialize_json")]
BufferFormat::Json => {
let mut d = serde_json::Deserializer::from_slice(bytes);
Ok(Deserialize::deserialize(&mut d)?)
}
#[cfg(feature = "deserialize_bincode_1")]
BufferFormat::Bincode1 => {
use bincode::Options;
let options = bincode::DefaultOptions::new()
.with_fixint_encoding()
.allow_trailing_bytes();
let mut d = bincode::de::Deserializer::from_slice(bytes, options);
Ok(Deserialize::deserialize(&mut d)?)
}
#[cfg(feature = "deserialize_postcard_1")]
BufferFormat::Postcard1 => {
let mut d = postcard::Deserializer::from_bytes(bytes);
Ok(Deserialize::deserialize(&mut d)?)
}
// Allowed for cases in which all features are enabled
#[allow(unreachable_patterns)]
_ => {
buffer_format.check_available()?;
unreachable!()
}
}
}
impl DataPayload<BufferMarker> {
/// Deserialize a [`DataPayload`]`<`[`BufferMarker`]`>` into a [`DataPayload`] of a
/// specific concrete type.
///
/// This requires enabling the deserialization Cargo feature
/// for the expected format(s):
///
/// - `deserialize_json`
/// - `deserialize_postcard_1`
/// - `deserialize_bincode_1`
///
/// This function takes the buffer format as an argument. When a buffer payload is returned
/// from a data provider, the buffer format is stored in the [`DataResponseMetadata`].
///
/// # Examples
///
/// Requires the `deserialize_json` Cargo feature:
///
/// ```
/// use icu_provider::buf::BufferFormat;
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
///
/// let buffer: &[u8] = br#"{"message":"Hallo Welt"}"#;
///
/// let buffer_payload = DataPayload::from_owned(buffer);
/// let payload: DataPayload<HelloWorldV1> = buffer_payload
/// .into_deserialized(BufferFormat::Json)
/// .expect("Deserialization successful");
///
/// assert_eq!(payload.get().message, "Hallo Welt");
/// ```
pub fn into_deserialized<M>(
self,
buffer_format: BufferFormat,
) -> Result<DataPayload<M>, DataError>
where
M: DynamicDataMarker,
for<'de> <M::DataStruct as Yokeable<'de>>::Output: Deserialize<'de>,
{
self.try_map_project(|bytes, _| deserialize_impl::<M>(bytes, buffer_format))
}
}
impl<P, M> DynamicDataProvider<M> for DeserializingBufferProvider<'_, P>
where
M: DynamicDataMarker,
P: BufferProvider + ?Sized,
for<'de> <M::DataStruct as Yokeable<'de>>::Output: Deserialize<'de>,
{
/// Converts a buffer into a concrete type by deserializing from a supported buffer format.
///
/// This requires enabling the deserialization Cargo feature
/// for the expected format(s):
///
/// - `deserialize_json`
/// - `deserialize_postcard_1`
/// - `deserialize_bincode_1`
fn load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponse<M>, DataError> {
let buffer_response = self.0.load_data(marker, req)?;
let buffer_format = buffer_response.metadata.buffer_format.ok_or_else(|| {
DataErrorKind::Deserialize
.with_str_context("BufferProvider didn't set BufferFormat")
.with_req(marker, req)
})?;
Ok(DataResponse {
metadata: buffer_response.metadata,
payload: buffer_response
.payload
.into_deserialized(buffer_format)
.map_err(|e| e.with_req(marker, req))?,
})
}
}
impl<P, M> DynamicDryDataProvider<M> for DeserializingBufferProvider<'_, P>
where
M: DynamicDataMarker,
P: DynamicDryDataProvider<BufferMarker> + ?Sized,
for<'de> <M::DataStruct as Yokeable<'de>>::Output: serde::de::Deserialize<'de>,
{
fn dry_load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponseMetadata, DataError> {
self.0.dry_load_data(marker, req)
}
}
impl<P, M> DataProvider<M> for DeserializingBufferProvider<'_, P>
where
M: DataMarker,
P: DynamicDataProvider<BufferMarker> + ?Sized,
for<'de> <M::DataStruct as Yokeable<'de>>::Output: Deserialize<'de>,
{
/// Converts a buffer into a concrete type by deserializing from a supported buffer format.
///
/// This requires enabling the deserialization Cargo feature
/// for the expected format(s):
///
/// - `deserialize_json`
/// - `deserialize_postcard_1`
/// - `deserialize_bincode_1`
fn load(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
self.load_data(M::INFO, req)
}
}
impl<P, M> DryDataProvider<M> for DeserializingBufferProvider<'_, P>
where
M: DataMarker,
P: DynamicDryDataProvider<BufferMarker> + ?Sized,
for<'de> <M::DataStruct as Yokeable<'de>>::Output: Deserialize<'de>,
{
fn dry_load(&self, req: DataRequest) -> Result<DataResponseMetadata, DataError> {
self.0.dry_load_data(M::INFO, req)
}
}
#[cfg(feature = "deserialize_json")]
impl From<serde_json::error::Error> for crate::DataError {
fn from(e: serde_json::error::Error) -> Self {
DataErrorKind::Deserialize
.with_str_context("serde_json")
.with_display_context(&e)
}
}
#[cfg(feature = "deserialize_bincode_1")]
impl From<bincode::Error> for crate::DataError {
fn from(e: bincode::Error) -> Self {
DataErrorKind::Deserialize
.with_str_context("bincode")
.with_display_context(&e)
}
}
#[cfg(feature = "deserialize_postcard_1")]
impl From<postcard::Error> for crate::DataError {
fn from(e: postcard::Error) -> Self {
DataErrorKind::Deserialize
.with_str_context("postcard")
.with_display_context(&e)
}
}

226
vendor/icu_provider/src/constructors.rs vendored Normal file
View File

@@ -0,0 +1,226 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
//! 📚 *This module documents ICU4X constructor signatures.*
//!
//! One of the key differences between ICU4X and its parent projects, ICU4C and ICU4J, is in how
//! it deals with locale data.
//!
//! In ICU4X, data can always be explicitly passed to any function that requires data.
//! This enables ICU4X to achieve the following value propositions:
//!
//! 1. Configurable data sources (machine-readable data file, baked into code, JSON, etc).
//! 2. Dynamic data loading at runtime (load data on demand).
//! 3. Reduced overhead and code size (data is resolved locally at each call site).
//! 4. Explicit support for multiple ICU4X instances sharing data.
//!
//! However, as manual data management can be tedious, ICU4X also has a `compiled_data`
//! default Cargo feature that includes data and makes ICU4X work out-of-the box.
//!
//! Subsequently, there are 3 versions of all Rust ICU4X functions that use data:
//!
//! 1. `*`
//! 2. `*_unstable`
//! 3. `*_with_buffer_provider`
//!
//! # Which constructor should I use?
//!
//! ## When to use `*`
//!
//! If you don't want to customize data at runtime (i.e. if you don't care about code size,
//! updating your data, etc.) you can use the `compiled_data` Cargo feature and don't have to think
//! about where your data comes from.
//!
//! These constructors are sometimes `const` functions, this way Rust can most effectively optimize
//! your usage of ICU4X.
//!
//! ## When to use `*_unstable`
//!
//! Use this constructor if your data provider implements the [`DataProvider`] trait for all
//! data structs in *current and future* ICU4X versions. Examples:
//!
//! 1. `BakedDataProvider` generated for the specific ICU4X minor version
//! 2. Anything with a _blanket_ [`DataProvider`] impl
//!
//! Since the exact set of bounds may change at any time, including in minor SemVer releases,
//! it is the client's responsibility to guarantee that the requirement is upheld.
//!
//! ## When to use `*_with_buffer_provider`
//!
//! Use this constructor if your data originates as byte buffers that need to be deserialized.
//! All such providers should implement [`BufferProvider`]. Examples:
//!
//! 1. [`BlobDataProvider`]
//! 2. [`FsDataProvider`]
//! 3. [`ForkByMarkerProvider`] between two providers implementing [`BufferProvider`]
//!
//! Please note that you must enable the `serde` Cargo feature on each crate in which you use the
//! `*_with_buffer_provider` constructor.
//!
//! # Data Versioning Policy
//!
//! The `*_with_buffer_provider` functions will succeed to compile and
//! run if given a data provider supporting all of the markers required for the object being
//! constructed, either the current or any previous version within the same SemVer major release.
//! For example, if a data file is built to support FooFormatter version 1.1, then FooFormatter
//! version 1.2 will be able to read the same data file. Likewise, backwards-compatible markers can
//! always be included by `icu_provider_export` to support older library versions.
//!
//! The `*_unstable` functions are only guaranteed to work on data built for the exact same minor version
//! of ICU4X. The advantage of the `*_unstable` functions is that they result in the smallest code
//! size and allow for automatic data slicing when `BakedDataProvider` is used. However, the type
//! bounds of this function may change over time, breaking SemVer guarantees. These functions
//! should therefore only be used when you have full control over your data lifecycle at compile
//! time.
//!
//! # Data Providers Over FFI
//!
//! Over FFI, there is only one data provider type: [`ICU4XDataProvider`]. Internally, it is an
//! `enum` between`dyn `[`BufferProvider`] and a unit compiled data variant.
//!
//! To control for code size, there are two Cargo features, `compiled_data` and `buffer_provider`,
//! that enable the corresponding items in the enum.
//!
//! In Rust ICU4X, a similar enum approach was not taken because:
//!
//! 1. Feature-gating the enum branches gets complex across crates.
//! 2. Without feature gating, users need to carry Serde code even if they're not using it,
//! violating one of the core value propositions of ICU4X.
//! 3. We could reduce the number of constructors from 4 to 2 but not to 1, so the educational
//! benefit is limited.
//!
//! [`DataProvider`]: crate::DataProvider
//! [`BufferProvider`]: crate::buf::BufferProvider
//! [`FixedProvider`]: ../../icu_provider_adapters/fixed/struct.FixedProvider.html
//! [`ForkByMarkerProvider`]: ../../icu_provider_adapters/fork/struct.ForkByMarkerProvider.html
//! [`BlobDataProvider`]: ../../icu_provider_blob/struct.BlobDataProvider.html
//! [`StaticDataProvider`]: ../../icu_provider_blob/struct.StaticDataProvider.html
//! [`FsDataProvider`]: ../../icu_provider_blob/struct.FsDataProvider.html
//! [`ICU4XDataProvider`]: ../../icu_capi/provider/ffi/struct.ICU4XDataProvider.html
#[doc(hidden)] // macro
#[macro_export]
macro_rules! gen_buffer_unstable_docs {
(BUFFER, $data:path) => {
concat!(
"A version of [`", stringify!($data), "`] that uses custom data ",
"provided by a [`BufferProvider`].\n\n",
"✨ *Enabled with the `serde` feature.*\n\n",
"[📚 Help choosing a constructor](icu_provider::constructors)",
)
};
(UNSTABLE, $data:path) => {
concat!(
"A version of [`", stringify!($data), "`] that uses custom data ",
"provided by a [`DataProvider`].\n\n",
"[📚 Help choosing a constructor](icu_provider::constructors)\n\n",
"<div class=\"stab unstable\">⚠️ The bounds on <tt>provider</tt> may change over time, including in SemVer minor releases.</div>"
)
};
}
/// Usage:
///
/// ```rust,ignore
/// gen_buffer_data_constructors!((locale, options: FooOptions) -> error: DataError,
/// /// Some docs
/// functions: [try_new, try_new_with_buffer_provider, try_new_unstable]
/// );
/// ```
///
/// `functions` can be omitted if using standard names. If `locale` is omitted, the method will not take a locale. You can specify any number
/// of options arguments, including zero.
///
/// By default the macro will generate a `try_new`. If you wish to skip it, write `try_new: skip`
///
/// Errors can be specified as `error: SomeError` or `result: SomeType`, where `error` will get it wrapped in `Result<Self, SomeError>`.
#[expect(clippy::crate_in_macro_def)] // by convention each crate's data provider is `crate::provider::Baked`
#[doc(hidden)] // macro
#[macro_export]
macro_rules! gen_buffer_data_constructors {
// Allow people to omit the functions
(($($args:tt)*) -> $error_kind:ident: $error_ty:ty, $(#[$doc:meta])*) => {
$crate::gen_buffer_data_constructors!(
($($args)*) -> $error_kind: $error_ty,
$(#[$doc])*
functions: [
try_new,
try_new_with_buffer_provider,
try_new_unstable,
Self,
]
);
};
// Allow people to specify errors instead of results
(($($args:tt)*) -> error: $error_ty:path, $(#[$doc:meta])* functions: [$baked:ident$(: $baked_cmd:ident)?, $buffer:ident, $unstable:ident $(, $struct:ident)? $(,)?]) => {
$crate::gen_buffer_data_constructors!(
($($args)*) -> result: Result<Self, $error_ty>,
$(#[$doc])*
functions: [
$baked$(: $baked_cmd)?,
$buffer,
$unstable
$(, $struct)?
]
);
};
// locale shorthand
((locale, $($args:tt)*) -> result: $result_ty:ty, $(#[$doc:meta])* functions: [$baked:ident$(: $baked_cmd:ident)?, $buffer:ident, $unstable:ident $(, $struct:ident)? $(,)?]) => {
$crate::gen_buffer_data_constructors!(
(locale: &$crate::DataLocale, $($args)*) -> result: $result_ty,
$(#[$doc])*
functions: [
$baked$(: $baked_cmd)?,
$buffer,
$unstable
$(, $struct)?
]
);
};
((locale) -> result: $result_ty:ty, $(#[$doc:meta])* functions: [$baked:ident$(: $baked_cmd:ident)?, $buffer:ident, $unstable:ident $(, $struct:ident)? $(,)?]) => {
$crate::gen_buffer_data_constructors!(
(locale: &$crate::DataLocale) -> result: $result_ty,
$(#[$doc])*
functions: [
$baked$(: $baked_cmd)?,
$buffer,
$unstable
$(, $struct)?
]
);
};
(($($options_arg:ident: $options_ty:ty),*) -> result: $result_ty:ty, $(#[$doc:meta])* functions: [$baked:ident, $buffer:ident, $unstable:ident $(, $struct:ident)? $(,)?]) => {
#[cfg(feature = "compiled_data")]
$(#[$doc])*
///
/// ✨ *Enabled with the `compiled_data` Cargo feature.*
///
/// [📚 Help choosing a constructor](icu_provider::constructors)
pub fn $baked($($options_arg: $options_ty),* ) -> $result_ty {
$($struct :: )? $unstable(&crate::provider::Baked $(, $options_arg)* )
}
$crate::gen_buffer_data_constructors!(
($($options_arg: $options_ty),*) -> result: $result_ty,
$(#[$doc])*
functions: [
$baked: skip,
$buffer,
$unstable
$(, $struct)?
]
);
};
(($($options_arg:ident: $options_ty:ty),*) -> result: $result_ty:ty, $(#[$doc:meta])* functions: [$baked:ident: skip, $buffer:ident, $unstable:ident $(, $struct:ident)? $(,)?]) => {
#[cfg(feature = "serde")]
#[doc = $crate::gen_buffer_unstable_docs!(BUFFER, $($struct ::)? $baked)]
pub fn $buffer(provider: &(impl $crate::buf::BufferProvider + ?Sized) $(, $options_arg: $options_ty)* ) -> $result_ty {
use $crate::buf::AsDeserializingBufferProvider;
$($struct :: )? $unstable(&provider.as_deserializing() $(, $options_arg)* )
}
};
}

639
vendor/icu_provider/src/data_provider.rs vendored Normal file
View File

@@ -0,0 +1,639 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
use core::marker::PhantomData;
use yoke::Yokeable;
use crate::prelude::*;
/// A data provider that loads data for a specific [`DataMarkerInfo`].
pub trait DataProvider<M>
where
M: DataMarker,
{
/// Query the provider for data, returning the result.
///
/// Returns [`Ok`] if the request successfully loaded data. If data failed to load, returns an
/// Error with more information.
fn load(&self, req: DataRequest) -> Result<DataResponse<M>, DataError>;
}
impl<M, P> DataProvider<M> for &P
where
M: DataMarker,
P: DataProvider<M> + ?Sized,
{
#[inline]
fn load(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
(*self).load(req)
}
}
#[cfg(feature = "alloc")]
impl<M, P> DataProvider<M> for alloc::boxed::Box<P>
where
M: DataMarker,
P: DataProvider<M> + ?Sized,
{
#[inline]
fn load(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
(**self).load(req)
}
}
#[cfg(feature = "alloc")]
impl<M, P> DataProvider<M> for alloc::rc::Rc<P>
where
M: DataMarker,
P: DataProvider<M> + ?Sized,
{
#[inline]
fn load(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
(**self).load(req)
}
}
#[cfg(target_has_atomic = "ptr")]
#[cfg(feature = "alloc")]
impl<M, P> DataProvider<M> for alloc::sync::Arc<P>
where
M: DataMarker,
P: DataProvider<M> + ?Sized,
{
#[inline]
fn load(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
(**self).load(req)
}
}
/// A data provider that can determine whether it can load a particular data identifier,
/// potentially cheaper than actually performing the load.
pub trait DryDataProvider<M: DataMarker>: DataProvider<M> {
/// This method goes through the motions of [`load`], but only returns the metadata.
///
/// If `dry_load` returns an error, [`load`] must return the same error, but
/// not vice-versa. Concretely, [`load`] could return deserialization or I/O errors
/// that `dry_load` cannot predict.
///
/// [`load`]: DataProvider::load
fn dry_load(&self, req: DataRequest) -> Result<DataResponseMetadata, DataError>;
}
impl<M, P> DryDataProvider<M> for &P
where
M: DataMarker,
P: DryDataProvider<M> + ?Sized,
{
#[inline]
fn dry_load(&self, req: DataRequest) -> Result<DataResponseMetadata, DataError> {
(*self).dry_load(req)
}
}
#[cfg(feature = "alloc")]
impl<M, P> DryDataProvider<M> for alloc::boxed::Box<P>
where
M: DataMarker,
P: DryDataProvider<M> + ?Sized,
{
#[inline]
fn dry_load(&self, req: DataRequest) -> Result<DataResponseMetadata, DataError> {
(**self).dry_load(req)
}
}
#[cfg(feature = "alloc")]
impl<M, P> DryDataProvider<M> for alloc::rc::Rc<P>
where
M: DataMarker,
P: DryDataProvider<M> + ?Sized,
{
#[inline]
fn dry_load(&self, req: DataRequest) -> Result<DataResponseMetadata, DataError> {
(**self).dry_load(req)
}
}
#[cfg(target_has_atomic = "ptr")]
#[cfg(feature = "alloc")]
impl<M, P> DryDataProvider<M> for alloc::sync::Arc<P>
where
M: DataMarker,
P: DryDataProvider<M> + ?Sized,
{
#[inline]
fn dry_load(&self, req: DataRequest) -> Result<DataResponseMetadata, DataError> {
(**self).dry_load(req)
}
}
/// A [`DataProvider`] that can iterate over all supported [`DataIdentifierCow`]s.
///
/// The provider is not allowed to return `Ok` for requests that were not returned by `iter_ids`,
/// and must not fail with a [`DataErrorKind::IdentifierNotFound`] for requests that were returned.
///
/// ✨ *Enabled with the `alloc` Cargo feature.*
#[cfg(feature = "alloc")]
pub trait IterableDataProvider<M: DataMarker>: DataProvider<M> {
/// Returns a set of [`DataIdentifierCow`].
fn iter_ids(&self) -> Result<alloc::collections::BTreeSet<DataIdentifierCow<'_>>, DataError>;
}
/// A data provider that loads data for a specific data type.
///
/// Unlike [`DataProvider`], there may be multiple markers corresponding to the same data type.
pub trait DynamicDataProvider<M>
where
M: DynamicDataMarker,
{
/// Query the provider for data, returning the result.
///
/// Returns [`Ok`] if the request successfully loaded data. If data failed to load, returns an
/// Error with more information.
fn load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponse<M>, DataError>;
}
impl<M, P> DynamicDataProvider<M> for &P
where
M: DynamicDataMarker,
P: DynamicDataProvider<M> + ?Sized,
{
#[inline]
fn load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponse<M>, DataError> {
(*self).load_data(marker, req)
}
}
#[cfg(feature = "alloc")]
impl<M, P> DynamicDataProvider<M> for alloc::boxed::Box<P>
where
M: DynamicDataMarker,
P: DynamicDataProvider<M> + ?Sized,
{
#[inline]
fn load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponse<M>, DataError> {
(**self).load_data(marker, req)
}
}
#[cfg(feature = "alloc")]
impl<M, P> DynamicDataProvider<M> for alloc::rc::Rc<P>
where
M: DynamicDataMarker,
P: DynamicDataProvider<M> + ?Sized,
{
#[inline]
fn load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponse<M>, DataError> {
(**self).load_data(marker, req)
}
}
#[cfg(target_has_atomic = "ptr")]
#[cfg(feature = "alloc")]
impl<M, P> DynamicDataProvider<M> for alloc::sync::Arc<P>
where
M: DynamicDataMarker,
P: DynamicDataProvider<M> + ?Sized,
{
#[inline]
fn load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponse<M>, DataError> {
(**self).load_data(marker, req)
}
}
/// A dynanmic data provider that can determine whether it can load a particular data identifier,
/// potentially cheaper than actually performing the load.
pub trait DynamicDryDataProvider<M: DynamicDataMarker>: DynamicDataProvider<M> {
/// This method goes through the motions of [`load_data`], but only returns the metadata.
///
/// If `dry_load_data` returns an error, [`load_data`] must return the same error, but
/// not vice-versa. Concretely, [`load_data`] could return deserialization or I/O errors
/// that `dry_load_data` cannot predict.
///
/// [`load_data`]: DynamicDataProvider::load_data
fn dry_load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponseMetadata, DataError>;
}
impl<M, P> DynamicDryDataProvider<M> for &P
where
M: DynamicDataMarker,
P: DynamicDryDataProvider<M> + ?Sized,
{
#[inline]
fn dry_load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponseMetadata, DataError> {
(*self).dry_load_data(marker, req)
}
}
#[cfg(feature = "alloc")]
impl<M, P> DynamicDryDataProvider<M> for alloc::boxed::Box<P>
where
M: DynamicDataMarker,
P: DynamicDryDataProvider<M> + ?Sized,
{
#[inline]
fn dry_load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponseMetadata, DataError> {
(**self).dry_load_data(marker, req)
}
}
#[cfg(feature = "alloc")]
impl<M, P> DynamicDryDataProvider<M> for alloc::rc::Rc<P>
where
M: DynamicDataMarker,
P: DynamicDryDataProvider<M> + ?Sized,
{
#[inline]
fn dry_load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponseMetadata, DataError> {
(**self).dry_load_data(marker, req)
}
}
#[cfg(target_has_atomic = "ptr")]
#[cfg(feature = "alloc")]
impl<M, P> DynamicDryDataProvider<M> for alloc::sync::Arc<P>
where
M: DynamicDataMarker,
P: DynamicDryDataProvider<M> + ?Sized,
{
#[inline]
fn dry_load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponseMetadata, DataError> {
(**self).dry_load_data(marker, req)
}
}
/// A [`DynamicDataProvider`] that can iterate over all supported [`DataIdentifierCow`]s for a certain marker.
///
/// The provider is not allowed to return `Ok` for requests that were not returned by `iter_ids`,
/// and must not fail with a [`DataErrorKind::IdentifierNotFound`] for requests that were returned.
///
/// ✨ *Enabled with the `alloc` Cargo feature.*
#[cfg(feature = "alloc")]
pub trait IterableDynamicDataProvider<M: DynamicDataMarker>: DynamicDataProvider<M> {
/// Given a [`DataMarkerInfo`], returns a set of [`DataIdentifierCow`].
fn iter_ids_for_marker(
&self,
marker: DataMarkerInfo,
) -> Result<alloc::collections::BTreeSet<DataIdentifierCow<'_>>, DataError>;
}
#[cfg(feature = "alloc")]
impl<M, P> IterableDynamicDataProvider<M> for alloc::boxed::Box<P>
where
M: DynamicDataMarker,
P: IterableDynamicDataProvider<M> + ?Sized,
{
fn iter_ids_for_marker(
&self,
marker: DataMarkerInfo,
) -> Result<alloc::collections::BTreeSet<DataIdentifierCow<'_>>, DataError> {
(**self).iter_ids_for_marker(marker)
}
}
/// A data provider that loads data for a specific data type.
///
/// Unlike [`DataProvider`], the provider is bound to a specific marker ahead of time.
///
/// This crate provides [`DataProviderWithMarker`] which implements this trait on a single provider
/// with a single marker. However, this trait can also be implemented on providers that fork between
/// multiple markers that all return the same data type. For example, it can abstract over many
/// calendar systems in the datetime formatter.
pub trait BoundDataProvider<M>
where
M: DynamicDataMarker,
{
/// Query the provider for data, returning the result.
///
/// Returns [`Ok`] if the request successfully loaded data. If data failed to load, returns an
/// Error with more information.
fn load_bound(&self, req: DataRequest) -> Result<DataResponse<M>, DataError>;
/// Returns the [`DataMarkerInfo`] that this provider uses for loading data.
fn bound_marker(&self) -> DataMarkerInfo;
}
impl<M, P> BoundDataProvider<M> for &P
where
M: DynamicDataMarker,
P: BoundDataProvider<M> + ?Sized,
{
#[inline]
fn load_bound(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
(*self).load_bound(req)
}
#[inline]
fn bound_marker(&self) -> DataMarkerInfo {
(*self).bound_marker()
}
}
#[cfg(feature = "alloc")]
impl<M, P> BoundDataProvider<M> for alloc::boxed::Box<P>
where
M: DynamicDataMarker,
P: BoundDataProvider<M> + ?Sized,
{
#[inline]
fn load_bound(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
(**self).load_bound(req)
}
#[inline]
fn bound_marker(&self) -> DataMarkerInfo {
(**self).bound_marker()
}
}
#[cfg(feature = "alloc")]
impl<M, P> BoundDataProvider<M> for alloc::rc::Rc<P>
where
M: DynamicDataMarker,
P: BoundDataProvider<M> + ?Sized,
{
#[inline]
fn load_bound(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
(**self).load_bound(req)
}
#[inline]
fn bound_marker(&self) -> DataMarkerInfo {
(**self).bound_marker()
}
}
#[cfg(target_has_atomic = "ptr")]
#[cfg(feature = "alloc")]
impl<M, P> BoundDataProvider<M> for alloc::sync::Arc<P>
where
M: DynamicDataMarker,
P: BoundDataProvider<M> + ?Sized,
{
#[inline]
fn load_bound(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
(**self).load_bound(req)
}
#[inline]
fn bound_marker(&self) -> DataMarkerInfo {
(**self).bound_marker()
}
}
/// A [`DataProvider`] associated with a specific marker.
///
/// Implements [`BoundDataProvider`].
#[derive(Debug)]
pub struct DataProviderWithMarker<M, P> {
inner: P,
_marker: PhantomData<M>,
}
impl<M, P> DataProviderWithMarker<M, P>
where
M: DataMarker,
P: DataProvider<M>,
{
/// Creates a [`DataProviderWithMarker`] from a [`DataProvider`] with a [`DataMarker`].
pub const fn new(inner: P) -> Self {
Self {
inner,
_marker: PhantomData,
}
}
}
impl<M, M0, Y, P> BoundDataProvider<M0> for DataProviderWithMarker<M, P>
where
M: DataMarker<DataStruct = Y>,
M0: DynamicDataMarker<DataStruct = Y>,
Y: for<'a> Yokeable<'a>,
P: DataProvider<M>,
{
#[inline]
fn load_bound(&self, req: DataRequest) -> Result<DataResponse<M0>, DataError> {
self.inner.load(req).map(DataResponse::cast)
}
#[inline]
fn bound_marker(&self) -> DataMarkerInfo {
M::INFO
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::hello_world::*;
use alloc::borrow::Cow;
use alloc::string::String;
use core::fmt::Debug;
use serde::{Deserialize, Serialize};
// This tests DataProvider borrow semantics with a dummy data provider based on a
// JSON string. It also exercises most of the data provider code paths.
/// A data struct serialization-compatible with HelloWorld used for testing mismatched types
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, yoke::Yokeable)]
pub struct HelloAlt {
message: String,
}
data_marker!(HelloAltMarkerV1, HelloAlt);
#[derive(Deserialize, Debug, Clone, Default, PartialEq)]
struct HelloCombined<'data> {
#[serde(borrow)]
pub hello_v1: HelloWorld<'data>,
pub hello_alt: HelloAlt,
}
/// A DataProvider that owns its data, returning an Rc-variant DataPayload.
/// Supports only key::HELLO_WORLD_V1. Uses `impl_dynamic_data_provider!()`.
#[derive(Debug)]
struct DataWarehouse {
hello_v1: HelloWorld<'static>,
hello_alt: HelloAlt,
}
impl DataProvider<HelloWorldV1> for DataWarehouse {
fn load(&self, _: DataRequest) -> Result<DataResponse<HelloWorldV1>, DataError> {
Ok(DataResponse {
metadata: DataResponseMetadata::default(),
payload: DataPayload::from_owned(self.hello_v1.clone()),
})
}
}
/// A DataProvider that supports both key::HELLO_WORLD_V1 and HELLO_ALT.
#[derive(Debug)]
struct DataProvider2 {
data: DataWarehouse,
}
impl From<DataWarehouse> for DataProvider2 {
fn from(warehouse: DataWarehouse) -> Self {
DataProvider2 { data: warehouse }
}
}
impl DataProvider<HelloWorldV1> for DataProvider2 {
fn load(&self, _: DataRequest) -> Result<DataResponse<HelloWorldV1>, DataError> {
Ok(DataResponse {
metadata: DataResponseMetadata::default(),
payload: DataPayload::from_owned(self.data.hello_v1.clone()),
})
}
}
impl DataProvider<HelloAltMarkerV1> for DataProvider2 {
fn load(&self, _: DataRequest) -> Result<DataResponse<HelloAltMarkerV1>, DataError> {
Ok(DataResponse {
metadata: DataResponseMetadata::default(),
payload: DataPayload::from_owned(self.data.hello_alt.clone()),
})
}
}
const DATA: &str = r#"{
"hello_v1": {
"message": "Hello "
},
"hello_alt": {
"message": "Hello Alt"
}
}"#;
fn get_warehouse(data: &'static str) -> DataWarehouse {
let data: HelloCombined = serde_json::from_str(data).expect("Well-formed data");
DataWarehouse {
hello_v1: data.hello_v1,
hello_alt: data.hello_alt,
}
}
fn get_payload_v1<P: DataProvider<HelloWorldV1> + ?Sized>(
provider: &P,
) -> Result<DataPayload<HelloWorldV1>, DataError> {
provider.load(Default::default()).map(|r| r.payload)
}
fn get_payload_alt<P: DataProvider<HelloAltMarkerV1> + ?Sized>(
provider: &P,
) -> Result<DataPayload<HelloAltMarkerV1>, DataError> {
provider.load(Default::default()).map(|r| r.payload)
}
#[test]
fn test_warehouse_owned() {
let warehouse = get_warehouse(DATA);
let hello_data = get_payload_v1(&warehouse).unwrap();
assert!(matches!(
hello_data.get(),
HelloWorld {
message: Cow::Borrowed(_),
}
));
}
#[test]
fn test_warehouse_owned_dyn_generic() {
let warehouse = get_warehouse(DATA);
let hello_data = get_payload_v1(&warehouse as &dyn DataProvider<HelloWorldV1>).unwrap();
assert!(matches!(
hello_data.get(),
HelloWorld {
message: Cow::Borrowed(_),
}
));
}
#[test]
fn test_provider2() {
let warehouse = get_warehouse(DATA);
let provider = DataProvider2::from(warehouse);
let hello_data = get_payload_v1(&provider).unwrap();
assert!(matches!(
hello_data.get(),
HelloWorld {
message: Cow::Borrowed(_),
}
));
}
#[test]
fn test_provider2_dyn_generic() {
let warehouse = get_warehouse(DATA);
let provider = DataProvider2::from(warehouse);
let hello_data = get_payload_v1(&provider as &dyn DataProvider<HelloWorldV1>).unwrap();
assert!(matches!(
hello_data.get(),
HelloWorld {
message: Cow::Borrowed(_),
}
));
}
#[test]
fn test_provider2_dyn_generic_alt() {
let warehouse = get_warehouse(DATA);
let provider = DataProvider2::from(warehouse);
let hello_data = get_payload_alt(&provider as &dyn DataProvider<HelloAltMarkerV1>).unwrap();
assert!(matches!(hello_data.get(), HelloAlt { .. }));
}
fn check_v1_v2<P>(d: &P)
where
P: DataProvider<HelloWorldV1> + DataProvider<HelloAltMarkerV1> + ?Sized,
{
let v1: DataPayload<HelloWorldV1> = d.load(Default::default()).unwrap().payload;
let v2: DataPayload<HelloAltMarkerV1> = d.load(Default::default()).unwrap().payload;
if v1.get().message == v2.get().message {
panic!()
}
}
#[test]
fn test_v1_v2_generic() {
let warehouse = get_warehouse(DATA);
let provider = DataProvider2::from(warehouse);
check_v1_v2(&provider);
}
}

184
vendor/icu_provider/src/dynutil.rs vendored Normal file
View File

@@ -0,0 +1,184 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
//! Utilities for using trait objects with `DataPayload`.
/// Trait to allow conversion from `DataPayload<T>` to `DataPayload<S>`.
///
/// This trait can be manually implemented in order to enable [`impl_dynamic_data_provider`].
pub trait UpcastDataPayload<M>
where
M: crate::DynamicDataMarker,
Self: Sized + crate::DynamicDataMarker,
{
/// Upcast a `DataPayload<T>` to a `DataPayload<S>` where `T` implements trait `S`.
fn upcast(other: crate::DataPayload<M>) -> crate::DataPayload<Self>;
}
/// Implements [`UpcastDataPayload`] from several data markers to a single data marker
/// that all share the same [`DynamicDataMarker::DataStruct`].
///
/// # Examples
///
/// ```
/// use icu_provider::prelude::*;
/// use std::borrow::Cow;
///
/// struct FooV1;
/// impl DynamicDataMarker for FooV1 {
/// type DataStruct = Foo<'static>;
/// }
/// icu_provider::data_marker!(BarV1, Foo<'static>);
/// icu_provider::data_marker!(BazV1, Foo<'static>);
///
/// #[derive(yoke::Yokeable)]
/// pub struct Foo<'data> {
/// message: Cow<'data, str>,
/// };
///
/// icu_provider::data_struct!(Foo<'_>);
///
/// icu_provider::dynutil::impl_casting_upcast!(FooV1, [BarV1, BazV1,]);
/// ```
///
/// [`DynamicDataMarker::DataStruct`]: crate::DynamicDataMarker::DataStruct
#[macro_export]
#[doc(hidden)] // macro
macro_rules! __impl_casting_upcast {
($dyn_m:path, [ $($struct_m:ident),+, ]) => {
$(
impl $crate::dynutil::UpcastDataPayload<$struct_m> for $dyn_m {
fn upcast(other: $crate::DataPayload<$struct_m>) -> $crate::DataPayload<$dyn_m> {
other.cast()
}
}
)+
}
}
#[doc(inline)]
pub use __impl_casting_upcast as impl_casting_upcast;
/// Implements [`DynamicDataProvider`] for a marker type `S` on a type that already implements
/// [`DynamicDataProvider`] or [`DataProvider`] for one or more `M`, where `M` is a concrete type
/// that is convertible to `S` via [`UpcastDataPayload`].
///
/// ## Wrapping DataProvider
///
/// If your type implements [`DataProvider`], pass a list of markers as the second argument.
/// This results in a `DynamicDataProvider` that delegates to a specific marker if the marker
/// matches or else returns [`DataErrorKind::MarkerNotFound`].
///
/// [`DynamicDataProvider`]: crate::DynamicDataProvider
/// [`DataProvider`]: crate::DataProvider
/// [`DataErrorKind::MarkerNotFound`]: (crate::DataErrorKind::MarkerNotFound)
/// [`SerializeMarker`]: (crate::buf::SerializeMarker)
#[doc(hidden)] // macro
#[macro_export]
macro_rules! __impl_dynamic_data_provider {
// allow passing in multiple things to do and get dispatched
($provider:ty, $arms:tt, $one:path, $($rest:path),+) => {
$crate::dynutil::impl_dynamic_data_provider!(
$provider,
$arms,
$one
);
$crate::dynutil::impl_dynamic_data_provider!(
$provider,
$arms,
$($rest),+
);
};
($provider:ty, { $($ident:ident = $marker:path => $struct_m:ty),+, $(_ => $struct_d:ty,)?}, $dyn_m:ty) => {
impl $crate::DynamicDataProvider<$dyn_m> for $provider
{
fn load_data(
&self,
marker: $crate::DataMarkerInfo,
req: $crate::DataRequest,
) -> Result<
$crate::DataResponse<$dyn_m>,
$crate::DataError,
> {
match marker.id.hashed() {
$(
h if h == $marker.id.hashed() => {
let result: $crate::DataResponse<$struct_m> =
$crate::DynamicDataProvider::<$struct_m>::load_data(self, marker, req)?;
Ok($crate::DataResponse {
metadata: result.metadata,
payload: $crate::dynutil::UpcastDataPayload::<$struct_m>::upcast(result.payload),
})
}
)+,
$(
_ => {
let result: $crate::DataResponse<$struct_d> =
$crate::DynamicDataProvider::<$struct_d>::load_data(self, marker, req)?;
Ok($crate::DataResponse {
metadata: result.metadata,
payload: $crate::dynutil::UpcastDataPayload::<$struct_d>::upcast(result.payload),
})
}
)?
_ => Err($crate::DataErrorKind::MarkerNotFound.with_req(marker, req))
}
}
}
};
($provider:ty, [ $($(#[$cfg:meta])? $struct_m:ty),+, ], $dyn_m:path) => {
impl $crate::DynamicDataProvider<$dyn_m> for $provider
{
fn load_data(
&self,
marker: $crate::DataMarkerInfo,
req: $crate::DataRequest,
) -> Result<
$crate::DataResponse<$dyn_m>,
$crate::DataError,
> {
match marker.id.hashed() {
$(
$(#[$cfg])?
h if h == <$struct_m as $crate::DataMarker>::INFO.id.hashed() => {
let result: $crate::DataResponse<$struct_m> =
$crate::DataProvider::load(self, req)?;
Ok($crate::DataResponse {
metadata: result.metadata,
payload: $crate::dynutil::UpcastDataPayload::<$struct_m>::upcast(result.payload),
})
}
)+,
_ => Err($crate::DataErrorKind::MarkerNotFound.with_req(marker, req))
}
}
}
};
}
#[doc(inline)]
pub use __impl_dynamic_data_provider as impl_dynamic_data_provider;
#[doc(hidden)] // macro
#[macro_export]
macro_rules! __impl_iterable_dynamic_data_provider {
($provider:ty, [ $($(#[$cfg:meta])? $struct_m:ty),+, ], $dyn_m:path) => {
impl $crate::IterableDynamicDataProvider<$dyn_m> for $provider {
fn iter_ids_for_marker(&self, marker: $crate::DataMarkerInfo) -> Result<alloc::collections::BTreeSet<$crate::DataIdentifierCow<'_>>, $crate::DataError> {
match marker.id.hashed() {
$(
$(#[$cfg])?
h if h == <$struct_m as $crate::DataMarker>::INFO.id.hashed() => {
$crate::IterableDataProvider::<$struct_m>::iter_ids(self)
}
)+,
_ => Err($crate::DataErrorKind::MarkerNotFound.with_marker(marker))
}
}
}
}
}
#[doc(inline)]
pub use __impl_iterable_dynamic_data_provider as impl_iterable_dynamic_data_provider;

281
vendor/icu_provider/src/error.rs vendored Normal file
View File

@@ -0,0 +1,281 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
use crate::log;
use crate::{marker::DataMarkerId, prelude::*};
use core::fmt;
use displaydoc::Display;
/// A list specifying general categories of data provider error.
///
/// Errors may be caused either by a malformed request or by the data provider
/// not being able to fulfill a well-formed request.
#[derive(Clone, Copy, Eq, PartialEq, Display, Debug)]
#[non_exhaustive]
pub enum DataErrorKind {
/// No data for the requested data marker. This is only returned by [`DynamicDataProvider`].
#[displaydoc("Missing data for marker")]
MarkerNotFound,
/// There is data for the data marker, but not for this particular data identifier.
#[displaydoc("Missing data for identifier")]
IdentifierNotFound,
/// The request is invalid, such as a request for a singleton marker containing a data identifier.
#[displaydoc("Invalid request")]
InvalidRequest,
/// The data for two [`DataMarker`]s is not consistent.
#[displaydoc("The data for two markers is not consistent: {0:?} (were they generated in different datagen invocations?)")]
InconsistentData(DataMarkerInfo),
/// An error occured during [`Any`](core::any::Any) downcasting.
#[displaydoc("Downcast: expected {0}, found")]
Downcast(&'static str),
/// An error occured during [`serde`] deserialization.
///
/// Check debug logs for potentially more information.
#[displaydoc("Deserialize")]
Deserialize,
/// An unspecified error occurred.
///
/// Check debug logs for potentially more information.
#[displaydoc("Custom")]
Custom,
/// An error occurred while accessing a system resource.
#[displaydoc("I/O: {0:?}")]
#[cfg(feature = "std")]
Io(std::io::ErrorKind),
}
/// The error type for ICU4X data provider operations.
///
/// To create one of these, either start with a [`DataErrorKind`] or use [`DataError::custom()`].
///
/// # Example
///
/// Create a IdentifierNotFound error and attach a data request for context:
///
/// ```no_run
/// # use icu_provider::prelude::*;
/// let marker: DataMarkerInfo = unimplemented!();
/// let req: DataRequest = unimplemented!();
/// DataErrorKind::IdentifierNotFound.with_req(marker, req);
/// ```
///
/// Create a named custom error:
///
/// ```
/// # use icu_provider::prelude::*;
/// DataError::custom("This is an example error");
/// ```
#[derive(Clone, Copy, Eq, PartialEq, Debug)]
#[non_exhaustive]
pub struct DataError {
/// Broad category of the error.
pub kind: DataErrorKind,
/// The data marker of the request, if available.
pub marker: Option<DataMarkerId>,
/// Additional context, if available.
pub str_context: Option<&'static str>,
/// Whether this error was created in silent mode to not log.
pub silent: bool,
}
impl fmt::Display for DataError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "ICU4X data error")?;
if self.kind != DataErrorKind::Custom {
write!(f, ": {}", self.kind)?;
}
if let Some(marker) = self.marker {
write!(f, " (marker: {marker:?})")?;
}
if let Some(str_context) = self.str_context {
write!(f, ": {str_context}")?;
}
Ok(())
}
}
impl DataErrorKind {
/// Converts this DataErrorKind into a DataError.
///
/// If possible, you should attach context using a `with_` function.
#[inline]
pub const fn into_error(self) -> DataError {
DataError {
kind: self,
marker: None,
str_context: None,
silent: false,
}
}
/// Creates a DataError with a data marker context.
#[inline]
pub const fn with_marker(self, marker: DataMarkerInfo) -> DataError {
self.into_error().with_marker(marker)
}
/// Creates a DataError with a string context.
#[inline]
pub const fn with_str_context(self, context: &'static str) -> DataError {
self.into_error().with_str_context(context)
}
/// Creates a DataError with a type name context.
#[inline]
pub fn with_type_context<T>(self) -> DataError {
self.into_error().with_type_context::<T>()
}
/// Creates a DataError with a request context.
#[inline]
pub fn with_req(self, marker: DataMarkerInfo, req: DataRequest) -> DataError {
self.into_error().with_req(marker, req)
}
}
impl DataError {
/// Returns a new, empty DataError with kind Custom and a string error message.
#[inline]
pub const fn custom(str_context: &'static str) -> Self {
Self {
kind: DataErrorKind::Custom,
marker: None,
str_context: Some(str_context),
silent: false,
}
}
/// Sets the data marker of a DataError, returning a modified error.
#[inline]
pub const fn with_marker(self, marker: DataMarkerInfo) -> Self {
Self {
kind: self.kind,
marker: Some(marker.id),
str_context: self.str_context,
silent: self.silent,
}
}
/// Sets the string context of a DataError, returning a modified error.
#[inline]
pub const fn with_str_context(self, context: &'static str) -> Self {
Self {
kind: self.kind,
marker: self.marker,
str_context: Some(context),
silent: self.silent,
}
}
/// Sets the string context of a DataError to the given type name, returning a modified error.
#[inline]
pub fn with_type_context<T>(self) -> Self {
if !self.silent {
log::warn!("{self}: Type context: {}", core::any::type_name::<T>());
}
self.with_str_context(core::any::type_name::<T>())
}
/// Logs the data error with the given request, returning an error containing the data marker.
///
/// If the "logging" Cargo feature is enabled, this logs the whole request. Either way,
/// it returns an error with the data marker portion of the request as context.
pub fn with_req(mut self, marker: DataMarkerInfo, req: DataRequest) -> Self {
if req.metadata.silent {
self.silent = true;
}
// Don't write out a log for MissingDataMarker since there is no context to add
if !self.silent && self.kind != DataErrorKind::MarkerNotFound {
log::warn!("{self} (marker: {marker:?}, request: {})", req.id);
}
self.with_marker(marker)
}
/// Logs the data error with the given context, then return self.
///
/// This does not modify the error, but if the "logging" Cargo feature is enabled,
/// it will print out the context.
#[cfg(feature = "std")]
pub fn with_path_context(self, _path: &std::path::Path) -> Self {
if !self.silent {
log::warn!("{self} (path: {_path:?})");
}
self
}
/// Logs the data error with the given context, then return self.
///
/// This does not modify the error, but if the "logging" Cargo feature is enabled,
/// it will print out the context.
#[cfg_attr(not(feature = "logging"), allow(unused_variables))]
#[inline]
pub fn with_display_context<D: fmt::Display + ?Sized>(self, context: &D) -> Self {
if !self.silent {
log::warn!("{self}: {context}");
}
self
}
/// Logs the data error with the given context, then return self.
///
/// This does not modify the error, but if the "logging" Cargo feature is enabled,
/// it will print out the context.
#[cfg_attr(not(feature = "logging"), allow(unused_variables))]
#[inline]
pub fn with_debug_context<D: fmt::Debug + ?Sized>(self, context: &D) -> Self {
if !self.silent {
log::warn!("{self}: {context:?}");
}
self
}
#[inline]
pub(crate) fn for_type<T>() -> DataError {
DataError {
kind: DataErrorKind::Downcast(core::any::type_name::<T>()),
marker: None,
str_context: None,
silent: false,
}
}
}
impl core::error::Error for DataError {}
#[cfg(feature = "std")]
impl From<std::io::Error> for DataError {
fn from(e: std::io::Error) -> Self {
log::warn!("I/O error: {e}");
DataErrorKind::Io(e.kind()).into_error()
}
}
/// Extension trait for `Result<T, DataError>`.
pub trait ResultDataError<T>: Sized {
/// Propagates all errors other than [`DataErrorKind::IdentifierNotFound`], and returns `None` in that case.
fn allow_identifier_not_found(self) -> Result<Option<T>, DataError>;
}
impl<T> ResultDataError<T> for Result<T, DataError> {
fn allow_identifier_not_found(self) -> Result<Option<T>, DataError> {
match self {
Ok(t) => Ok(Some(t)),
Err(DataError {
kind: DataErrorKind::IdentifierNotFound,
..
}) => Ok(None),
Err(e) => Err(e),
}
}
}

213
vendor/icu_provider/src/export/mod.rs vendored Normal file
View File

@@ -0,0 +1,213 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
//! This module contains types required to export ICU4X data via the `icu_provider_export` crate.
//! End users should not need to consume anything in this module.
//!
//! This module is enabled with the `export` Cargo feature.
mod payload;
#[doc(hidden)] // macro
pub use payload::ExportBox;
pub use payload::ExportMarker;
use crate::prelude::*;
use alloc::collections::BTreeSet;
/// An object capable of exporting data payloads in some form.
pub trait DataExporter: Sync {
/// Save a `payload` corresponding to the given marker and locale.
///
/// Takes non-mut self as it can be called concurrently.
fn put_payload(
&self,
marker: DataMarkerInfo,
id: DataIdentifierBorrowed,
payload: &DataPayload<ExportMarker>,
) -> Result<(), DataError>;
/// Function called for singleton markers.
///
/// Takes non-mut self as it can be called concurrently.
fn flush_singleton(
&self,
marker: DataMarkerInfo,
payload: &DataPayload<ExportMarker>,
metadata: FlushMetadata,
) -> Result<(), DataError> {
self.put_payload(marker, Default::default(), payload)?;
self.flush(marker, metadata)
}
/// Function called after a non-singleton marker has been fully enumerated.
///
/// Takes non-mut self as it can be called concurrently.
fn flush(&self, _marker: DataMarkerInfo, _metadata: FlushMetadata) -> Result<(), DataError> {
Ok(())
}
/// This function has to be called before the object is dropped (after all
/// markers have been fully dumped). This conceptually takes ownership, so
/// clients *may not* interact with this object after close has been called.
fn close(&mut self) -> Result<ExporterCloseMetadata, DataError> {
Ok(ExporterCloseMetadata::default())
}
}
#[derive(Debug, Default)]
#[allow(clippy::exhaustive_structs)] // newtype
/// Contains information about a successful export.
pub struct ExporterCloseMetadata(pub Option<Box<dyn core::any::Any>>);
/// Metadata for [`DataExporter::flush`]
#[non_exhaustive]
#[derive(Debug, Copy, Clone, Default)]
pub struct FlushMetadata {
/// Whether the data was generated in such a way that a [`DryDataProvider`] implementation
/// makes sense.
pub supports_dry_provider: bool,
/// The checksum to return with this data marker.
pub checksum: Option<u64>,
}
impl DataExporter for Box<dyn DataExporter> {
fn put_payload(
&self,
marker: DataMarkerInfo,
id: DataIdentifierBorrowed,
payload: &DataPayload<ExportMarker>,
) -> Result<(), DataError> {
(**self).put_payload(marker, id, payload)
}
fn flush_singleton(
&self,
marker: DataMarkerInfo,
payload: &DataPayload<ExportMarker>,
metadata: FlushMetadata,
) -> Result<(), DataError> {
(**self).flush_singleton(marker, payload, metadata)
}
fn flush(&self, marker: DataMarkerInfo, metadata: FlushMetadata) -> Result<(), DataError> {
(**self).flush(marker, metadata)
}
fn close(&mut self) -> Result<ExporterCloseMetadata, DataError> {
(**self).close()
}
}
/// A [`DynamicDataProvider`] that can be used for exporting data.
///
/// Use [`make_exportable_provider`] to implement this.
pub trait ExportableProvider:
crate::data_provider::IterableDynamicDataProvider<ExportMarker> + Sync
{
/// Returns the set of supported markers
fn supported_markers(&self) -> BTreeSet<DataMarkerInfo>;
}
impl ExportableProvider for Box<dyn ExportableProvider> {
fn supported_markers(&self) -> BTreeSet<DataMarkerInfo> {
(**self).supported_markers()
}
}
/// This macro can be used on a data provider to allow it to be exported by `ExportDriver`.
///
/// Data generation 'compiles' data by using this data provider (which usually translates data from
/// different sources and doesn't have to be efficient) to generate data structs, and then writing
/// them to an efficient format like `BlobDataProvider` or `BakedDataProvider`. The requirements
/// for `make_exportable_provider` are:
/// * The data struct has to implement [`serde::Serialize`](::serde::Serialize) and [`databake::Bake`]
/// * The provider needs to implement [`IterableDataProvider`] for all specified [`DataMarker`]s.
/// This allows the generating code to know which [`DataIdentifierCow`]s to export.
#[macro_export]
#[doc(hidden)] // macro
macro_rules! __make_exportable_provider {
($provider:ty, [ $($(#[$cfg:meta])? $struct_m:ty),+, ]) => {
impl $crate::export::ExportableProvider for $provider {
fn supported_markers(&self) -> alloc::collections::BTreeSet<$crate::DataMarkerInfo> {
alloc::collections::BTreeSet::from_iter([
$(
$(#[$cfg])?
<$struct_m>::INFO,
)+
])
}
}
$crate::dynutil::impl_dynamic_data_provider!(
$provider,
[ $($(#[$cfg])? $struct_m),+, ],
$crate::export::ExportMarker
);
$crate::dynutil::impl_iterable_dynamic_data_provider!(
$provider,
[ $($(#[$cfg])? $struct_m),+, ],
$crate::export::ExportMarker
);
};
}
#[doc(inline)]
pub use __make_exportable_provider as make_exportable_provider;
/// A `DataExporter` that forks to multiple `DataExporter`s.
#[derive(Default)]
pub struct MultiExporter(Vec<Box<dyn DataExporter>>);
impl MultiExporter {
/// Creates a `MultiExporter` for the given exporters.
pub const fn new(exporters: Vec<Box<dyn DataExporter>>) -> Self {
Self(exporters)
}
}
impl core::fmt::Debug for MultiExporter {
fn fmt(&self, f: &mut alloc::fmt::Formatter<'_>) -> alloc::fmt::Result {
f.debug_struct("MultiExporter")
.field("0", &format!("vec[len = {}]", self.0.len()))
.finish()
}
}
impl DataExporter for MultiExporter {
fn put_payload(
&self,
marker: DataMarkerInfo,
id: DataIdentifierBorrowed,
payload: &DataPayload<ExportMarker>,
) -> Result<(), DataError> {
self.0
.iter()
.try_for_each(|e| e.put_payload(marker, id, payload))
}
fn flush_singleton(
&self,
marker: DataMarkerInfo,
payload: &DataPayload<ExportMarker>,
metadata: FlushMetadata,
) -> Result<(), DataError> {
self.0
.iter()
.try_for_each(|e| e.flush_singleton(marker, payload, metadata))
}
fn flush(&self, marker: DataMarkerInfo, metadata: FlushMetadata) -> Result<(), DataError> {
self.0.iter().try_for_each(|e| e.flush(marker, metadata))
}
fn close(&mut self) -> Result<ExporterCloseMetadata, DataError> {
Ok(ExporterCloseMetadata(Some(Box::new(
self.0.iter_mut().try_fold(vec![], |mut m, e| {
m.push(e.close()?.0);
Ok::<_, DataError>(m)
})?,
))))
}
}

View File

@@ -0,0 +1,335 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
use core::any::Any;
use crate::prelude::*;
use crate::ule::MaybeEncodeAsVarULE;
use crate::{dynutil::UpcastDataPayload, ule::MaybeAsVarULE};
use alloc::sync::Arc;
use databake::{Bake, BakeSize, CrateEnv, TokenStream};
use yoke::*;
use zerovec::VarZeroVec;
#[cfg(doc)]
use zerovec::ule::VarULE;
trait ExportableDataPayload {
fn bake_yoke(&self, ctx: &CrateEnv) -> TokenStream;
fn bake_size(&self) -> usize;
fn serialize_yoke(
&self,
serializer: &mut dyn erased_serde::Serializer,
) -> Result<(), DataError>;
fn maybe_bake_varule_encoded(
&self,
rest: &[&DataPayload<ExportMarker>],
ctx: &CrateEnv,
) -> Option<TokenStream>;
fn as_any(&self) -> &dyn Any;
fn eq_dyn(&self, other: &dyn ExportableDataPayload) -> bool;
}
impl<M: DynamicDataMarker> ExportableDataPayload for DataPayload<M>
where
for<'a> <M::DataStruct as Yokeable<'a>>::Output:
Bake + BakeSize + serde::Serialize + MaybeEncodeAsVarULE + PartialEq,
{
fn bake_yoke(&self, ctx: &CrateEnv) -> TokenStream {
self.get().bake(ctx)
}
fn bake_size(&self) -> usize {
core::mem::size_of::<<M::DataStruct as Yokeable>::Output>() + self.get().borrows_size()
}
fn serialize_yoke(
&self,
serializer: &mut dyn erased_serde::Serializer,
) -> Result<(), DataError> {
use erased_serde::Serialize;
self.get()
.erased_serialize(serializer)
.map_err(|e| DataError::custom("Serde export").with_display_context(&e))?;
Ok(())
}
fn maybe_bake_varule_encoded(
&self,
rest: &[&DataPayload<ExportMarker>],
ctx: &CrateEnv,
) -> Option<TokenStream> {
let first_varule = self.get().maybe_encode_as_varule()?;
let recovered_vec: Vec<
&<<M::DataStruct as Yokeable<'_>>::Output as MaybeAsVarULE>::EncodedStruct,
> = core::iter::once(first_varule)
.chain(rest.iter().map(|v| {
#[expect(clippy::expect_used)] // exporter code
v.get()
.payload
.as_any()
.downcast_ref::<Self>()
.expect("payloads expected to be same type")
.get()
.maybe_encode_as_varule()
.expect("MaybeEncodeAsVarULE impl should be symmetric")
}))
.collect();
let vzv: VarZeroVec<
<<M::DataStruct as Yokeable<'_>>::Output as MaybeAsVarULE>::EncodedStruct,
> = VarZeroVec::from(&recovered_vec);
let vzs = vzv.as_slice();
Some(vzs.bake(ctx))
}
fn as_any(&self) -> &dyn Any {
self
}
fn eq_dyn(&self, other: &dyn ExportableDataPayload) -> bool {
match other.as_any().downcast_ref::<Self>() {
Some(downcasted) => (*self).eq(downcasted),
None => {
debug_assert!(
false,
"cannot compare ExportableDataPayloads of different types: self is {:?} but other is {:?}",
self.type_id(),
other.as_any().type_id(),
);
false
}
}
}
}
#[derive(yoke::Yokeable, Clone)]
#[allow(missing_docs)]
pub struct ExportBox {
payload: Arc<dyn ExportableDataPayload + Sync + Send>,
}
impl PartialEq for ExportBox {
fn eq(&self, other: &Self) -> bool {
self.payload.eq_dyn(&*other.payload)
}
}
impl Eq for ExportBox {}
impl core::fmt::Debug for ExportBox {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("ExportBox")
.field("payload", &"<payload>")
.finish()
}
}
impl<M> UpcastDataPayload<M> for ExportMarker
where
M: DynamicDataMarker,
M::DataStruct: Sync + Send,
for<'a> <M::DataStruct as Yokeable<'a>>::Output:
Bake + BakeSize + serde::Serialize + MaybeEncodeAsVarULE + PartialEq,
{
fn upcast(other: DataPayload<M>) -> DataPayload<ExportMarker> {
DataPayload::from_owned(ExportBox {
payload: Arc::new(other),
})
}
}
impl DataPayload<ExportMarker> {
/// Serializes this [`DataPayload`] into a serializer using Serde.
///
/// # Examples
///
/// ```
/// use icu_provider::dynutil::UpcastDataPayload;
/// use icu_provider::export::*;
/// use icu_provider::hello_world::HelloWorldV1;
/// use icu_provider::prelude::*;
///
/// // Create an example DataPayload
/// let payload: DataPayload<HelloWorldV1> = Default::default();
/// let export: DataPayload<ExportMarker> = UpcastDataPayload::upcast(payload);
///
/// // Serialize the payload to a JSON string
/// let mut buffer: Vec<u8> = vec![];
/// export
/// .serialize(&mut serde_json::Serializer::new(&mut buffer))
/// .expect("Serialization should succeed");
/// assert_eq!(r#"{"message":"(und) Hello World"}"#.as_bytes(), buffer);
/// ```
pub fn serialize<S>(&self, serializer: S) -> Result<(), DataError>
where
S: serde::Serializer,
S::Ok: 'static, // erased_serde requirement, cannot return values in `Ok`
{
self.get()
.payload
.serialize_yoke(&mut <dyn erased_serde::Serializer>::erase(serializer))
}
/// Serializes this [`DataPayload`]'s value into a [`TokenStream`]
/// using its [`Bake`] implementations.
///
/// # Examples
///
/// ```
/// use icu_provider::dynutil::UpcastDataPayload;
/// use icu_provider::export::*;
/// use icu_provider::hello_world::HelloWorldV1;
/// use icu_provider::prelude::*;
/// # use databake::quote;
/// # use std::collections::BTreeSet;
///
/// // Create an example DataPayload
/// let payload: DataPayload<HelloWorldV1> = Default::default();
/// let export: DataPayload<ExportMarker> = UpcastDataPayload::upcast(payload);
///
/// let env = databake::CrateEnv::default();
/// let tokens = export.tokenize(&env);
/// assert_eq!(
/// quote! {
/// icu_provider::hello_world::HelloWorld {
/// message: alloc::borrow::Cow::Borrowed("(und) Hello World"),
/// }
/// }
/// .to_string(),
/// tokens.to_string()
/// );
/// assert_eq!(
/// env.into_iter().collect::<BTreeSet<_>>(),
/// ["icu_provider", "alloc"]
/// .into_iter()
/// .collect::<BTreeSet<_>>()
/// );
/// ```
pub fn tokenize(&self, ctx: &CrateEnv) -> TokenStream {
self.get().payload.bake_yoke(ctx)
}
/// If this payload's struct can be dereferenced as a [`VarULE`],
/// returns a [`TokenStream`] of the slice encoded as a [`VarZeroVec`].
pub fn tokenize_encoded_seq(structs: &[&Self], ctx: &CrateEnv) -> Option<TokenStream> {
let (first, rest) = structs.split_first()?;
first.get().payload.maybe_bake_varule_encoded(rest, ctx)
}
/// Returns the data size using postcard encoding
pub fn postcard_size(&self) -> usize {
use postcard::ser_flavors::{Flavor, Size};
let mut serializer = postcard::Serializer {
output: Size::default(),
};
let _infallible = self
.get()
.payload
.serialize_yoke(&mut <dyn erased_serde::Serializer>::erase(&mut serializer));
serializer.output.finalize().unwrap_or_default()
}
/// Returns an estimate of the baked size, made up of the size of the struct itself,
/// as well as the sizes of all its static borrows.
///
/// As static borrows are deduplicated by the linker, this is often overcounting.
pub fn baked_size(&self) -> usize {
self.get().payload.bake_size()
}
}
impl core::hash::Hash for DataPayload<ExportMarker> {
fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
self.hash_and_postcard_size(state);
}
}
impl DataPayload<ExportMarker> {
/// Calculates a payload hash and the postcard size
pub fn hash_and_postcard_size<H: core::hash::Hasher>(&self, state: &mut H) -> usize {
use postcard::ser_flavors::Flavor;
struct HashFlavor<'a, H>(&'a mut H, usize);
impl<H: core::hash::Hasher> Flavor for HashFlavor<'_, H> {
type Output = usize;
fn try_push(&mut self, data: u8) -> postcard::Result<()> {
self.0.write_u8(data);
self.1 += 1;
Ok(())
}
fn finalize(self) -> postcard::Result<Self::Output> {
Ok(self.1)
}
}
let mut serializer = postcard::Serializer {
output: HashFlavor(state, 0),
};
let _infallible = self
.get()
.payload
.serialize_yoke(&mut <dyn erased_serde::Serializer>::erase(&mut serializer));
serializer.output.1
}
}
/// Marker type for [`ExportBox`].
#[allow(clippy::exhaustive_structs)] // marker type
#[derive(Debug)]
pub struct ExportMarker {}
impl DynamicDataMarker for ExportMarker {
type DataStruct = ExportBox;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::hello_world::*;
#[test]
fn test_compare_with_dyn() {
let payload1: DataPayload<HelloWorldV1> = DataPayload::from_owned(HelloWorld {
message: "abc".into(),
});
let payload2: DataPayload<HelloWorldV1> = DataPayload::from_owned(HelloWorld {
message: "abc".into(),
});
let payload3: DataPayload<HelloWorldV1> = DataPayload::from_owned(HelloWorld {
message: "def".into(),
});
assert!(payload1.eq_dyn(&payload2));
assert!(payload2.eq_dyn(&payload1));
assert!(!payload1.eq_dyn(&payload3));
assert!(!payload3.eq_dyn(&payload1));
}
#[test]
fn test_export_marker_partial_eq() {
let payload1: DataPayload<ExportMarker> =
UpcastDataPayload::upcast(DataPayload::<HelloWorldV1>::from_owned(HelloWorld {
message: "abc".into(),
}));
let payload2: DataPayload<ExportMarker> =
UpcastDataPayload::upcast(DataPayload::<HelloWorldV1>::from_owned(HelloWorld {
message: "abc".into(),
}));
let payload3: DataPayload<ExportMarker> =
UpcastDataPayload::upcast(DataPayload::<HelloWorldV1>::from_owned(HelloWorld {
message: "def".into(),
}));
assert_eq!(payload1, payload2);
assert_eq!(payload2, payload1);
assert_ne!(payload1, payload3);
assert_ne!(payload3, payload1);
}
}

123
vendor/icu_provider/src/fallback.rs vendored Normal file
View File

@@ -0,0 +1,123 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
//! Options to define fallback behaviour.
//!
//! These options are consumed by the `LocaleFallbacker` in the `icu_locales` crate
//! (or the `icu::locales` module), but are defined here because they are used by `DataMarkerInfo`.
/// Hint for which subtag to prioritize during fallback.
///
/// For example, `"en-US"` might fall back to either `"en"` or `"und-US"` depending
/// on this enum.
#[derive(Debug, PartialEq, Eq, Copy, Clone, PartialOrd, Ord)]
#[non_exhaustive]
pub enum LocaleFallbackPriority {
/// Prioritize the language. This is the default behavior.
///
/// For example, `"en-US"` should go to `"en"` and then `"und"`.
Language,
/// Prioritize the script.
///
/// For example, `"en-US"` should go to `"en"` and then `"und-Latn"` and then `"und"`.
Script,
/// Prioritize the region.
///
/// For example, `"en-US"` should go to `"und-US"` and then `"und"`.
Region,
}
impl LocaleFallbackPriority {
/// Const-friendly version of [`Default::default`].
pub const fn default() -> Self {
Self::Language
}
}
impl Default for LocaleFallbackPriority {
fn default() -> Self {
Self::default()
}
}
/// Configuration settings for a particular fallback operation.
#[derive(Debug, Clone, PartialEq, Eq, Copy)]
#[non_exhaustive]
pub struct LocaleFallbackConfig {
/// Strategy for choosing which subtags to drop during locale fallback.
///
/// # Examples
///
/// Retain the language and script subtags until the final step:
///
/// ```
/// use icu::locale::fallback::LocaleFallbackConfig;
/// use icu::locale::fallback::LocaleFallbackPriority;
/// use icu::locale::locale;
/// use icu::locale::LocaleFallbacker;
///
/// // Set up the fallback iterator.
/// let fallbacker = LocaleFallbacker::new();
/// let mut config = LocaleFallbackConfig::default();
/// config.priority = LocaleFallbackPriority::Language;
/// let mut fallback_iterator = fallbacker
/// .for_config(config)
/// .fallback_for(locale!("ca-ES-valencia").into());
///
/// // Run the algorithm and check the results.
/// assert_eq!(fallback_iterator.get(), &locale!("ca-ES-valencia").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("ca-ES").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("ca-valencia").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("ca").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("und").into());
/// ```
///
/// Retain the region subtag until the final step:
///
/// ```
/// use icu::locale::fallback::LocaleFallbackConfig;
/// use icu::locale::fallback::LocaleFallbackPriority;
/// use icu::locale::locale;
/// use icu::locale::LocaleFallbacker;
///
/// // Set up the fallback iterator.
/// let fallbacker = LocaleFallbacker::new();
/// let mut config = LocaleFallbackConfig::default();
/// config.priority = LocaleFallbackPriority::Region;
/// let mut fallback_iterator = fallbacker
/// .for_config(config)
/// .fallback_for(locale!("ca-ES-valencia").into());
///
/// // Run the algorithm and check the results.
/// assert_eq!(fallback_iterator.get(), &locale!("ca-ES-valencia").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("ca-ES").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("und-ES-valencia").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("und-ES").into());
/// fallback_iterator.step();
/// assert_eq!(fallback_iterator.get(), &locale!("und").into());
/// ```
pub priority: LocaleFallbackPriority,
}
impl LocaleFallbackConfig {
/// Const version of [`Default::default`].
pub const fn default() -> Self {
Self {
priority: LocaleFallbackPriority::default(),
}
}
}
impl Default for LocaleFallbackConfig {
fn default() -> Self {
Self::default()
}
}

400
vendor/icu_provider/src/hello_world.rs vendored Normal file
View File

@@ -0,0 +1,400 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
//! Data provider returning multilingual "Hello World" strings for testing.
#![allow(clippy::exhaustive_structs)] // data struct module
use crate as icu_provider;
use crate::prelude::*;
use alloc::borrow::Cow;
use alloc::collections::BTreeSet;
use alloc::string::String;
use core::fmt::Debug;
use icu_locale_core::preferences::define_preferences;
use writeable::Writeable;
use yoke::*;
use zerofrom::*;
/// A struct containing "Hello World" in the requested language.
#[derive(Debug, PartialEq, Clone, Yokeable, ZeroFrom)]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
#[cfg_attr(
any(feature = "deserialize_json", feature = "export"),
derive(serde::Serialize)
)]
#[cfg_attr(feature = "export", derive(databake::Bake))]
#[cfg_attr(feature = "export", databake(path = icu_provider::hello_world))]
pub struct HelloWorld<'data> {
/// The translation of "Hello World".
#[cfg_attr(feature = "serde", serde(borrow))]
pub message: Cow<'data, str>,
}
impl Default for HelloWorld<'_> {
fn default() -> Self {
HelloWorld {
message: Cow::Borrowed("(und) Hello World"),
}
}
}
impl<'a> ZeroFrom<'a, str> for HelloWorld<'a> {
fn zero_from(message: &'a str) -> Self {
HelloWorld {
message: Cow::Borrowed(message),
}
}
}
crate::data_struct!(
HelloWorld<'data>,
varule: str,
#[cfg(feature = "export")]
encode_as_varule: |v: &HelloWorld<'_>| &*v.message
);
data_marker!(
/// Marker type for [`HelloWorld`].
#[derive(Debug)]
HelloWorldV1,
HelloWorld<'static>,
has_checksum = true,
);
/// A data provider returning Hello World strings in different languages.
///
/// Mostly useful for testing.
///
/// # Examples
///
/// ```
/// use icu_locale_core::langid;
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
///
/// let german_hello_world: DataResponse<HelloWorldV1> = HelloWorldProvider
/// .load(DataRequest {
/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()),
/// ..Default::default()
/// })
/// .expect("Loading should succeed");
///
/// assert_eq!("Hallo Welt", german_hello_world.payload.get().message);
/// ```
///
/// Load the reverse string using an auxiliary key:
///
/// ```
/// use icu_locale_core::langid;
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
///
/// let reverse_hello_world: DataResponse<HelloWorldV1> = HelloWorldProvider
/// .load(DataRequest {
/// id: DataIdentifierBorrowed::for_marker_attributes_and_locale(
/// DataMarkerAttributes::from_str_or_panic("reverse"),
/// &langid!("en").into(),
/// ),
/// ..Default::default()
/// })
/// .expect("Loading should succeed");
///
/// assert_eq!("Olleh Dlrow", reverse_hello_world.payload.get().message);
/// ```
#[derive(Debug, PartialEq, Default)]
pub struct HelloWorldProvider;
impl HelloWorldProvider {
// Data from https://en.wiktionary.org/wiki/Hello_World#Translations
// Keep this sorted!
const DATA: &'static [(&'static str, &'static str, &'static str)] = &[
("bn", "", "ওহে বিশ্ব"),
("cs", "", "Ahoj světe"),
("de", "", "Hallo Welt"),
("de-AT", "", "Servus Welt"),
("el", "", "Καλημέρα κόσμε"),
("en", "", "Hello World"),
// WORLD
("en-001", "", "Hello from 🗺️"),
// AFRICA
("en-002", "", "Hello from 🌍"),
// AMERICAS
("en-019", "", "Hello from 🌎"),
// ASIA
("en-142", "", "Hello from 🌏"),
// GREAT BRITAIN
("en-GB", "", "Hello from 🇬🇧"),
// ENGLAND
("en-GB-u-sd-gbeng", "", "Hello from 🏴󠁧󠁢󠁥󠁮󠁧󠁿"),
("en", "reverse", "Olleh Dlrow"),
("eo", "", "Saluton, Mondo"),
("fa", "", "سلام دنیا‎"),
("fi", "", "hei maailma"),
("is", "", "Halló, heimur"),
("ja", "", "こんにちは世界"),
("ja", "reverse", "界世はちにんこ"),
("la", "", "Ave, munde"),
("pt", "", "Olá, mundo"),
("ro", "", "Salut, lume"),
("ru", "", "Привет, мир"),
("sr", "", "Поздрав свете"),
("sr-Latn", "", "Pozdrav svete"),
("vi", "", "Xin chào thế giới"),
("zh", "", "你好世界"),
];
/// Converts this provider into a [`BufferProvider`] that uses JSON serialization.
#[cfg(feature = "deserialize_json")]
pub fn into_json_provider(self) -> HelloWorldJsonProvider {
HelloWorldJsonProvider
}
}
impl DataProvider<HelloWorldV1> for HelloWorldProvider {
fn load(&self, req: DataRequest) -> Result<DataResponse<HelloWorldV1>, DataError> {
let data = Self::DATA
.iter()
.find(|(l, a, _)| {
req.id.locale.strict_cmp(l.as_bytes()).is_eq()
&& *a == req.id.marker_attributes.as_str()
})
.map(|(_, _, v)| v)
.ok_or_else(|| DataErrorKind::IdentifierNotFound.with_req(HelloWorldV1::INFO, req))?;
Ok(DataResponse {
metadata: DataResponseMetadata::default().with_checksum(1234),
payload: DataPayload::from_static_str(data),
})
}
}
impl DryDataProvider<HelloWorldV1> for HelloWorldProvider {
fn dry_load(&self, req: DataRequest) -> Result<DataResponseMetadata, DataError> {
self.load(req).map(|r| r.metadata)
}
}
impl DataPayload<HelloWorldV1> {
/// Make a [`DataPayload`]`<`[`HelloWorldV1`]`>` from a static string slice.
pub fn from_static_str(s: &'static str) -> DataPayload<HelloWorldV1> {
DataPayload::from_owned(HelloWorld {
message: Cow::Borrowed(s),
})
}
}
#[cfg(feature = "deserialize_json")]
/// A data provider returning Hello World strings in different languages as JSON blobs.
///
/// Mostly useful for testing.
///
/// # Examples
///
/// ```
/// use icu_locale_core::langid;
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
///
/// let german_hello_world = HelloWorldProvider
/// .into_json_provider()
/// .load_data(HelloWorldV1::INFO, DataRequest {
/// id: DataIdentifierBorrowed::for_locale(&langid!("de").into()),
/// ..Default::default()
/// })
/// .expect("Loading should succeed");
///
/// assert_eq!(german_hello_world.payload.get(), br#"{"message":"Hallo Welt"}"#);
#[derive(Debug)]
pub struct HelloWorldJsonProvider;
#[cfg(feature = "deserialize_json")]
impl DynamicDataProvider<BufferMarker> for HelloWorldJsonProvider {
fn load_data(
&self,
marker: DataMarkerInfo,
req: DataRequest,
) -> Result<DataResponse<BufferMarker>, DataError> {
marker.match_marker(HelloWorldV1::INFO)?;
let result = HelloWorldProvider.load(req)?;
Ok(DataResponse {
metadata: DataResponseMetadata {
buffer_format: Some(icu_provider::buf::BufferFormat::Json),
..result.metadata
},
#[expect(clippy::unwrap_used)] // HelloWorld::serialize is infallible
payload: DataPayload::from_owned_buffer(
serde_json::to_string(result.payload.get())
.unwrap()
.into_bytes()
.into_boxed_slice(),
),
})
}
}
impl IterableDataProvider<HelloWorldV1> for HelloWorldProvider {
fn iter_ids(&self) -> Result<BTreeSet<DataIdentifierCow<'_>>, DataError> {
#[expect(clippy::unwrap_used)] // hello-world
Ok(Self::DATA
.iter()
.map(|(l, a, _)| {
DataIdentifierCow::from_borrowed_and_owned(
DataMarkerAttributes::from_str_or_panic(a),
l.parse().unwrap(),
)
})
.collect())
}
}
#[cfg(feature = "export")]
icu_provider::export::make_exportable_provider!(HelloWorldProvider, [HelloWorldV1,]);
define_preferences!(
/// Hello World Preferences.
[Copy]
HelloWorldFormatterPreferences, {}
);
/// A type that formats localized "hello world" strings.
///
/// This type is intended to take the shape of a typical ICU4X formatter API.
///
/// # Examples
///
/// ```
/// use icu_locale_core::locale;
/// use icu_provider::hello_world::{HelloWorldFormatter, HelloWorldProvider};
/// use writeable::assert_writeable_eq;
///
/// let fmt = HelloWorldFormatter::try_new_unstable(
/// &HelloWorldProvider,
/// locale!("eo").into(),
/// )
/// .expect("locale exists");
///
/// assert_writeable_eq!(fmt.format(), "Saluton, Mondo");
/// ```
#[derive(Debug)]
pub struct HelloWorldFormatter {
data: DataPayload<HelloWorldV1>,
}
/// A formatted hello world message. Implements [`Writeable`].
///
/// For an example, see [`HelloWorldFormatter`].
#[derive(Debug)]
pub struct FormattedHelloWorld<'l> {
data: &'l HelloWorld<'l>,
}
impl HelloWorldFormatter {
/// Creates a new [`HelloWorldFormatter`] for the specified locale.
///
/// [📚 Help choosing a constructor](icu_provider::constructors)
pub fn try_new(prefs: HelloWorldFormatterPreferences) -> Result<Self, DataError> {
Self::try_new_unstable(&HelloWorldProvider, prefs)
}
icu_provider::gen_buffer_data_constructors!((prefs: HelloWorldFormatterPreferences) -> error: DataError,
functions: [
try_new: skip,
try_new_with_buffer_provider,
try_new_unstable,
Self,
]);
#[doc = icu_provider::gen_buffer_unstable_docs!(UNSTABLE, Self::try_new)]
pub fn try_new_unstable<P>(
provider: &P,
prefs: HelloWorldFormatterPreferences,
) -> Result<Self, DataError>
where
P: DataProvider<HelloWorldV1>,
{
let locale = HelloWorldV1::make_locale(prefs.locale_preferences);
let data = provider
.load(DataRequest {
id: crate::request::DataIdentifierBorrowed::for_locale(&locale),
..Default::default()
})?
.payload;
Ok(Self { data })
}
/// Formats a hello world message, returning a [`FormattedHelloWorld`].
pub fn format<'l>(&'l self) -> FormattedHelloWorld<'l> {
FormattedHelloWorld {
data: self.data.get(),
}
}
/// Formats a hello world message, returning a [`String`].
pub fn format_to_string(&self) -> String {
self.format().write_to_string().into_owned()
}
}
impl Writeable for FormattedHelloWorld<'_> {
fn write_to<W: core::fmt::Write + ?Sized>(&self, sink: &mut W) -> core::fmt::Result {
self.data.message.write_to(sink)
}
fn writeable_borrow(&self) -> Option<&str> {
self.data.message.writeable_borrow()
}
fn writeable_length_hint(&self) -> writeable::LengthHint {
self.data.message.writeable_length_hint()
}
}
writeable::impl_display_with_writeable!(FormattedHelloWorld<'_>);
#[cfg(feature = "export")]
#[test]
fn test_iter() {
use crate::IterableDataProvider;
use icu_locale_core::locale;
assert_eq!(
HelloWorldProvider.iter_ids().unwrap(),
BTreeSet::from_iter([
DataIdentifierCow::from_locale(locale!("bn").into()),
DataIdentifierCow::from_locale(locale!("cs").into()),
DataIdentifierCow::from_locale(locale!("de").into()),
DataIdentifierCow::from_locale(locale!("de-AT").into()),
DataIdentifierCow::from_locale(locale!("el").into()),
DataIdentifierCow::from_locale(locale!("en").into()),
DataIdentifierCow::from_locale(locale!("en-001").into()),
DataIdentifierCow::from_locale(locale!("en-002").into()),
DataIdentifierCow::from_locale(locale!("en-019").into()),
DataIdentifierCow::from_locale(locale!("en-142").into()),
DataIdentifierCow::from_locale(locale!("en-GB").into()),
DataIdentifierCow::from_locale(locale!("en-GB-u-sd-gbeng").into()),
DataIdentifierCow::from_borrowed_and_owned(
DataMarkerAttributes::from_str_or_panic("reverse"),
locale!("en").into()
),
DataIdentifierCow::from_locale(locale!("eo").into()),
DataIdentifierCow::from_locale(locale!("fa").into()),
DataIdentifierCow::from_locale(locale!("fi").into()),
DataIdentifierCow::from_locale(locale!("is").into()),
DataIdentifierCow::from_locale(locale!("ja").into()),
DataIdentifierCow::from_borrowed_and_owned(
DataMarkerAttributes::from_str_or_panic("reverse"),
locale!("ja").into()
),
DataIdentifierCow::from_locale(locale!("la").into()),
DataIdentifierCow::from_locale(locale!("pt").into()),
DataIdentifierCow::from_locale(locale!("ro").into()),
DataIdentifierCow::from_locale(locale!("ru").into()),
DataIdentifierCow::from_locale(locale!("sr").into()),
DataIdentifierCow::from_locale(locale!("sr-Latn").into()),
DataIdentifierCow::from_locale(locale!("vi").into()),
DataIdentifierCow::from_locale(locale!("zh").into()),
])
);
}

231
vendor/icu_provider/src/lib.rs vendored Normal file
View File

@@ -0,0 +1,231 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
//! `icu_provider` is one of the `ICU4X` components.
//!
//! Unicode's experience with ICU4X's parent projects, ICU4C and ICU4J, led the team to realize
//! that data management is the most critical aspect of deploying internationalization, and that it requires
//! a high level of customization for the needs of the platform it is embedded in. As a result
//! ICU4X comes with a selection of providers that should allow for ICU4X to naturally fit into
//! different business and technological needs of customers.
//!
//! `icu_provider` defines traits and structs for transmitting data through the ICU4X locale
//! data pipeline. The primary trait is [`DataProvider`]. It is parameterized by a
//! [`DataMarker`], which is the type-system-level data identifier. [`DataProvider`] has a single method,
//! [`DataProvider::load`], which transforms a [`DataRequest`] into a [`DataResponse`].
//!
//! - [`DataRequest`] contains selectors to choose a specific variant of the marker, such as a locale.
//! - [`DataResponse`] contains the data if the request was successful.
//!
//! The most common types required for this crate are included via the prelude:
//!
//! ```
//! use icu_provider::prelude::*;
//! ```
//!
//! ## Dynamic Data Providers
//!
//! If the type system cannot be leveraged to load data (such as when dynamically loading from I/O),
//! there's another form of the [`DataProvider`]: [`DynamicDataProvider`]. While [`DataProvider`] is parametrized
//! on the type-system level by a [`DataMarker`] (which are distinct types implementing this trait),
//! [`DynamicDataProvider`]s are parametrized at runtime by a [`DataMarkerInfo`] struct, which essentially is the runtime
//! representation of the [`DataMarker`] type.
//!
//! The [`DynamicDataProvider`] is still type-level parametrized by the type that it loads, and there are two
//! implementations that should be called out
//!
//! - [`DynamicDataProvider<BufferMarker>`], a.k.a. [`BufferProvider`](buf::BufferProvider) returns data as `[u8]` buffers.
//!
//! ### BufferProvider
//!
//! These providers are able to return unstructured data typically represented as
//! [`serde`]-serialized buffers. Users can call [`as_deserializing()`] to get an object
//! implementing [`DataProvider`] by invoking Serde Deserialize.
//!
//! Examples of BufferProviders:
//!
//! - [`FsDataProvider`] reads individual buffers from the filesystem.
//! - [`BlobDataProvider`] reads buffers from a large in-memory blob.
//!
//! ## Provider Adapters
//!
//! ICU4X offers several built-in modules to combine providers in interesting ways.
//! These can be found in the [`icu_provider_adapters`] crate.
//!
//! ## Testing Provider
//!
//! This crate also contains a concrete provider for demonstration purposes:
//!
//! - [`HelloWorldProvider`] returns "hello world" strings in several languages.
//!
//! ## Types and Lifetimes
//!
//! Types compatible with [`Yokeable`] can be passed through the data provider, so long as they are
//! associated with a marker type implementing [`DynamicDataMarker`].
//!
//! Data structs should generally have one lifetime argument: `'data`. This lifetime allows data
//! structs to borrow zero-copy data.
//!
//! [`FixedProvider`]: https://docs.rs/icu_provider_adapters/latest/fixed/any_payload/struct.FixedProvider.html
//! [`HelloWorldProvider`]: hello_world::HelloWorldProvider
//! [`Yokeable`]: yoke::Yokeable
//! [`impl_dynamic_data_provider!`]: dynutil::impl_dynamic_data_provider
//! [`icu_provider_adapters`]: https://docs.rs/icu_provider_adapters/latest/icu_provider_adapters/index.html
//! [`SourceDataProvider`]: https://docs.rs/icu_provider_source/latest/icu_provider_source/struct.SourceDataProvider.html
//! [`as_deserializing()`]: buf::AsDeserializingBufferProvider::as_deserializing
//! [`FsDataProvider`]: https://docs.rs/icu_provider_fs/latest/icu_provider_fs/struct.FsDataProvider.html
//! [`BlobDataProvider`]: https://docs.rs/icu_provider_blob/latest/icu_provider_blob/struct.BlobDataProvider.html
// https://github.com/unicode-org/icu4x/blob/main/documents/process/boilerplate.md#library-annotations
#![cfg_attr(not(any(test, feature = "std")), no_std)]
#![cfg_attr(
not(test),
deny(
clippy::indexing_slicing,
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::exhaustive_structs,
clippy::exhaustive_enums,
clippy::trivially_copy_pass_by_ref,
missing_debug_implementations,
)
)]
#![warn(missing_docs)]
#[cfg(feature = "alloc")]
extern crate alloc;
#[cfg(feature = "baked")]
pub mod baked;
pub mod buf;
pub mod constructors;
pub mod dynutil;
#[cfg(feature = "export")]
pub mod export;
#[cfg(feature = "alloc")]
pub mod hello_world;
// TODO: put this in a separate crate
#[cfg(all(feature = "serde", feature = "alloc"))]
#[doc(hidden)]
pub mod serde_borrow_de_utils;
mod data_provider;
pub use data_provider::{
BoundDataProvider, DataProvider, DataProviderWithMarker, DryDataProvider, DynamicDataProvider,
DynamicDryDataProvider,
};
#[cfg(feature = "alloc")]
pub use data_provider::{IterableDataProvider, IterableDynamicDataProvider};
mod error;
pub use error::{DataError, DataErrorKind, ResultDataError};
mod request;
pub use request::{DataLocale, DataMarkerAttributes, DataRequest, DataRequestMetadata, *};
mod response;
#[doc(hidden)] // TODO(#4467): establish this as an internal API
pub use response::DataPayloadOr;
pub use response::{Cart, DataPayload, DataResponse, DataResponseMetadata};
#[path = "marker.rs"]
mod marker_full;
pub use marker_full::{DataMarker, DataMarkerInfo, DynamicDataMarker};
pub mod marker {
//! Additional [`DataMarker`](super::DataMarker) helpers.
#[doc(inline)]
pub use super::marker_full::impl_data_provider_never_marker;
pub use super::marker_full::{
DataMarkerExt, DataMarkerId, DataMarkerIdHash, ErasedMarker, NeverMarker,
};
}
mod varule_traits;
pub mod ule {
//! Traits that data provider implementations can use to optimize storage
//! by using [`VarULE`](zerovec::ule::VarULE).
//!
//! See [`MaybeAsVarULE`] for details.
pub use super::varule_traits::MaybeAsVarULE;
#[cfg(feature = "export")]
pub use super::varule_traits::MaybeEncodeAsVarULE;
}
/// Core selection of APIs and structures for the ICU4X data provider.
pub mod prelude {
#[doc(no_inline)]
#[cfg(feature = "serde")]
pub use crate::buf::AsDeserializingBufferProvider;
#[doc(no_inline)]
pub use crate::buf::{BufferMarker, BufferProvider};
#[doc(no_inline)]
pub use crate::{
data_marker, data_struct, marker::DataMarkerExt, request::AttributeParseError,
request::DataIdentifierBorrowed, BoundDataProvider, DataError, DataErrorKind, DataLocale,
DataMarker, DataMarkerAttributes, DataMarkerInfo, DataPayload, DataProvider, DataRequest,
DataRequestMetadata, DataResponse, DataResponseMetadata, DryDataProvider,
DynamicDataMarker, DynamicDataProvider, DynamicDryDataProvider, ResultDataError,
};
#[cfg(feature = "alloc")]
#[doc(no_inline)]
pub use crate::{
request::DataIdentifierCow, IterableDataProvider, IterableDynamicDataProvider,
};
#[doc(no_inline)]
pub use icu_locale_core;
#[doc(no_inline)]
pub use yoke;
#[doc(no_inline)]
pub use zerofrom;
}
#[doc(hidden)] // internal
pub mod fallback;
#[doc(hidden)] // internal
#[cfg(feature = "logging")]
pub use log;
#[doc(hidden)] // internal
#[cfg(all(
not(feature = "logging"),
all(debug_assertions, feature = "alloc", not(target_os = "none"))
))]
pub mod log {
extern crate std;
pub use std::eprintln as error;
pub use std::eprintln as warn;
pub use std::eprintln as info;
pub use std::eprintln as debug;
pub use std::eprintln as trace;
}
#[cfg(all(
not(feature = "logging"),
not(all(debug_assertions, feature = "alloc", not(target_os = "none"),))
))]
#[doc(hidden)] // internal
pub mod log {
#[macro_export]
macro_rules! _internal_noop_log {
($($t:expr),*) => {};
}
pub use crate::_internal_noop_log as error;
pub use crate::_internal_noop_log as warn;
pub use crate::_internal_noop_log as info;
pub use crate::_internal_noop_log as debug;
pub use crate::_internal_noop_log as trace;
}
#[test]
fn test_logging() {
// This should compile on all combinations of features
crate::log::info!("Hello World");
}

665
vendor/icu_provider/src/marker.rs vendored Normal file
View File

@@ -0,0 +1,665 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
use crate::fallback::{LocaleFallbackConfig, LocaleFallbackPriority};
use crate::{DataError, DataErrorKind, DataLocale, DataProvider, DataProviderWithMarker};
use core::fmt;
use core::marker::PhantomData;
use icu_locale_core::preferences::LocalePreferences;
use yoke::Yokeable;
use zerovec::ule::*;
/// Trait marker for data structs. All types delivered by the data provider must be associated with
/// something implementing this trait.
///
/// Data markers normally generated with the [`data_marker`](crate::data_marker) macro.
///
/// Also see [`DataMarker`].
///
/// Note: `DynamicDataMarker`s are quasi-const-generic compile-time objects, and as such are expected
/// to be unit structs. As this is not something that can be enforced by the type system, we
/// currently only have a `'static` bound on them (which is needed by a lot of our code).
///
/// # Examples
///
/// Manually implementing DynamicDataMarker for a custom type:
///
/// ```
/// use icu_provider::prelude::*;
/// use std::borrow::Cow;
///
/// #[derive(yoke::Yokeable, zerofrom::ZeroFrom)]
/// struct MyDataStruct<'data> {
/// message: Cow<'data, str>,
/// }
///
/// struct MyDataStructMarker;
///
/// impl DynamicDataMarker for MyDataStructMarker {
/// type DataStruct = MyDataStruct<'static>;
/// }
///
/// // We can now use MyDataStruct with DataProvider:
/// let s = MyDataStruct {
/// message: Cow::Owned("Hello World".into()),
/// };
/// let payload = DataPayload::<MyDataStructMarker>::from_owned(s);
/// assert_eq!(payload.get().message, "Hello World");
/// ```
///
/// [`data_struct`]: crate::data_struct
pub trait DynamicDataMarker: 'static {
/// A type that implements [`Yokeable`]. This should typically be the `'static` version of a
/// data struct.
type DataStruct: for<'a> Yokeable<'a>;
}
/// A [`DynamicDataMarker`] with a [`DataMarkerInfo`] attached.
///
/// Structs implementing this trait are normally generated with the [`data_struct!`] macro.
///
/// Implementing this trait enables this marker to be used with the main [`DataProvider`] trait.
/// Most markers should be associated with a specific marker and should therefore implement this
/// trait.
///
/// [`BufferMarker`] is an example of a marker that does _not_ implement this trait.
///
/// Note: `DataMarker`s are quasi-const-generic compile-time objects, and as such are expected
/// to be unit structs. As this is not something that can be enforced by the type system, we
/// currently only have a `'static` bound on them (which is needed by a lot of our code).
///
/// [`data_struct!`]: crate::data_struct
/// [`DataProvider`]: crate::DataProvider
/// [`BufferMarker`]: crate::buf::BufferMarker
pub trait DataMarker: DynamicDataMarker {
/// The single [`DataMarkerInfo`] associated with this marker.
const INFO: DataMarkerInfo;
}
/// Extension trait for methods on [`DataMarker`]
pub trait DataMarkerExt: DataMarker + Sized {
/// Binds a [`DataMarker`] to a provider supporting it.
fn bind<P>(provider: P) -> DataProviderWithMarker<Self, P>
where
P: DataProvider<Self>;
/// Constructs a [`DataLocale`] using fallback preferences from this [`DataMarker`].
fn make_locale(locale: LocalePreferences) -> DataLocale;
}
impl<M: DataMarker + Sized> DataMarkerExt for M {
fn bind<P>(provider: P) -> DataProviderWithMarker<Self, P>
where
P: DataProvider<Self>,
{
DataProviderWithMarker::new(provider)
}
fn make_locale(locale: LocalePreferences) -> DataLocale {
M::INFO.make_locale(locale)
}
}
/// A [`DynamicDataMarker`] that never returns data.
///
/// All types that have non-blanket impls of `DataProvider<M>` are expected to explicitly
/// implement `DataProvider<NeverMarker<Y>>`, returning [`DataErrorKind::MarkerNotFound`].
/// See [`impl_data_provider_never_marker!`].
///
/// [`DataErrorKind::MarkerNotFound`]: crate::DataErrorKind::MarkerNotFound
/// [`impl_data_provider_never_marker!`]: crate::marker::impl_data_provider_never_marker
///
/// # Examples
///
/// ```
/// use icu_locale_core::langid;
/// use icu_provider::hello_world::*;
/// use icu_provider::marker::NeverMarker;
/// use icu_provider::prelude::*;
///
/// let buffer_provider = HelloWorldProvider.into_json_provider();
///
/// let result = DataProvider::<NeverMarker<HelloWorld<'static>>>::load(
/// &buffer_provider.as_deserializing(),
/// DataRequest {
/// id: DataIdentifierBorrowed::for_locale(&langid!("en").into()),
/// ..Default::default()
/// },
/// );
///
/// assert!(matches!(
/// result,
/// Err(DataError {
/// kind: DataErrorKind::MarkerNotFound,
/// ..
/// })
/// ));
/// ```
#[derive(Debug, Copy, Clone)]
pub struct NeverMarker<Y>(PhantomData<Y>);
impl<Y> DynamicDataMarker for NeverMarker<Y>
where
for<'a> Y: Yokeable<'a>,
{
type DataStruct = Y;
}
impl<Y> DataMarker for NeverMarker<Y>
where
for<'a> Y: Yokeable<'a>,
{
const INFO: DataMarkerInfo = DataMarkerInfo::from_id(DataMarkerId {
#[cfg(any(feature = "export", debug_assertions))]
debug: "NeverMarker",
hash: *b"nevermar",
});
}
/// Implements `DataProvider<NeverMarker<Y>>` on a struct.
///
/// For more information, see [`NeverMarker`].
///
/// # Examples
///
/// ```
/// use icu_locale_core::langid;
/// use icu_provider::hello_world::*;
/// use icu_provider::marker::NeverMarker;
/// use icu_provider::prelude::*;
///
/// struct MyProvider;
///
/// icu_provider::marker::impl_data_provider_never_marker!(MyProvider);
///
/// let result = DataProvider::<NeverMarker<HelloWorld<'static>>>::load(
/// &MyProvider,
/// DataRequest {
/// id: DataIdentifierBorrowed::for_locale(&langid!("und").into()),
/// ..Default::default()
/// },
/// );
///
/// assert!(matches!(
/// result,
/// Err(DataError {
/// kind: DataErrorKind::MarkerNotFound,
/// ..
/// })
/// ));
/// ```
#[doc(hidden)] // macro
#[macro_export]
macro_rules! __impl_data_provider_never_marker {
($ty:path) => {
impl<Y> $crate::DataProvider<$crate::marker::NeverMarker<Y>> for $ty
where
for<'a> Y: $crate::prelude::yoke::Yokeable<'a>,
{
fn load(
&self,
req: $crate::DataRequest,
) -> Result<$crate::DataResponse<$crate::marker::NeverMarker<Y>>, $crate::DataError>
{
Err($crate::DataErrorKind::MarkerNotFound.with_req(
<$crate::marker::NeverMarker<Y> as $crate::DataMarker>::INFO,
req,
))
}
}
};
}
#[doc(inline)]
pub use __impl_data_provider_never_marker as impl_data_provider_never_marker;
/// A compact hash of a [`DataMarkerInfo`]. Useful for keys in maps.
///
/// The hash will be stable over time within major releases.
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, ULE)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[repr(transparent)]
pub struct DataMarkerIdHash([u8; 4]);
impl DataMarkerIdHash {
/// Magic bytes to locate [`DataMarkerIdHash`]es in binaries.
pub const LEADING_TAG: &[u8] = b"tdmh";
/// Gets the hash value as a byte array.
pub const fn to_bytes(self) -> [u8; 4] {
self.0
}
}
/// Const function to compute the FxHash of a byte array.
///
/// FxHash is a speedy hash algorithm used within rustc. The algorithm is satisfactory for our
/// use case since the strings being hashed originate from a trusted source (the ICU4X
/// components), and the hashes are computed at compile time, so we can check for collisions.
///
/// We could have considered a SHA or other cryptographic hash function. However, we are using
/// FxHash because:
///
/// 1. There is precedent for this algorithm in Rust
/// 2. The algorithm is easy to implement as a const function
/// 3. The amount of code is small enough that we can reasonably keep the algorithm in-tree
/// 4. FxHash is designed to output 32-bit or 64-bit values, whereas SHA outputs more bits,
/// such that truncation would be required in order to fit into a u32, partially reducing
/// the benefit of a cryptographically secure algorithm
// The indexing operations in this function have been reviewed in detail and won't panic.
#[expect(clippy::indexing_slicing)]
const fn fxhash_32(bytes: &[u8]) -> u32 {
// This code is adapted from https://github.com/rust-lang/rustc-hash,
// whose license text is reproduced below.
//
// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#[inline]
const fn hash_word_32(mut hash: u32, word: u32) -> u32 {
const ROTATE: u32 = 5;
const SEED32: u32 = 0x9e_37_79_b9;
hash = hash.rotate_left(ROTATE);
hash ^= word;
hash = hash.wrapping_mul(SEED32);
hash
}
let mut cursor = 0;
let end = bytes.len();
let mut hash = 0;
while end - cursor >= 4 {
let word = u32::from_le_bytes([
bytes[cursor],
bytes[cursor + 1],
bytes[cursor + 2],
bytes[cursor + 3],
]);
hash = hash_word_32(hash, word);
cursor += 4;
}
if end - cursor >= 2 {
let word = u16::from_le_bytes([bytes[cursor], bytes[cursor + 1]]);
hash = hash_word_32(hash, word as u32);
cursor += 2;
}
if end - cursor >= 1 {
hash = hash_word_32(hash, bytes[cursor] as u32);
}
hash
}
#[cfg(feature = "alloc")]
impl<'a> zerovec::maps::ZeroMapKV<'a> for DataMarkerIdHash {
type Container = zerovec::ZeroVec<'a, DataMarkerIdHash>;
type Slice = zerovec::ZeroSlice<DataMarkerIdHash>;
type GetType = <DataMarkerIdHash as AsULE>::ULE;
type OwnedType = DataMarkerIdHash;
}
impl AsULE for DataMarkerIdHash {
type ULE = Self;
#[inline]
fn to_unaligned(self) -> Self::ULE {
self
}
#[inline]
fn from_unaligned(unaligned: Self::ULE) -> Self {
unaligned
}
}
// Safe since the ULE type is `self`.
unsafe impl EqULE for DataMarkerIdHash {}
/// The ID of a data marker.
///
/// This is generally a [`DataMarkerIdHash`]. If debug assertions or the `export` Cargo feature
/// are enabled, this also contains a human-readable string for an improved `Debug` implementation.
#[derive(Debug, Copy, Clone, Eq)]
pub struct DataMarkerId {
#[cfg(any(feature = "export", debug_assertions))]
debug: &'static str,
hash: [u8; 8],
}
impl PartialEq for DataMarkerId {
#[inline]
fn eq(&self, other: &Self) -> bool {
self.hash == other.hash
}
}
impl Ord for DataMarkerId {
#[inline]
fn cmp(&self, other: &Self) -> core::cmp::Ordering {
self.hash.cmp(&other.hash)
}
}
impl PartialOrd for DataMarkerId {
#[inline]
fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl core::hash::Hash for DataMarkerId {
#[inline]
fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
self.hash.hash(state)
}
}
impl DataMarkerId {
#[doc(hidden)]
// macro use
// Error is a str of the expected character class and the index where it wasn't encountered
// The indexing operations in this function have been reviewed in detail and won't panic.
pub const fn from_name(name: &'static str) -> Result<Self, (&'static str, usize)> {
#![allow(clippy::indexing_slicing)]
if !name.as_bytes()[name.len() - 1].is_ascii_digit() {
return Err(("[0-9]", name.len()));
}
let mut i = name.len() - 1;
while name.as_bytes()[i - 1].is_ascii_digit() {
i -= 1;
}
if name.as_bytes()[i - 1] != b'V' {
return Err(("V", i));
}
let magic = DataMarkerIdHash::LEADING_TAG;
let hash = fxhash_32(name.as_bytes()).to_le_bytes();
Ok(Self {
#[cfg(any(feature = "export", debug_assertions))]
debug: name,
hash: [
magic[0], magic[1], magic[2], magic[3], hash[0], hash[1], hash[2], hash[3],
],
})
}
/// Gets a platform-independent hash of a [`DataMarkerId`].
///
/// The hash is 4 bytes and allows for fast comparison.
///
/// # Example
///
/// ```
/// use icu_provider::prelude::*;
///
/// icu_provider::data_marker!(FooV1, &'static str);
///
/// assert_eq!(FooV1::INFO.id.hashed().to_bytes(), [198, 217, 86, 48]);
/// ```
#[inline]
pub const fn hashed(self) -> DataMarkerIdHash {
let [.., h1, h2, h3, h4] = self.hash;
DataMarkerIdHash([h1, h2, h3, h4])
}
/// Returns the marker name.
///
/// For size reasons, this is only available with the `export` Cargo feature.
#[cfg(feature = "export")]
pub const fn name(self) -> &'static str {
self.debug
}
}
/// Used for loading data from a dynamic ICU4X data provider.
///
/// A data marker is tightly coupled with the code that uses it to load data at runtime.
/// Executables can be searched for `DataMarkerInfo` instances to produce optimized data files.
/// Therefore, users should not generally create DataMarkerInfo instances; they should instead use
/// the ones exported by a component.
#[derive(Copy, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct DataMarkerInfo {
/// The ID of this marker.
pub id: DataMarkerId,
/// Whether this data marker only has a single payload, not keyed by a data identifier.
pub is_singleton: bool,
/// Whether this data marker uses checksums for integrity purposes.
pub has_checksum: bool,
/// The fallback to use for this data marker.
pub fallback_config: LocaleFallbackConfig,
/// The attributes domain for this data marker. This can be used for filtering marker
/// attributes during provider export.
#[cfg(feature = "export")]
pub attributes_domain: &'static str,
/// Whether to create constants for each data struct in baked data.
#[cfg(feature = "export")]
pub expose_baked_consts: bool,
}
impl PartialOrd for DataMarkerInfo {
fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for DataMarkerInfo {
fn cmp(&self, other: &Self) -> core::cmp::Ordering {
self.id.cmp(&other.id)
}
}
impl core::hash::Hash for DataMarkerInfo {
fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
self.id.hash(state)
}
}
impl DataMarkerInfo {
/// See [`Default::default`]
pub const fn from_id(id: DataMarkerId) -> Self {
Self {
id,
fallback_config: LocaleFallbackConfig::default(),
is_singleton: false,
has_checksum: false,
#[cfg(feature = "export")]
attributes_domain: "",
#[cfg(feature = "export")]
expose_baked_consts: false,
}
}
/// Returns [`Ok`] if this data marker matches the argument, or the appropriate error.
///
/// Convenience method for data providers that support a single [`DataMarkerInfo`].
///
/// # Examples
///
/// ```
/// use icu_provider::hello_world::*;
/// use icu_provider::prelude::*;
///
/// icu_provider::data_marker!(
/// DummyV1,
/// <HelloWorldV1 as DynamicDataMarker>::DataStruct
/// );
///
/// assert!(matches!(
/// HelloWorldV1::INFO.match_marker(HelloWorldV1::INFO),
/// Ok(())
/// ));
/// assert!(matches!(
/// HelloWorldV1::INFO.match_marker(DummyV1::INFO),
/// Err(DataError {
/// kind: DataErrorKind::MarkerNotFound,
/// ..
/// })
/// ));
///
/// // The error context contains the argument:
/// assert_eq!(
/// HelloWorldV1::INFO
/// .match_marker(DummyV1::INFO)
/// .unwrap_err()
/// .marker,
/// Some(DummyV1::INFO.id)
/// );
/// ```
pub fn match_marker(self, marker: Self) -> Result<(), DataError> {
if self == marker {
Ok(())
} else {
Err(DataErrorKind::MarkerNotFound.with_marker(marker))
}
}
/// Constructs a [`DataLocale`] for this [`DataMarkerInfo`].
pub fn make_locale(self, locale: LocalePreferences) -> DataLocale {
if self.fallback_config.priority == LocaleFallbackPriority::Region {
locale.to_data_locale_region_priority()
} else {
locale.to_data_locale_language_priority()
}
}
}
/// Creates a data marker.
///
/// # Examples
///
/// ```
/// icu_provider::data_marker!(DummyV1, &'static str);
/// ```
///
/// The identifier needs to end with a `V` followed by one or more digits (the version number).
///
/// Invalid identifiers are compile-time errors (as [`data_marker!`](crate::data_marker) uses `const`).
///
/// ```compile_fail,E0080
/// icu_provider::data_marker!(Dummy, &'static str);
/// ```
#[macro_export] // canonical location is crate root
macro_rules! data_marker {
($(#[$doc:meta])* $name:ident, $($debug:literal,)? $struct:ty $(, $(#[$meta:meta])* $info_field:ident = $info_val:expr)* $(,)?) => {
$(#[$doc])*
#[non_exhaustive]
pub struct $name;
impl $crate::DynamicDataMarker for $name {
type DataStruct = $struct;
}
impl $crate::DataMarker for $name {
const INFO: $crate::DataMarkerInfo = {
$(
/// ```rust
#[doc = concat!("let ident = \"", stringify!($name), "\";")]
#[doc = concat!("let debug = \"", $debug, "\";")]
/// assert_eq!(
/// debug.split('/').map(|s| {
/// let mut b = s.to_ascii_lowercase().into_bytes();
/// b[0] = b[0].to_ascii_uppercase();
/// String::from_utf8(b).unwrap()
/// })
/// .collect::<Vec<_>>()
/// .join(""),
/// ident
/// );
/// ```
#[allow(dead_code)]
struct DebugTest;
)?
#[allow(unused_mut)]
// Force evaluation even if marker is unused
let mut info = const { $crate::DataMarkerInfo::from_id(
match $crate::marker::DataMarkerId::from_name(stringify!($name)) {
Ok(path) => path,
Err(_) => panic!(concat!("Invalid marker name: ", stringify!($name))),
})};
$(
$(#[$meta])*
{info.$info_field = $info_val;}
)*
info
};
}
}
}
impl fmt::Debug for DataMarkerInfo {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
#[cfg(any(feature = "export", debug_assertions))]
return f.write_str(self.id.debug);
#[cfg(not(any(feature = "export", debug_assertions)))]
return write!(f, "{:?}", self.id);
}
}
/// A marker for the given `DataStruct`.
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
pub struct ErasedMarker<DataStruct: for<'a> Yokeable<'a>>(PhantomData<DataStruct>);
impl<DataStruct: for<'a> Yokeable<'a>> DynamicDataMarker for ErasedMarker<DataStruct> {
type DataStruct = DataStruct;
}
#[test]
fn test_marker_syntax() {
// Valid markers:
DataMarkerId::from_name("HelloWorldV1").unwrap();
DataMarkerId::from_name("HelloWorldFooV1").unwrap();
DataMarkerId::from_name("HelloWorldV999").unwrap();
DataMarkerId::from_name("Hello485FooV1").unwrap();
// No version:
assert_eq!(
DataMarkerId::from_name("HelloWorld"),
Err(("[0-9]", "HelloWorld".len()))
);
assert_eq!(
DataMarkerId::from_name("HelloWorldV"),
Err(("[0-9]", "HelloWorldV".len()))
);
assert_eq!(
DataMarkerId::from_name("HelloWorldVFoo"),
Err(("[0-9]", "HelloWorldVFoo".len()))
);
assert_eq!(
DataMarkerId::from_name("HelloWorldV1Foo"),
Err(("[0-9]", "HelloWorldV1Foo".len()))
);
}
#[test]
fn test_id_debug() {
assert_eq!(DataMarkerId::from_name("BarV1").unwrap().debug, "BarV1");
}
#[test]
fn test_hash_word_32() {
assert_eq!(0, fxhash_32(b""));
assert_eq!(0xF3051F19, fxhash_32(b"a"));
assert_eq!(0x2F9DF119, fxhash_32(b"ab"));
assert_eq!(0xCB1D9396, fxhash_32(b"abc"));
assert_eq!(0x8628F119, fxhash_32(b"abcd"));
assert_eq!(0xBEBDB56D, fxhash_32(b"abcde"));
assert_eq!(0x1CE8476D, fxhash_32(b"abcdef"));
assert_eq!(0xC0F176A4, fxhash_32(b"abcdefg"));
assert_eq!(0x09AB476D, fxhash_32(b"abcdefgh"));
assert_eq!(0xB72F5D88, fxhash_32(b"abcdefghi"));
}
#[test]
fn test_id_hash() {
assert_eq!(
DataMarkerId::from_name("BarV1").unwrap().hashed(),
DataMarkerIdHash([212, 77, 158, 241]),
);
}

376
vendor/icu_provider/src/request.rs vendored Normal file
View File

@@ -0,0 +1,376 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
#[cfg(feature = "alloc")]
use alloc::borrow::Cow;
#[cfg(feature = "alloc")]
use alloc::borrow::ToOwned;
#[cfg(feature = "alloc")]
use alloc::boxed::Box;
#[cfg(feature = "alloc")]
use alloc::string::String;
#[cfg(feature = "alloc")]
use core::cmp::Ordering;
use core::default::Default;
use core::fmt;
use core::fmt::Debug;
use core::hash::Hash;
use core::ops::Deref;
#[cfg(feature = "alloc")]
use zerovec::ule::VarULE;
pub use icu_locale_core::DataLocale;
/// The request type passed into all data provider implementations.
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
#[allow(clippy::exhaustive_structs)] // this type is stable
pub struct DataRequest<'a> {
/// The data identifier for which to load data.
///
/// If locale fallback is enabled, the resulting data may be from a different identifier
/// than the one requested here.
pub id: DataIdentifierBorrowed<'a>,
/// Metadata that may affect the behavior of the data provider.
pub metadata: DataRequestMetadata,
}
/// Metadata for data requests. This is currently empty, but it may be extended with options
/// for tuning locale fallback, buffer layout, and so forth.
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[non_exhaustive]
pub struct DataRequestMetadata {
/// Silent requests do not log errors. This can be used for exploratory querying, such as fallbacks.
pub silent: bool,
/// Whether to allow prefix matches for the data marker attributes.
pub attributes_prefix_match: bool,
}
/// The borrowed version of a [`DataIdentifierCow`].
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct DataIdentifierBorrowed<'a> {
/// Marker-specific request attributes
pub marker_attributes: &'a DataMarkerAttributes,
/// The CLDR locale
pub locale: &'a DataLocale,
}
impl fmt::Display for DataIdentifierBorrowed<'_> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::Display::fmt(self.locale, f)?;
if !self.marker_attributes.is_empty() {
write!(f, "/{}", self.marker_attributes.as_str())?;
}
Ok(())
}
}
impl<'a> DataIdentifierBorrowed<'a> {
/// Creates a [`DataIdentifierBorrowed`] for a borrowed [`DataLocale`].
pub fn for_locale(locale: &'a DataLocale) -> Self {
Self {
locale,
..Default::default()
}
}
/// Creates a [`DataIdentifierBorrowed`] for a borrowed [`DataMarkerAttributes`].
pub fn for_marker_attributes(marker_attributes: &'a DataMarkerAttributes) -> Self {
Self {
marker_attributes,
..Default::default()
}
}
/// Creates a [`DataIdentifierBorrowed`] for a borrowed [`DataMarkerAttributes`] and [`DataLocale`].
pub fn for_marker_attributes_and_locale(
marker_attributes: &'a DataMarkerAttributes,
locale: &'a DataLocale,
) -> Self {
Self {
marker_attributes,
locale,
}
}
/// Converts this [`DataIdentifierBorrowed`] into a [`DataIdentifierCow<'static>`].
///
/// ✨ *Enabled with the `alloc` Cargo feature.*
#[cfg(feature = "alloc")]
pub fn into_owned(self) -> DataIdentifierCow<'static> {
DataIdentifierCow {
marker_attributes: Cow::Owned(self.marker_attributes.to_owned()),
locale: *self.locale,
}
}
/// Borrows this [`DataIdentifierBorrowed`] as a [`DataIdentifierCow<'a>`].
///
/// ✨ *Enabled with the `alloc` Cargo feature.*
#[cfg(feature = "alloc")]
pub fn as_cow(self) -> DataIdentifierCow<'a> {
DataIdentifierCow {
marker_attributes: Cow::Borrowed(self.marker_attributes),
locale: *self.locale,
}
}
}
/// A data identifier identifies a particular version of data, such as "English".
///
/// It is a wrapper around a [`DataLocale`] and a [`DataMarkerAttributes`].
///
/// ✨ *Enabled with the `alloc` Cargo feature.*
#[derive(Debug, PartialEq, Eq, Hash, Clone)]
#[non_exhaustive]
#[cfg(feature = "alloc")]
pub struct DataIdentifierCow<'a> {
/// Marker-specific request attributes
pub marker_attributes: Cow<'a, DataMarkerAttributes>,
/// The CLDR locale
pub locale: DataLocale,
}
#[cfg(feature = "alloc")]
impl PartialOrd for DataIdentifierCow<'_> {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
#[cfg(feature = "alloc")]
impl Ord for DataIdentifierCow<'_> {
fn cmp(&self, other: &Self) -> Ordering {
self.marker_attributes
.cmp(&other.marker_attributes)
.then_with(|| self.locale.total_cmp(&other.locale))
}
}
#[cfg(feature = "alloc")]
impl fmt::Display for DataIdentifierCow<'_> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::Display::fmt(&self.locale, f)?;
if !self.marker_attributes.is_empty() {
write!(f, "/{}", self.marker_attributes.as_str())?;
}
Ok(())
}
}
#[cfg(feature = "alloc")]
impl<'a> DataIdentifierCow<'a> {
/// Borrows this [`DataIdentifierCow`] as a [`DataIdentifierBorrowed<'a>`].
pub fn as_borrowed(&'a self) -> DataIdentifierBorrowed<'a> {
DataIdentifierBorrowed {
marker_attributes: &self.marker_attributes,
locale: &self.locale,
}
}
/// Creates a [`DataIdentifierCow`] from an owned [`DataLocale`].
pub fn from_locale(locale: DataLocale) -> Self {
Self {
marker_attributes: Cow::Borrowed(DataMarkerAttributes::empty()),
locale,
}
}
/// Creates a [`DataIdentifierCow`] from a borrowed [`DataMarkerAttributes`].
pub fn from_marker_attributes(marker_attributes: &'a DataMarkerAttributes) -> Self {
Self {
marker_attributes: Cow::Borrowed(marker_attributes),
locale: Default::default(),
}
}
/// Creates a [`DataIdentifierCow`] from an owned [`DataMarkerAttributes`].
pub fn from_marker_attributes_owned(marker_attributes: Box<DataMarkerAttributes>) -> Self {
Self {
marker_attributes: Cow::Owned(marker_attributes),
locale: Default::default(),
}
}
/// Creates a [`DataIdentifierCow`] from an owned [`DataMarkerAttributes`] and an owned [`DataLocale`].
pub fn from_owned(marker_attributes: Box<DataMarkerAttributes>, locale: DataLocale) -> Self {
Self {
marker_attributes: Cow::Owned(marker_attributes),
locale,
}
}
/// Creates a [`DataIdentifierCow`] from a borrowed [`DataMarkerAttributes`] and an owned [`DataLocale`].
pub fn from_borrowed_and_owned(
marker_attributes: &'a DataMarkerAttributes,
locale: DataLocale,
) -> Self {
Self {
marker_attributes: Cow::Borrowed(marker_attributes),
locale,
}
}
/// Returns whether this id is equal to the default.
pub fn is_unknown(&self) -> bool {
self.marker_attributes.is_empty() && self.locale.is_unknown()
}
}
#[cfg(feature = "alloc")]
impl Default for DataIdentifierCow<'_> {
fn default() -> Self {
Self {
marker_attributes: Cow::Borrowed(Default::default()),
locale: Default::default(),
}
}
}
/// An additional key to identify data beyond a [`DataLocale`].
///
/// The is a loose wrapper around a string, with semantics defined by each [`DataMarker`](crate::DataMarker).
#[derive(PartialEq, Eq, Ord, PartialOrd, Hash)]
#[repr(transparent)]
pub struct DataMarkerAttributes {
// Validated to be non-empty ASCII alphanumeric + hyphen + underscore
value: str,
}
impl Default for &DataMarkerAttributes {
fn default() -> Self {
DataMarkerAttributes::empty()
}
}
impl Deref for DataMarkerAttributes {
type Target = str;
#[inline]
fn deref(&self) -> &Self::Target {
&self.value
}
}
impl Debug for DataMarkerAttributes {
#[inline]
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.value.fmt(f)
}
}
/// Invalid character
#[derive(Debug)]
#[non_exhaustive]
pub struct AttributeParseError;
impl DataMarkerAttributes {
/// Safety-usable invariant: validated bytes are ASCII only
const fn validate(s: &[u8]) -> Result<(), AttributeParseError> {
let mut i = 0;
while i < s.len() {
#[expect(clippy::indexing_slicing)] // duh
if !matches!(s[i], b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_') {
return Err(AttributeParseError);
}
i += 1;
}
Ok(())
}
/// Creates a borrowed [`DataMarkerAttributes`] from a borrowed string.
///
/// Returns an error if the string contains characters other than `[a-zA-Z0-9_\-]`.
pub const fn try_from_str(s: &str) -> Result<&Self, AttributeParseError> {
Self::try_from_utf8(s.as_bytes())
}
/// Attempts to create a borrowed [`DataMarkerAttributes`] from a borrowed UTF-8 encoded byte slice.
///
/// # Examples
///
/// ```
/// use icu_provider::prelude::*;
///
/// let bytes = b"long-meter";
/// let marker = DataMarkerAttributes::try_from_utf8(bytes).unwrap();
/// assert_eq!(marker.to_string(), "long-meter");
/// ```
///
/// # Errors
///
/// Returns an error if the byte slice contains code units other than `[a-zA-Z0-9_\-]`.
pub const fn try_from_utf8(code_units: &[u8]) -> Result<&Self, AttributeParseError> {
let Ok(()) = Self::validate(code_units) else {
return Err(AttributeParseError);
};
// SAFETY: `validate` requires a UTF-8 subset
let s = unsafe { core::str::from_utf8_unchecked(code_units) };
// SAFETY: `Self` has the same layout as `str`
Ok(unsafe { &*(s as *const str as *const Self) })
}
/// Creates an owned [`DataMarkerAttributes`] from an owned string.
///
/// Returns an error if the string contains characters other than `[a-zA-Z0-9_\-]`.
///
/// ✨ *Enabled with the `alloc` Cargo feature.*
#[cfg(feature = "alloc")]
pub fn try_from_string(s: String) -> Result<Box<Self>, AttributeParseError> {
let Ok(()) = Self::validate(s.as_bytes()) else {
return Err(AttributeParseError);
};
// SAFETY: `Self` has the same layout as `str`
Ok(unsafe { core::mem::transmute::<Box<str>, Box<Self>>(s.into_boxed_str()) })
}
/// Creates a borrowed [`DataMarkerAttributes`] from a borrowed string.
///
/// Panics if the string contains characters other than `[a-zA-Z0-9_\-]`.
pub const fn from_str_or_panic(s: &str) -> &Self {
let Ok(r) = Self::try_from_str(s) else {
panic!("Invalid marker attribute syntax")
};
r
}
/// Creates an empty [`DataMarkerAttributes`].
pub const fn empty() -> &'static Self {
// SAFETY: `Self` has the same layout as `str`
unsafe { &*("" as *const str as *const Self) }
}
/// Returns this [`DataMarkerAttributes`] as a `&str`.
pub const fn as_str(&self) -> &str {
&self.value
}
}
/// ✨ *Enabled with the `alloc` Cargo feature.*
#[cfg(feature = "alloc")]
impl ToOwned for DataMarkerAttributes {
type Owned = Box<Self>;
fn to_owned(&self) -> Self::Owned {
// SAFETY: `Self` has the same layout as `str`
unsafe { core::mem::transmute::<Box<str>, Box<Self>>(self.as_str().to_boxed()) }
}
}
#[test]
fn test_data_marker_attributes_from_utf8() {
let bytes_vec: Vec<&[u8]> = vec![
b"long-meter",
b"long",
b"meter",
b"short-meter-second",
b"usd",
];
for bytes in bytes_vec {
let marker = DataMarkerAttributes::try_from_utf8(bytes).unwrap();
assert_eq!(marker.to_string().as_bytes(), bytes);
}
}

1119
vendor/icu_provider/src/response.rs vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,82 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
use alloc::borrow::Cow;
use serde::de::Deserializer;
use serde::Deserialize;
#[derive(Deserialize)]
#[serde(transparent)]
// Cows fail to borrow in some situations (array, option), but structs of Cows don't.
#[allow(clippy::exhaustive_structs)] // newtype
#[derive(Debug)]
pub struct CowWrap<'data>(#[serde(borrow)] pub Cow<'data, str>);
#[derive(Deserialize)]
#[serde(transparent)]
// Cows fail to borrow in some situations (array, option), but structs of Cows don't.
#[allow(clippy::exhaustive_structs)] // newtype
#[derive(Debug)]
pub struct CowBytesWrap<'data>(#[serde(borrow)] pub Cow<'data, [u8]>);
pub fn array_of_cow<'de, D, const N: usize>(deserializer: D) -> Result<[Cow<'de, str>; N], D::Error>
where
D: Deserializer<'de>,
[CowWrap<'de>; N]: Deserialize<'de>,
{
<[CowWrap<'de>; N]>::deserialize(deserializer).map(|array| array.map(|wrap| wrap.0))
}
pub fn option_of_cow<'de, D>(deserializer: D) -> Result<Option<Cow<'de, str>>, D::Error>
where
D: Deserializer<'de>,
{
<Option<CowWrap<'de>>>::deserialize(deserializer).map(|opt| opt.map(|wrap| wrap.0))
}
pub fn tuple_of_cow<'de, D>(deserializer: D) -> Result<(Cow<'de, str>, Cow<'de, str>), D::Error>
where
D: Deserializer<'de>,
(CowWrap<'de>, CowWrap<'de>): Deserialize<'de>,
{
<(CowWrap<'de>, CowWrap<'de>)>::deserialize(deserializer).map(|x| (x.0 .0, x.1 .0))
}
#[test]
fn test_option() {
#[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)]
struct Demo<'s>(#[serde(borrow, deserialize_with = "option_of_cow")] Option<Cow<'s, str>>);
let data_orig = Demo(Some("Hello world".into()));
let json = serde_json::to_string(&data_orig).expect("serialize");
let data_new = serde_json::from_str::<Demo>(&json).expect("deserialize");
assert_eq!(data_orig, data_new);
assert!(matches!(data_new.0, Some(Cow::Borrowed(_))));
}
#[test]
fn test_tuple() {
#[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)]
struct Demo<'s>(
#[serde(borrow, deserialize_with = "tuple_of_cow")] (Cow<'s, str>, Cow<'s, str>),
);
let data_orig = Demo(("Hello world".into(), "Hello earth".into()));
let json = serde_json::to_string(&data_orig).expect("serialize");
let data_new = serde_json::from_str::<Demo>(&json).expect("deserialize");
assert_eq!(data_orig, data_new);
assert!(matches!(data_new.0, (Cow::Borrowed(_), Cow::Borrowed(_))));
}
#[test]
fn test_array() {
#[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)]
struct Demo<'s>(#[serde(borrow, deserialize_with = "array_of_cow")] [Cow<'s, str>; 1]);
let data_orig = Demo(["Hello world".into()]);
let json = serde_json::to_string(&data_orig).expect("serialize");
let data_new = serde_json::from_str::<Demo>(&json).expect("deserialize");
assert_eq!(data_orig, data_new);
assert!(matches!(data_new.0, [Cow::Borrowed(_)]));
}

159
vendor/icu_provider/src/varule_traits.rs vendored Normal file
View File

@@ -0,0 +1,159 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
use zerovec::ule::VarULE;
#[cfg(feature = "alloc")]
use zerovec::{maps::ZeroMapKV, ZeroMap, ZeroMap2d};
/// A trait that associates a [`VarULE`] type with a data struct.
///
/// Some data structs can be represented compactly as a single [`VarULE`],
/// such as `str` or a packed pattern. This trait allows for data providers
/// to use optimizations for such types.
///
/// ❗ Not all data structs benefit from this optimization. It works best when the
/// data struct is multiplied across a large number of data marker attributes.
///
/// Both [`MaybeAsVarULE`] and [`MaybeEncodeAsVarULE`] should be implemented
/// on all data structs. The [`data_struct!`](crate::data_struct) macro provides an impl.
pub trait MaybeAsVarULE {
/// The [`VarULE`] type for this data struct, or `[()]`
/// if it cannot be represented as [`VarULE`].
type EncodedStruct: ?Sized + VarULE;
}
/// Export-only trait associated with [`MaybeAsVarULE`]. See that trait
/// for additional details.
///
/// ✨ *Enabled with the `export` Cargo feature.*
#[cfg(feature = "export")]
pub trait MaybeEncodeAsVarULE: MaybeAsVarULE {
/// Returns the [`MaybeAsVarULE::EncodedStruct`] that represents this data struct,
/// or `None` if the data struct does not support this representation.
fn maybe_encode_as_varule(&self) -> Option<&Self::EncodedStruct>;
}
/// Implements required traits on data structs, such as [`MaybeEncodeAsVarULE`].
#[macro_export] // canonical location is crate root
macro_rules! data_struct {
(<$generic:ident: $bound:tt> $ty:path $(, $(#[$attr:meta])*)?) => {
impl<$generic: $bound> $crate::ule::MaybeAsVarULE for $ty {
type EncodedStruct = [()];
}
$($(#[$attr])*)?
impl<$generic: $bound> $crate::ule::MaybeEncodeAsVarULE for $ty {
fn maybe_encode_as_varule(&self) -> Option<&Self::EncodedStruct> {
None
}
}
};
($ty:path $(, $(#[$attr:meta])*)?) => {
impl $crate::ule::MaybeAsVarULE for $ty {
type EncodedStruct = [()];
}
$($(#[$attr])*)?
impl $crate::ule::MaybeEncodeAsVarULE for $ty {
fn maybe_encode_as_varule(&self) -> Option<&Self::EncodedStruct> {
None
}
}
};
(
$ty:ty,
varule: $varule:ty,
$(#[$attr:meta])*
encode_as_varule: $encode_as_varule:expr
) => {
impl<'data> $crate::ule::MaybeAsVarULE for $ty {
type EncodedStruct = $varule;
}
$(#[$attr])*
impl<'data> $crate::ule::MaybeEncodeAsVarULE for $ty {
fn maybe_encode_as_varule(&self) -> Option<&Self::EncodedStruct> {
// Workaround for <https://rust-lang.github.io/rfcs/3216-closure-lifetime-binder.html>
fn bind_lifetimes<F>(f: F) -> F where F: for<'data> Fn(&'data $ty) -> &'data $varule { f }
Some(bind_lifetimes($encode_as_varule)(self))
}
}
};
}
//=== Standard impls ===//
#[cfg(feature = "alloc")]
impl<'a, K0, V> MaybeAsVarULE for ZeroMap<'a, K0, V>
where
K0: ZeroMapKV<'a>,
V: ZeroMapKV<'a>,
K0: ?Sized,
V: ?Sized,
{
type EncodedStruct = [()];
}
#[cfg(feature = "alloc")]
#[cfg(feature = "export")]
impl<'a, K0, V> MaybeEncodeAsVarULE for ZeroMap<'a, K0, V>
where
K0: ZeroMapKV<'a>,
V: ZeroMapKV<'a>,
K0: ?Sized,
V: ?Sized,
{
fn maybe_encode_as_varule(&self) -> Option<&Self::EncodedStruct> {
None
}
}
#[cfg(feature = "alloc")]
impl<'a, K0, K1, V> MaybeAsVarULE for ZeroMap2d<'a, K0, K1, V>
where
K0: ZeroMapKV<'a>,
K1: ZeroMapKV<'a>,
V: ZeroMapKV<'a>,
K0: ?Sized,
K1: ?Sized,
V: ?Sized,
{
type EncodedStruct = [()];
}
#[cfg(feature = "alloc")]
#[cfg(feature = "export")]
impl<'a, K0, K1, V> MaybeEncodeAsVarULE for ZeroMap2d<'a, K0, K1, V>
where
K0: ZeroMapKV<'a>,
K1: ZeroMapKV<'a>,
V: ZeroMapKV<'a>,
K0: ?Sized,
K1: ?Sized,
V: ?Sized,
{
fn maybe_encode_as_varule(&self) -> Option<&Self::EncodedStruct> {
None
}
}
impl<T, const N: usize> MaybeAsVarULE for [T; N] {
type EncodedStruct = [()];
}
#[cfg(feature = "export")]
impl<T, const N: usize> MaybeEncodeAsVarULE for [T; N] {
fn maybe_encode_as_varule(&self) -> Option<&Self::EncodedStruct> {
None
}
}
impl MaybeAsVarULE for u16 {
type EncodedStruct = [()];
}
#[cfg(feature = "export")]
impl MaybeEncodeAsVarULE for u16 {
fn maybe_encode_as_varule(&self) -> Option<&Self::EncodedStruct> {
None
}
}