chore: checkpoint before Python removal

vendor/zerovec-derive/examples/derives.rs

@@ -0,0 +1,157 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+use zerovec::ule::AsULE;
+use zerovec::ule::EncodeAsVarULE;
+use zerovec::*;
+
+#[repr(C, packed)]
+#[derive(ule::ULE, Copy, Clone)]
+pub struct FooULE {
+    a: u8,
+    b: <u32 as AsULE>::ULE,
+    c: <char as AsULE>::ULE,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct Foo {
+    a: u8,
+    b: u32,
+    c: char,
+}
+
+impl AsULE for Foo {
+    type ULE = FooULE;
+    fn to_unaligned(self) -> FooULE {
+        FooULE {
+            a: self.a,
+            b: self.b.to_unaligned(),
+            c: self.c.to_unaligned(),
+        }
+    }
+
+    fn from_unaligned(other: FooULE) -> Self {
+        Self {
+            a: other.a,
+            b: AsULE::from_unaligned(other.b),
+            c: AsULE::from_unaligned(other.c),
+        }
+    }
+}
+
+#[repr(C, packed)]
+#[derive(ule::VarULE)]
+pub struct RelationULE {
+    /// This maps to (AndOr, Polarity, Operand),
+    /// with the first bit mapping to AndOr (1 == And), the second bit
+    /// to Polarity (1 == Positive), and the remaining bits to Operand
+    /// encoded via Operand::encode. It is unsound for the Operand bits to
+    /// not be a valid encoded Operand.
+    andor_polarity_operand: u8,
+    modulo: <u32 as AsULE>::ULE,
+    range_list: ZeroSlice<Foo>,
+}
+
+#[derive(Clone, PartialEq, Debug)]
+pub struct Relation<'a> {
+    andor_polarity_operand: u8,
+    modulo: u32,
+    range_list: ZeroVec<'a, Foo>,
+}
+
+unsafe impl EncodeAsVarULE<RelationULE> for Relation<'_> {
+    fn encode_var_ule_as_slices<R>(&self, cb: impl FnOnce(&[&[u8]]) -> R) -> R {
+        cb(&[
+            &[self.andor_polarity_operand],
+            ule::ULE::slice_as_bytes(&[self.modulo.to_unaligned()]),
+            self.range_list.as_bytes(),
+        ])
+    }
+}
+
+impl RelationULE {
+    pub fn as_relation(&self) -> Relation<'_> {
+        Relation {
+            andor_polarity_operand: self.andor_polarity_operand,
+            modulo: u32::from_unaligned(self.modulo),
+            range_list: self.range_list.as_zerovec(),
+        }
+    }
+}
+
+const TEST_SLICE: &[Foo] = &[
+    Foo {
+        a: 101,
+        b: 924,
+        c: '⸘',
+    },
+    Foo {
+        a: 217,
+        b: 4228,
+        c: 'ə',
+    },
+    Foo {
+        a: 117,
+        b: 9090,
+        c: 'ø',
+    },
+];
+
+const TEST_SLICE2: &[Foo] = &[
+    Foo {
+        a: 92,
+        b: 4,
+        c: 'å',
+    },
+    Foo {
+        a: 9,
+        b: 49993,
+        c: '±',
+    },
+];
+fn test_zerovec() {
+    let zerovec: ZeroVec<Foo> = TEST_SLICE.iter().copied().collect();
+
+    assert_eq!(zerovec, TEST_SLICE);
+
+    let bytes = zerovec.as_bytes();
+    let reparsed: ZeroVec<Foo> = ZeroVec::parse_bytes(bytes).expect("Parsing should succeed");
+
+    assert_eq!(reparsed, TEST_SLICE);
+}
+
+fn test_varzerovec() {
+    let relation1 = Relation {
+        andor_polarity_operand: 1,
+        modulo: 5004,
+        range_list: TEST_SLICE.iter().copied().collect(),
+    };
+    let relation2 = Relation {
+        andor_polarity_operand: 5,
+        modulo: 909,
+        range_list: TEST_SLICE2.iter().copied().collect(),
+    };
+
+    let relations = &[relation1, relation2];
+
+    let vzv = VarZeroVec::<_>::from(relations);
+
+    for (ule, stack) in vzv.iter().zip(relations.iter()) {
+        assert_eq!(*stack, ule.as_relation());
+    }
+
+    let bytes = vzv.as_bytes();
+
+    let recovered: VarZeroVec<RelationULE> =
+        VarZeroVec::parse_bytes(bytes).expect("Parsing should succeed");
+
+    for (ule, stack) in recovered.iter().zip(relations.iter()) {
+        assert_eq!(*stack, ule.as_relation());
+    }
+}
+
+fn main() {
+    test_zerovec();
+    test_varzerovec();
+}

vendor/zerovec-derive/examples/make.rs

@@ -0,0 +1,125 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+use std::fmt::Debug;
+use ule::ULE;
+use zerovec::*;
+
+#[make_ule(StructULE)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Struct {
+    a: u8,
+    b: u32,
+    c: Option<char>,
+}
+
+#[make_ule(HashedStructULE)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[zerovec::derive(Debug, Hash)]
+pub struct HashedStruct {
+    a: u64,
+    b: i16,
+    c: Option<char>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[make_ule(TupleStructULE)]
+pub struct TupleStruct(u8, char);
+
+#[make_ule(EnumULE)]
+#[repr(u8)]
+#[derive(Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Debug)]
+#[zerovec::derive(Debug, Hash)]
+pub enum Enum {
+    A = 0,
+    B = 1,
+    D = 2,
+    E = 3,
+    FooBar = 4,
+    F = 5,
+}
+
+#[make_ule(OutOfOrderMissingZeroEnumULE)]
+#[repr(u8)]
+#[derive(Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Debug)]
+#[allow(unused)]
+pub enum OutOfOrderMissingZeroEnum {
+    E = 3,
+    B = 1,
+    FooBar = 4,
+    D = 2,
+    F = 5,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Ord, PartialOrd)]
+#[make_ule(NoKVULE)]
+#[zerovec::skip_derive(ZeroMapKV)]
+pub struct NoKV(u8, char);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[make_ule(NoOrdULE)]
+#[zerovec::skip_derive(ZeroMapKV, Ord)]
+pub struct NoOrd(u8, char);
+
+fn test_zerovec<T: ule::AsULE + Debug + PartialEq>(slice: &[T]) {
+    let zerovec: ZeroVec<T> = slice.iter().copied().collect();
+
+    assert_eq!(zerovec, slice);
+
+    let bytes = zerovec.as_bytes();
+    let name = std::any::type_name::<T>();
+    let reparsed: ZeroVec<T> =
+        ZeroVec::parse_bytes(bytes).unwrap_or_else(|_| panic!("Parsing {name} should succeed"));
+
+    assert_eq!(reparsed, slice);
+}
+
+fn main() {
+    test_zerovec(TEST_SLICE_STRUCT);
+    test_zerovec(TEST_SLICE_TUPLESTRUCT);
+    test_zerovec(TEST_SLICE_ENUM);
+
+    assert!(EnumULE::parse_bytes_to_slice(&[0]).is_ok());
+    assert!(EnumULE::parse_bytes_to_slice(&[1]).is_ok());
+    assert!(EnumULE::parse_bytes_to_slice(&[5]).is_ok());
+    assert!(EnumULE::parse_bytes_to_slice(&[6]).is_err());
+    assert!(OutOfOrderMissingZeroEnumULE::parse_bytes_to_slice(&[0]).is_err());
+    assert!(OutOfOrderMissingZeroEnumULE::parse_bytes_to_slice(&[1]).is_ok());
+    assert!(OutOfOrderMissingZeroEnumULE::parse_bytes_to_slice(&[5]).is_ok());
+    assert!(OutOfOrderMissingZeroEnumULE::parse_bytes_to_slice(&[6]).is_err());
+}
+
+const TEST_SLICE_STRUCT: &[Struct] = &[
+    Struct {
+        a: 101,
+        b: 924,
+        c: Some('⸘'),
+    },
+    Struct {
+        a: 217,
+        b: 4228,
+        c: Some('ə'),
+    },
+    Struct {
+        a: 117,
+        b: 9090,
+        c: Some('ø'),
+    },
+];
+
+const TEST_SLICE_TUPLESTRUCT: &[TupleStruct] = &[
+    TupleStruct(101, 'ř'),
+    TupleStruct(76, '°'),
+    TupleStruct(15, 'a'),
+];
+
+const TEST_SLICE_ENUM: &[Enum] = &[
+    Enum::A,
+    Enum::FooBar,
+    Enum::F,
+    Enum::D,
+    Enum::B,
+    Enum::FooBar,
+    Enum::E,
+];

vendor/zerovec-derive/examples/make_var.rs

@@ -0,0 +1,261 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+use std::borrow::Cow;
+use std::fmt::Debug;
+
+use zerofrom::ZeroFrom;
+use zerovec::{ule::AsULE, *};
+
+#[make_varule(VarStructULE)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, serde::Serialize, serde::Deserialize)]
+#[zerovec::derive(Serialize, Deserialize, Debug)]
+struct VarStruct<'a> {
+    a: u32,
+    b: char,
+    #[serde(borrow)]
+    c: Cow<'a, str>,
+}
+
+#[make_varule(VarStructOutOfOrderULE)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, serde::Serialize, serde::Deserialize)]
+#[zerovec::derive(Serialize, Deserialize, Debug)]
+struct VarStructOutOfOrder<'a> {
+    a: u32,
+    #[serde(borrow)]
+    b: Cow<'a, str>,
+    c: char,
+    d: u8,
+}
+
+#[make_varule(VarTupleStructULE)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, serde::Serialize, serde::Deserialize)]
+#[zerovec::derive(Serialize, Deserialize, Debug)]
+struct VarTupleStruct<'a>(u32, char, #[serde(borrow)] VarZeroVec<'a, str>);
+
+#[make_varule(NoKVULE)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, serde::Serialize, serde::Deserialize)]
+#[zerovec::skip_derive(ZeroMapKV)]
+#[zerovec::derive(Serialize, Deserialize, Debug)]
+struct NoKV<'a>(u32, char, #[serde(borrow)] VarZeroVec<'a, str>);
+
+#[make_varule(NoOrdULE)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, serde::Serialize, serde::Deserialize)]
+#[zerovec::skip_derive(ZeroMapKV, Ord)]
+#[zerovec::derive(Serialize, Deserialize, Debug)]
+struct NoOrd<'a>(u32, char, #[serde(borrow)] VarZeroVec<'a, str>);
+
+#[make_varule(MultiFieldStructULE)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, serde::Serialize, serde::Deserialize)]
+#[zerovec::derive(Serialize, Deserialize, Debug)]
+struct MultiFieldStruct<'a> {
+    a: u32,
+    b: char,
+    #[serde(borrow)]
+    c: Cow<'a, str>,
+    d: u8,
+    #[serde(borrow)]
+    e: Cow<'a, str>,
+    f: char,
+}
+
+#[make_varule(MultiFieldConsecutiveStructULE)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, serde::Serialize, serde::Deserialize)]
+#[zerovec::derive(Serialize, Deserialize, Debug)]
+#[zerovec::format(zerovec::vecs::Index8)]
+struct MultiFieldConsecutiveStruct<'a> {
+    #[serde(borrow)]
+    a: Cow<'a, str>,
+    #[serde(borrow)]
+    b: Cow<'a, str>,
+    #[serde(borrow)]
+    c: Cow<'a, str>,
+    #[serde(borrow)]
+    d: Cow<'a, str>,
+}
+
+#[make_varule(CustomVarFieldULE)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, serde::Serialize, serde::Deserialize)]
+#[zerovec::derive(Serialize, Deserialize, Debug)]
+struct CustomVarField<'a> {
+    #[zerovec::varule(MultiFieldStructULE)]
+    #[serde(borrow)]
+    a: MultiFieldStruct<'a>,
+    b: u32,
+}
+
+#[make_varule(MultiFieldTupleULE)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug, serde::Serialize, serde::Deserialize)]
+#[zerovec::derive(Serialize, Deserialize, Debug)]
+#[zerovec::format(zerovec::vecs::Index32)]
+struct MultiFieldTuple<'a>(
+    u8,
+    char,
+    #[serde(borrow)] VarZeroVec<'a, str>,
+    #[serde(borrow)] VarZeroVec<'a, [u8]>,
+    #[serde(borrow)] Cow<'a, str>,
+);
+
+/// The `assert` function should have the body `|(stack, zero)| assert_eq!(stack, &U::zero_from(&zero))`
+///
+/// We cannot do this internally because we technically need a different `U` with a shorter lifetime here
+/// which would require some gnarly lifetime bounds and perhaps a Yoke dependency. This is just a test, so it's
+/// not important to get this 100% perfect
+fn assert_zerovec<T, U, F>(slice: &[U], assert: F)
+where
+    T: ule::VarULE + ?Sized + serde::Serialize,
+    U: ule::EncodeAsVarULE<T> + serde::Serialize,
+    F: Fn(&U, &T),
+    for<'a> Box<T>: serde::Deserialize<'a>,
+    for<'a> &'a T: serde::Deserialize<'a>,
+    T: PartialEq + Debug,
+{
+    let varzerovec: VarZeroVec<T> = slice.into();
+
+    assert_eq!(varzerovec.len(), slice.len());
+
+    for (stack, zero) in slice.iter().zip(varzerovec.iter()) {
+        assert(stack, zero)
+    }
+
+    let bytes = varzerovec.as_bytes();
+    let name = std::any::type_name::<T>();
+    let reparsed: VarZeroVec<T> = VarZeroVec::parse_bytes(bytes)
+        .unwrap_or_else(|_| panic!("Parsing VarZeroVec<{name}> should succeed"));
+
+    assert_eq!(reparsed.len(), slice.len());
+
+    for (stack, zero) in slice.iter().zip(reparsed.iter()) {
+        assert(stack, zero)
+    }
+
+    let bincode = bincode::serialize(&varzerovec).unwrap();
+    let deserialized: VarZeroVec<T> = bincode::deserialize(&bincode).unwrap();
+
+    for (stack, zero) in slice.iter().zip(deserialized.iter()) {
+        assert(stack, zero)
+    }
+
+    let json_slice = serde_json::to_string(&slice).unwrap();
+    let json_vzv = serde_json::to_string(&varzerovec).unwrap();
+
+    assert_eq!(json_slice, json_vzv);
+
+    let deserialized: VarZeroVec<T> = serde_json::from_str(&json_vzv).unwrap();
+
+    for (stack, zero) in slice.iter().zip(deserialized.iter()) {
+        assert(stack, zero)
+    }
+
+    if let Some(first) = varzerovec.get(0) {
+        let bincode = bincode::serialize(first).unwrap();
+        let deserialized: &T = bincode::deserialize(&bincode).unwrap();
+        let deserialized_box: Box<T> = bincode::deserialize(&bincode).unwrap();
+        assert_eq!(
+            first, deserialized,
+            "Single element roundtrips with bincode"
+        );
+        assert_eq!(
+            first, &*deserialized_box,
+            "Single element roundtrips with bincode"
+        );
+
+        let json = serde_json::to_string(first).unwrap();
+        let deserialized: Box<T> = serde_json::from_str(&json).unwrap();
+        assert_eq!(
+            first, &*deserialized,
+            "Single element roundtrips with serde"
+        );
+    }
+}
+
+fn main() {
+    assert_zerovec::<VarStructULE, VarStruct, _>(TEST_VARSTRUCTS, |stack, zero| {
+        assert_eq!(stack, &VarStruct::zero_from(zero))
+    });
+
+    assert_zerovec::<MultiFieldStructULE, MultiFieldStruct, _>(TEST_MULTIFIELD, |stack, zero| {
+        assert_eq!(stack, &MultiFieldStruct::zero_from(zero))
+    });
+
+    assert_zerovec::<MultiFieldConsecutiveStructULE, MultiFieldConsecutiveStruct, _>(
+        TEST_MULTICONSECUTIVE,
+        |stack, zero| assert_eq!(stack, &MultiFieldConsecutiveStruct::zero_from(zero)),
+    );
+
+    let vartuples = &[
+        VarTupleStruct(101, 'ø', TEST_STRINGS1.into()),
+        VarTupleStruct(9499, '⸘', TEST_STRINGS2.into()),
+        VarTupleStruct(3478, '月', TEST_STRINGS3.into()),
+    ];
+    assert_zerovec::<VarTupleStructULE, VarTupleStruct, _>(vartuples, |stack, zero| {
+        assert_eq!(stack, &VarTupleStruct::zero_from(zero))
+    });
+
+    // Test that all fields are accessible on a type using multifieldule
+    let multi_ule = ule::encode_varule_to_box(&TEST_MULTIFIELD[0]);
+    assert_eq!(multi_ule.a, TEST_MULTIFIELD[0].a.to_unaligned());
+    assert_eq!(multi_ule.b, TEST_MULTIFIELD[0].b.to_unaligned());
+    assert_eq!(multi_ule.c(), TEST_MULTIFIELD[0].c);
+    assert_eq!(multi_ule.d, TEST_MULTIFIELD[0].d);
+    assert_eq!(multi_ule.e(), TEST_MULTIFIELD[0].e);
+    assert_eq!(multi_ule.f, TEST_MULTIFIELD[0].f.to_unaligned());
+}
+
+const TEST_VARSTRUCTS: &[VarStruct<'static>] = &[
+    VarStruct {
+        a: 101,
+        b: 'ø',
+        c: Cow::Borrowed("testīng strīng"),
+    },
+    VarStruct {
+        a: 9499,
+        b: '⸘',
+        c: Cow::Borrowed("a diffərənt ştring"),
+    },
+    VarStruct {
+        a: 3478,
+        b: '月',
+        c: Cow::Borrowed("好多嘅 string"),
+    },
+];
+
+const TEST_STRINGS1: &[&str] = &["foo", "bar", "baz"];
+const TEST_STRINGS2: &[&str] = &["hellø", "wørłd"];
+const TEST_STRINGS3: &[&str] = &["łořem", "ɨpsu₥"];
+
+const TEST_MULTIFIELD: &[MultiFieldStruct<'static>] = &[
+    MultiFieldStruct {
+        a: 101,
+        b: 'ø',
+        c: Cow::Borrowed("testīng strīng"),
+        d: 8,
+        e: Cow::Borrowed("another testīng strīng"),
+        f: 'å',
+    },
+    MultiFieldStruct {
+        a: 9499,
+        b: '⸘',
+        c: Cow::Borrowed("a diffərənt ştring"),
+        d: 120,
+        e: Cow::Borrowed("a diffərənt testing ştring"),
+        f: 'ł',
+    },
+    MultiFieldStruct {
+        a: 3478,
+        b: '月',
+        c: Cow::Borrowed("好多嘅 string"),
+        d: 89,
+        e: Cow::Borrowed("many 好多嘅 string"),
+        f: 'ə',
+    },
+];
+
+const TEST_MULTICONSECUTIVE: &[MultiFieldConsecutiveStruct<'static>] =
+    &[MultiFieldConsecutiveStruct {
+        a: Cow::Borrowed("one"),
+        b: Cow::Borrowed("2"),
+        c: Cow::Borrowed("three"),
+        d: Cow::Borrowed("four"),
+    }];