193 lines
8.1 KiB
Rust
193 lines
8.1 KiB
Rust
// Copyright 2016-2024 Brian Smith.
|
|
//
|
|
// Permission to use, copy, modify, and/or distribute this software for any
|
|
// purpose with or without fee is hereby granted, provided that the above
|
|
// copyright notice and this permission notice appear in all copies.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
|
|
// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
|
|
// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
|
|
// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
|
|
use super::CAPS_STATIC;
|
|
|
|
mod abi_assumptions {
|
|
use core::mem::size_of;
|
|
|
|
// TODO: Support ARM64_32; see
|
|
// https://github.com/briansmith/ring/issues/1832#issuecomment-1892928147. This also requires
|
|
// replacing all `cfg(target_pointer_width)` logic for non-pointer/reference things
|
|
// (`N0`, `Limb`, `LimbMask`, `crypto_word_t` etc.).
|
|
#[cfg(target_arch = "aarch64")]
|
|
const _ASSUMED_POINTER_SIZE: usize = 8;
|
|
#[cfg(target_arch = "arm")]
|
|
const _ASSUMED_POINTER_SIZE: usize = 4;
|
|
const _ASSUMED_USIZE_SIZE: () = assert!(size_of::<usize>() == _ASSUMED_POINTER_SIZE);
|
|
const _ASSUMED_REF_SIZE: () = assert!(size_of::<&'static u8>() == _ASSUMED_POINTER_SIZE);
|
|
|
|
// To support big-endian, we'd need to make several changes as described in
|
|
// https://github.com/briansmith/ring/issues/1832.
|
|
const _ASSUMED_ENDIANNESS: () = assert!(cfg!(target_endian = "little"));
|
|
}
|
|
|
|
// uclibc: When linked statically, uclibc doesn't provide getauxval.
|
|
// When linked dynamically, recent versions do provide it, but we
|
|
// want to support older versions too. Assume that if uclibc is being
|
|
// used, this is an embedded target where the user cares a lot about
|
|
// minimizing code size and also that they know in advance exactly
|
|
// what target features are supported, so rely only on static feature
|
|
// detection.
|
|
|
|
cfg_if::cfg_if! {
|
|
if #[cfg(all(all(target_arch = "aarch64", target_endian = "little"),
|
|
any(target_os = "ios", target_os = "macos", target_os = "tvos", target_os = "visionos", target_os = "watchos")))] {
|
|
mod darwin;
|
|
use darwin as detect;
|
|
} else if #[cfg(all(all(target_arch = "aarch64", target_endian = "little"), target_os = "fuchsia"))] {
|
|
mod fuchsia;
|
|
use fuchsia as detect;
|
|
} else if #[cfg(any(target_os = "android", target_os = "linux"))] {
|
|
mod linux;
|
|
use linux as detect;
|
|
} else if #[cfg(all(all(target_arch = "aarch64", target_endian = "little"), target_os = "windows"))] {
|
|
mod windows;
|
|
use windows as detect;
|
|
} else {
|
|
mod detect {
|
|
pub const FORCE_DYNAMIC_DETECTION: u32 = 0;
|
|
pub fn detect_features() -> u32 { 0 }
|
|
}
|
|
}
|
|
}
|
|
|
|
impl_get_feature! {
|
|
features: [
|
|
// TODO(MSRV): 32-bit ARM doesn't have `target_feature = "neon"` yet.
|
|
{ ("aarch64", "arm") => Neon },
|
|
|
|
// TODO(MSRV): There is no "pmull" feature listed from
|
|
// `rustc --print cfg --target=aarch64-apple-darwin`. Originally ARMv8 tied
|
|
// PMULL detection into AES detection, but later versions split it; see
|
|
// https://developer.arm.com/downloads/-/exploration-tools/feature-names-for-a-profile
|
|
// "Features introduced prior to 2020." Change this to use "pmull" when
|
|
// that is supported.
|
|
{ ("aarch64") => PMull },
|
|
|
|
{ ("aarch64") => Aes },
|
|
|
|
{ ("aarch64") => Sha256 },
|
|
|
|
// Keep in sync with `ARMV8_SHA512`.
|
|
|
|
// "sha3" is overloaded for both SHA-3 and SHA-512.
|
|
{ ("aarch64") => Sha512 },
|
|
],
|
|
}
|
|
|
|
pub(super) mod featureflags {
|
|
pub(in super::super) use super::detect::FORCE_DYNAMIC_DETECTION;
|
|
use super::*;
|
|
use crate::{
|
|
cpu,
|
|
polyfill::{once_cell::race, usize_from_u32},
|
|
};
|
|
use core::num::NonZeroUsize;
|
|
#[cfg(all(target_arch = "arm", target_endian = "little"))]
|
|
use core::sync::atomic::{AtomicU32, Ordering};
|
|
|
|
pub(in super::super) fn get_or_init() -> cpu::Features {
|
|
fn init() -> NonZeroUsize {
|
|
let detected = detect::detect_features();
|
|
let filtered = (if cfg!(feature = "unstable-testing-arm-no-hw") {
|
|
!Neon::mask()
|
|
} else {
|
|
0
|
|
}) | (if cfg!(feature = "unstable-testing-arm-no-neon") {
|
|
Neon::mask()
|
|
} else {
|
|
0
|
|
});
|
|
let detected = detected & !filtered;
|
|
let merged = CAPS_STATIC | detected;
|
|
|
|
#[cfg(all(
|
|
target_arch = "arm",
|
|
target_endian = "little",
|
|
target_has_atomic = "32"
|
|
))]
|
|
if (merged & Neon::mask()) == Neon::mask() {
|
|
// `neon_available` is declared as `alignas(4) uint32_t` in the C code.
|
|
// AtomicU32 is `#[repr(C, align(4))]`.
|
|
prefixed_extern! {
|
|
static neon_available: AtomicU32;
|
|
}
|
|
// SAFETY: The C code only reads `neon_available`, and its
|
|
// reads are synchronized through the `OnceNonZeroUsize`
|
|
// Acquire/Release semantics as we ensure we have a
|
|
// `cpu::Features` instance before calling into the C code.
|
|
let p = unsafe { &neon_available };
|
|
p.store(1, Ordering::Relaxed);
|
|
}
|
|
|
|
let merged = usize_from_u32(merged) | (1 << (Shift::Initialized as u32));
|
|
NonZeroUsize::new(merged).unwrap() // Can't fail because we just set a bit.
|
|
}
|
|
|
|
// SAFETY: This is the only caller. Any concurrent reading doesn't
|
|
// affect the safety of the writing.
|
|
let _: NonZeroUsize = FEATURES.get_or_init(init);
|
|
|
|
// SAFETY: We initialized the CPU features as required.
|
|
unsafe { cpu::Features::new_after_feature_flags_written_and_synced_unchecked() }
|
|
}
|
|
|
|
pub(in super::super) fn get(_cpu_features: cpu::Features) -> u32 {
|
|
// SAFETY: Since only `get_or_init()` could have created
|
|
// `_cpu_features`, and it only does so after `FEATURES.get_or_init()`,
|
|
// we know we are reading from `FEATURES` after initializing it.
|
|
//
|
|
// Also, 0 means "no features detected" to users, which is designed to
|
|
// be a safe configuration.
|
|
let features = FEATURES.get().map(NonZeroUsize::get).unwrap_or(0);
|
|
|
|
// The truncation is lossless, as we set the value with a u32.
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
let features = features as u32;
|
|
|
|
features
|
|
}
|
|
|
|
static FEATURES: race::OnceNonZeroUsize = race::OnceNonZeroUsize::new();
|
|
|
|
// TODO(MSRV): There is no "pmull" feature listed from
|
|
// `rustc --print cfg --target=aarch64-apple-darwin`. Originally ARMv8 tied
|
|
// PMULL detection into AES detection, but later versions split it; see
|
|
// https://developer.arm.com/downloads/-/exploration-tools/feature-names-for-a-profile
|
|
// "Features introduced prior to 2020." Change this to use "pmull" when
|
|
// that is supported.
|
|
//
|
|
// "sha3" is overloaded for both SHA-3 and SHA-512.
|
|
#[cfg(all(target_arch = "aarch64", target_endian = "little"))]
|
|
#[rustfmt::skip]
|
|
pub(in super::super) const STATIC_DETECTED: u32 = 0
|
|
| (if cfg!(target_feature = "neon") { Neon::mask() } else { 0 })
|
|
| (if cfg!(target_feature = "aes") { Aes::mask() } else { 0 })
|
|
| (if cfg!(target_feature = "aes") { PMull::mask() } else { 0 })
|
|
| (if cfg!(target_feature = "sha2") { Sha256::mask() } else { 0 })
|
|
| (if cfg!(target_feature = "sha3") { Sha512::mask() } else { 0 })
|
|
;
|
|
|
|
// TODO(MSRV): 32-bit ARM doesn't support any static feature detection yet.
|
|
#[cfg(all(target_arch = "arm", target_endian = "little"))]
|
|
pub(in super::super) const STATIC_DETECTED: u32 = 0;
|
|
}
|
|
|
|
#[allow(clippy::assertions_on_constants)]
|
|
const _AARCH64_HAS_NEON: () = assert!(
|
|
((CAPS_STATIC & Neon::mask()) == Neon::mask())
|
|
|| !cfg!(all(target_arch = "aarch64", target_endian = "little"))
|
|
);
|