diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-17 12:02:58 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-17 12:02:58 +0000 |
| commit | 698f8c2f01ea549d77d7dc3338a12e04c11057b9 (patch) | |
| tree | 173a775858bd501c378080a10dca74132f05bc50 /compiler/rustc_data_structures | |
| parent | Initial commit. (diff) | |
| download | rustc-698f8c2f01ea549d77d7dc3338a12e04c11057b9.tar.xz rustc-698f8c2f01ea549d77d7dc3338a12e04c11057b9.zip | |
Adding upstream version 1.64.0+dfsg1.upstream/1.64.0+dfsg1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'compiler/rustc_data_structures')
81 files changed, 14721 insertions, 0 deletions
diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml new file mode 100644 index 000000000..5c641f54f --- /dev/null +++ b/compiler/rustc_data_structures/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "rustc_data_structures" +version = "0.0.0" +edition = "2021" + +[lib] +doctest = false + +[dependencies] +arrayvec = { version = "0.7", default-features = false } +ena = "0.14" +indexmap = { version = "1.9.1" } +tracing = "0.1" +jobserver_crate = { version = "0.1.13", package = "jobserver" } +rustc_serialize = { path = "../rustc_serialize" } +rustc_macros = { path = "../rustc_macros" } +rustc_graphviz = { path = "../rustc_graphviz" } +cfg-if = "0.1.2" +stable_deref_trait = "1.0.0" +rayon = { version = "0.4.0", package = "rustc-rayon", optional = true } +rayon-core = { version = "0.4.0", package = "rustc-rayon-core", optional = true } +rustc-hash = "1.1.0" +smallvec = { version = "1.8.1", features = ["const_generics", "union", "may_dangle"] } +rustc_index = { path = "../rustc_index", package = "rustc_index" } +bitflags = "1.2.1" +measureme = "10.0.0" +libc = "0.2" +stacker = "0.1.14" +tempfile = "3.2" + +[dependencies.parking_lot] +version = "0.11" + +[target.'cfg(windows)'.dependencies] +winapi = { version = "0.3", features = ["fileapi", "psapi", "winerror"] } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +memmap2 = "0.2.1" + +[features] +rustc_use_parallel_compiler = ["indexmap/rustc-rayon", "rayon", "rayon-core"] diff --git a/compiler/rustc_data_structures/src/atomic_ref.rs b/compiler/rustc_data_structures/src/atomic_ref.rs new file mode 100644 index 000000000..eeb1b3092 --- /dev/null +++ b/compiler/rustc_data_structures/src/atomic_ref.rs @@ -0,0 +1,26 @@ +use std::marker::PhantomData; +use std::sync::atomic::{AtomicPtr, Ordering}; + +/// This is essentially an `AtomicPtr` but is guaranteed to always be valid +pub struct AtomicRef<T: 'static>(AtomicPtr<T>, PhantomData<&'static T>); + +impl<T: 'static> AtomicRef<T> { + pub const fn new(initial: &'static T) -> AtomicRef<T> { + AtomicRef(AtomicPtr::new(initial as *const T as *mut T), PhantomData) + } + + pub fn swap(&self, new: &'static T) -> &'static T { + // We never allow storing anything but a `'static` reference so it's safe to + // return it for the same. + unsafe { &*self.0.swap(new as *const T as *mut T, Ordering::SeqCst) } + } +} + +impl<T: 'static> std::ops::Deref for AtomicRef<T> { + type Target = T; + fn deref(&self) -> &Self::Target { + // We never allow storing anything but a `'static` reference so it's safe to lend + // it out for any amount of time. + unsafe { &*self.0.load(Ordering::SeqCst) } + } +} diff --git a/compiler/rustc_data_structures/src/base_n.rs b/compiler/rustc_data_structures/src/base_n.rs new file mode 100644 index 000000000..3c7bea271 --- /dev/null +++ b/compiler/rustc_data_structures/src/base_n.rs @@ -0,0 +1,42 @@ +/// Converts unsigned integers into a string representation with some base. +/// Bases up to and including 36 can be used for case-insensitive things. +use std::str; + +#[cfg(test)] +mod tests; + +pub const MAX_BASE: usize = 64; +pub const ALPHANUMERIC_ONLY: usize = 62; +pub const CASE_INSENSITIVE: usize = 36; + +const BASE_64: &[u8; MAX_BASE as usize] = + b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ@$"; + +#[inline] +pub fn push_str(mut n: u128, base: usize, output: &mut String) { + debug_assert!(base >= 2 && base <= MAX_BASE); + let mut s = [0u8; 128]; + let mut index = 0; + + let base = base as u128; + + loop { + s[index] = BASE_64[(n % base) as usize]; + index += 1; + n /= base; + + if n == 0 { + break; + } + } + s[0..index].reverse(); + + output.push_str(str::from_utf8(&s[0..index]).unwrap()); +} + +#[inline] +pub fn encode(n: u128, base: usize) -> String { + let mut s = String::new(); + push_str(n, base, &mut s); + s +} diff --git a/compiler/rustc_data_structures/src/base_n/tests.rs b/compiler/rustc_data_structures/src/base_n/tests.rs new file mode 100644 index 000000000..2be2f0532 --- /dev/null +++ b/compiler/rustc_data_structures/src/base_n/tests.rs @@ -0,0 +1,24 @@ +use super::*; + +#[test] +fn test_encode() { + fn test(n: u128, base: usize) { + assert_eq!(Ok(n), u128::from_str_radix(&encode(n, base), base as u32)); + } + + for base in 2..37 { + test(0, base); + test(1, base); + test(35, base); + test(36, base); + test(37, base); + test(u64::MAX as u128, base); + test(u128::MAX, base); + + const N: u128 = if cfg!(miri) { 10 } else { 1000 }; + + for i in 0..N { + test(i * 983, base); + } + } +} diff --git a/compiler/rustc_data_structures/src/binary_search_util/mod.rs b/compiler/rustc_data_structures/src/binary_search_util/mod.rs new file mode 100644 index 000000000..d40172a2e --- /dev/null +++ b/compiler/rustc_data_structures/src/binary_search_util/mod.rs @@ -0,0 +1,68 @@ +#[cfg(test)] +mod tests; + +/// Uses a sorted slice `data: &[E]` as a kind of "multi-map". The +/// `key_fn` extracts a key of type `K` from the data, and this +/// function finds the range of elements that match the key. `data` +/// must have been sorted as if by a call to `sort_by_key` for this to +/// work. +pub fn binary_search_slice<'d, E, K>(data: &'d [E], key_fn: impl Fn(&E) -> K, key: &K) -> &'d [E] +where + K: Ord, +{ + let Ok(mid) = data.binary_search_by_key(key, &key_fn) else { + return &[]; + }; + let size = data.len(); + + // We get back *some* element with the given key -- so do + // a galloping search backwards to find the *first* one. + let mut start = mid; + let mut previous = mid; + let mut step = 1; + loop { + start = start.saturating_sub(step); + if start == 0 || key_fn(&data[start]) != *key { + break; + } + previous = start; + step *= 2; + } + step = previous - start; + while step > 1 { + let half = step / 2; + let mid = start + half; + if key_fn(&data[mid]) != *key { + start = mid; + } + step -= half; + } + // adjust by one if we have overshot + if start < size && key_fn(&data[start]) != *key { + start += 1; + } + + // Now search forward to find the *last* one. + let mut end = mid; + let mut previous = mid; + let mut step = 1; + loop { + end = end.saturating_add(step).min(size); + if end == size || key_fn(&data[end]) != *key { + break; + } + previous = end; + step *= 2; + } + step = end - previous; + while step > 1 { + let half = step / 2; + let mid = end - half; + if key_fn(&data[mid]) != *key { + end = mid; + } + step -= half; + } + + &data[start..end] +} diff --git a/compiler/rustc_data_structures/src/binary_search_util/tests.rs b/compiler/rustc_data_structures/src/binary_search_util/tests.rs new file mode 100644 index 000000000..d74febb5c --- /dev/null +++ b/compiler/rustc_data_structures/src/binary_search_util/tests.rs @@ -0,0 +1,23 @@ +use super::*; + +type Element = (usize, &'static str); + +fn test_map() -> Vec<Element> { + let mut data = vec![(3, "three-a"), (0, "zero"), (3, "three-b"), (22, "twenty-two")]; + data.sort_by_key(get_key); + data +} + +fn get_key(data: &Element) -> usize { + data.0 +} + +#[test] +fn binary_search_slice_test() { + let map = test_map(); + assert_eq!(binary_search_slice(&map, get_key, &0), &[(0, "zero")]); + assert_eq!(binary_search_slice(&map, get_key, &1), &[]); + assert_eq!(binary_search_slice(&map, get_key, &3), &[(3, "three-a"), (3, "three-b")]); + assert_eq!(binary_search_slice(&map, get_key, &22), &[(22, "twenty-two")]); + assert_eq!(binary_search_slice(&map, get_key, &23), &[]); +} diff --git a/compiler/rustc_data_structures/src/captures.rs b/compiler/rustc_data_structures/src/captures.rs new file mode 100644 index 000000000..677ccb314 --- /dev/null +++ b/compiler/rustc_data_structures/src/captures.rs @@ -0,0 +1,8 @@ +/// "Signaling" trait used in impl trait to tag lifetimes that you may +/// need to capture but don't really need for other reasons. +/// Basically a workaround; see [this comment] for details. +/// +/// [this comment]: https://github.com/rust-lang/rust/issues/34511#issuecomment-373423999 +pub trait Captures<'a> {} + +impl<'a, T: ?Sized> Captures<'a> for T {} diff --git a/compiler/rustc_data_structures/src/fingerprint.rs b/compiler/rustc_data_structures/src/fingerprint.rs new file mode 100644 index 000000000..5ff2d18dd --- /dev/null +++ b/compiler/rustc_data_structures/src/fingerprint.rs @@ -0,0 +1,215 @@ +use crate::stable_hasher; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; +use std::convert::TryInto; +use std::hash::{Hash, Hasher}; + +#[cfg(test)] +mod tests; + +#[derive(Eq, PartialEq, Ord, PartialOrd, Debug, Clone, Copy)] +#[repr(C)] +pub struct Fingerprint(u64, u64); + +impl Fingerprint { + pub const ZERO: Fingerprint = Fingerprint(0, 0); + + #[inline] + pub fn new(_0: u64, _1: u64) -> Fingerprint { + Fingerprint(_0, _1) + } + + #[inline] + pub fn from_smaller_hash(hash: u64) -> Fingerprint { + Fingerprint(hash, hash) + } + + #[inline] + pub fn to_smaller_hash(&self) -> u64 { + // Even though both halves of the fingerprint are expected to be good + // quality hash values, let's still combine the two values because the + // Fingerprints in DefPathHash have the StableCrateId portion which is + // the same for all DefPathHashes from the same crate. Combining the + // two halfs makes sure we get a good quality hash in such cases too. + self.0.wrapping_mul(3).wrapping_add(self.1) + } + + #[inline] + pub fn as_value(&self) -> (u64, u64) { + (self.0, self.1) + } + + #[inline] + pub fn combine(self, other: Fingerprint) -> Fingerprint { + // See https://stackoverflow.com/a/27952689 on why this function is + // implemented this way. + Fingerprint( + self.0.wrapping_mul(3).wrapping_add(other.0), + self.1.wrapping_mul(3).wrapping_add(other.1), + ) + } + + // Combines two hashes in an order independent way. Make sure this is what + // you want. + #[inline] + pub fn combine_commutative(self, other: Fingerprint) -> Fingerprint { + let a = u128::from(self.1) << 64 | u128::from(self.0); + let b = u128::from(other.1) << 64 | u128::from(other.0); + + let c = a.wrapping_add(b); + + Fingerprint(c as u64, (c >> 64) as u64) + } + + pub fn to_hex(&self) -> String { + format!("{:x}{:x}", self.0, self.1) + } + + #[inline] + pub fn to_le_bytes(&self) -> [u8; 16] { + // This seems to optimize to the same machine code as + // `unsafe { mem::transmute(*k) }`. Well done, LLVM! :) + let mut result = [0u8; 16]; + + let first_half: &mut [u8; 8] = (&mut result[0..8]).try_into().unwrap(); + *first_half = self.0.to_le_bytes(); + + let second_half: &mut [u8; 8] = (&mut result[8..16]).try_into().unwrap(); + *second_half = self.1.to_le_bytes(); + + result + } + + #[inline] + pub fn from_le_bytes(bytes: [u8; 16]) -> Fingerprint { + Fingerprint( + u64::from_le_bytes(bytes[0..8].try_into().unwrap()), + u64::from_le_bytes(bytes[8..16].try_into().unwrap()), + ) + } +} + +impl std::fmt::Display for Fingerprint { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "{:x}-{:x}", self.0, self.1) + } +} + +impl Hash for Fingerprint { + #[inline] + fn hash<H: Hasher>(&self, state: &mut H) { + state.write_fingerprint(self); + } +} + +trait FingerprintHasher { + fn write_fingerprint(&mut self, fingerprint: &Fingerprint); +} + +impl<H: Hasher> FingerprintHasher for H { + #[inline] + default fn write_fingerprint(&mut self, fingerprint: &Fingerprint) { + self.write_u64(fingerprint.0); + self.write_u64(fingerprint.1); + } +} + +impl FingerprintHasher for crate::unhash::Unhasher { + #[inline] + fn write_fingerprint(&mut self, fingerprint: &Fingerprint) { + // Even though both halves of the fingerprint are expected to be good + // quality hash values, let's still combine the two values because the + // Fingerprints in DefPathHash have the StableCrateId portion which is + // the same for all DefPathHashes from the same crate. Combining the + // two halfs makes sure we get a good quality hash in such cases too. + // + // Since `Unhasher` is used only in the context of HashMaps, it is OK + // to combine the two components in an order-independent way (which is + // cheaper than the more robust Fingerprint::to_smaller_hash()). For + // HashMaps we don't really care if Fingerprint(x,y) and + // Fingerprint(y, x) result in the same hash value. Collision + // probability will still be much better than with FxHash. + self.write_u64(fingerprint.0.wrapping_add(fingerprint.1)); + } +} + +impl stable_hasher::StableHasherResult for Fingerprint { + #[inline] + fn finish(hasher: stable_hasher::StableHasher) -> Self { + let (_0, _1) = hasher.finalize(); + Fingerprint(_0, _1) + } +} + +impl_stable_hash_via_hash!(Fingerprint); + +impl<E: Encoder> Encodable<E> for Fingerprint { + #[inline] + fn encode(&self, s: &mut E) { + s.emit_raw_bytes(&self.to_le_bytes()); + } +} + +impl<D: Decoder> Decodable<D> for Fingerprint { + #[inline] + fn decode(d: &mut D) -> Self { + Fingerprint::from_le_bytes(d.read_raw_bytes(16).try_into().unwrap()) + } +} + +// `PackedFingerprint` wraps a `Fingerprint`. Its purpose is to, on certain +// architectures, behave like a `Fingerprint` without alignment requirements. +// This behavior is only enabled on x86 and x86_64, where the impact of +// unaligned accesses is tolerable in small doses. +// +// This may be preferable to use in large collections of structs containing +// fingerprints, as it can reduce memory consumption by preventing the padding +// that the more strictly-aligned `Fingerprint` can introduce. An application of +// this is in the query dependency graph, which contains a large collection of +// `DepNode`s. As of this writing, the size of a `DepNode` decreases by ~30% +// (from 24 bytes to 17) by using the packed representation here, which +// noticeably decreases total memory usage when compiling large crates. +// +// The wrapped `Fingerprint` is private to reduce the chance of a client +// invoking undefined behavior by taking a reference to the packed field. +#[cfg_attr(any(target_arch = "x86", target_arch = "x86_64"), repr(packed))] +#[derive(Eq, PartialEq, Ord, PartialOrd, Debug, Clone, Copy, Hash)] +pub struct PackedFingerprint(Fingerprint); + +impl std::fmt::Display for PackedFingerprint { + #[inline] + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Copy to avoid taking reference to packed field. + let copy = self.0; + copy.fmt(formatter) + } +} + +impl<E: Encoder> Encodable<E> for PackedFingerprint { + #[inline] + fn encode(&self, s: &mut E) { + // Copy to avoid taking reference to packed field. + let copy = self.0; + copy.encode(s); + } +} + +impl<D: Decoder> Decodable<D> for PackedFingerprint { + #[inline] + fn decode(d: &mut D) -> Self { + Self(Fingerprint::decode(d)) + } +} + +impl From<Fingerprint> for PackedFingerprint { + #[inline] + fn from(f: Fingerprint) -> PackedFingerprint { + PackedFingerprint(f) + } +} + +impl From<PackedFingerprint> for Fingerprint { + #[inline] + fn from(f: PackedFingerprint) -> Fingerprint { + f.0 + } +} diff --git a/compiler/rustc_data_structures/src/fingerprint/tests.rs b/compiler/rustc_data_structures/src/fingerprint/tests.rs new file mode 100644 index 000000000..9b0783e33 --- /dev/null +++ b/compiler/rustc_data_structures/src/fingerprint/tests.rs @@ -0,0 +1,14 @@ +use super::*; + +// Check that `combine_commutative` is order independent. +#[test] +fn combine_commutative_is_order_independent() { + let a = Fingerprint::new(0xf6622fb349898b06, 0x70be9377b2f9c610); + let b = Fingerprint::new(0xa9562bf5a2a5303c, 0x67d9b6c82034f13d); + let c = Fingerprint::new(0x0d013a27811dbbc3, 0x9a3f7b3d9142ec43); + let permutations = [(a, b, c), (a, c, b), (b, a, c), (b, c, a), (c, a, b), (c, b, a)]; + let f = a.combine_commutative(b).combine_commutative(c); + for p in &permutations { + assert_eq!(f, p.0.combine_commutative(p.1).combine_commutative(p.2)); + } +} diff --git a/compiler/rustc_data_structures/src/flock.rs b/compiler/rustc_data_structures/src/flock.rs new file mode 100644 index 000000000..e395d8dbb --- /dev/null +++ b/compiler/rustc_data_structures/src/flock.rs @@ -0,0 +1,26 @@ +//! Simple file-locking apis for each OS. +//! +//! This is not meant to be in the standard library, it does nothing with +//! green/native threading. This is just a bare-bones enough solution for +//! librustdoc, it is not production quality at all. + +#![allow(non_camel_case_types)] +#![allow(nonstandard_style)] + +cfg_if! { + if #[cfg(target_os = "linux")] { + mod linux; + use linux as imp; + } else if #[cfg(unix)] { + mod unix; + use unix as imp; + } else if #[cfg(windows)] { + mod windows; + use windows as imp; + } else { + mod unsupported; + use unsupported as imp; + } +} + +pub use imp::Lock; diff --git a/compiler/rustc_data_structures/src/flock/linux.rs b/compiler/rustc_data_structures/src/flock/linux.rs new file mode 100644 index 000000000..bb3ecfbc3 --- /dev/null +++ b/compiler/rustc_data_structures/src/flock/linux.rs @@ -0,0 +1,40 @@ +//! We use `flock` rather than `fcntl` on Linux, because WSL1 does not support +//! `fcntl`-style advisory locks properly (rust-lang/rust#72157). For other Unix +//! targets we still use `fcntl` because it's more portable than `flock`. + +use std::fs::{File, OpenOptions}; +use std::io; +use std::os::unix::prelude::*; +use std::path::Path; + +#[derive(Debug)] +pub struct Lock { + _file: File, +} + +impl Lock { + pub fn new(p: &Path, wait: bool, create: bool, exclusive: bool) -> io::Result<Lock> { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(create) + .mode(libc::S_IRWXU as u32) + .open(p)?; + + let mut operation = if exclusive { libc::LOCK_EX } else { libc::LOCK_SH }; + if !wait { + operation |= libc::LOCK_NB + } + + let ret = unsafe { libc::flock(file.as_raw_fd(), operation) }; + if ret == -1 { Err(io::Error::last_os_error()) } else { Ok(Lock { _file: file }) } + } + + pub fn error_unsupported(err: &io::Error) -> bool { + matches!(err.raw_os_error(), Some(libc::ENOTSUP) | Some(libc::ENOSYS)) + } +} + +// Note that we don't need a Drop impl to execute `flock(fd, LOCK_UN)`. A lock acquired by +// `flock` is associated with the file descriptor and closing the file releases it +// automatically. diff --git a/compiler/rustc_data_structures/src/flock/unix.rs b/compiler/rustc_data_structures/src/flock/unix.rs new file mode 100644 index 000000000..4e5297d58 --- /dev/null +++ b/compiler/rustc_data_structures/src/flock/unix.rs @@ -0,0 +1,51 @@ +use std::fs::{File, OpenOptions}; +use std::io; +use std::mem; +use std::os::unix::prelude::*; +use std::path::Path; + +#[derive(Debug)] +pub struct Lock { + file: File, +} + +impl Lock { + pub fn new(p: &Path, wait: bool, create: bool, exclusive: bool) -> io::Result<Lock> { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(create) + .mode(libc::S_IRWXU as u32) + .open(p)?; + + let lock_type = if exclusive { libc::F_WRLCK } else { libc::F_RDLCK }; + + let mut flock: libc::flock = unsafe { mem::zeroed() }; + flock.l_type = lock_type as libc::c_short; + flock.l_whence = libc::SEEK_SET as libc::c_short; + flock.l_start = 0; + flock.l_len = 0; + + let cmd = if wait { libc::F_SETLKW } else { libc::F_SETLK }; + let ret = unsafe { libc::fcntl(file.as_raw_fd(), cmd, &flock) }; + if ret == -1 { Err(io::Error::last_os_error()) } else { Ok(Lock { file }) } + } + + pub fn error_unsupported(err: &io::Error) -> bool { + matches!(err.raw_os_error(), Some(libc::ENOTSUP) | Some(libc::ENOSYS)) + } +} + +impl Drop for Lock { + fn drop(&mut self) { + let mut flock: libc::flock = unsafe { mem::zeroed() }; + flock.l_type = libc::F_UNLCK as libc::c_short; + flock.l_whence = libc::SEEK_SET as libc::c_short; + flock.l_start = 0; + flock.l_len = 0; + + unsafe { + libc::fcntl(self.file.as_raw_fd(), libc::F_SETLK, &flock); + } + } +} diff --git a/compiler/rustc_data_structures/src/flock/unsupported.rs b/compiler/rustc_data_structures/src/flock/unsupported.rs new file mode 100644 index 000000000..9245fca37 --- /dev/null +++ b/compiler/rustc_data_structures/src/flock/unsupported.rs @@ -0,0 +1,16 @@ +use std::io; +use std::path::Path; + +#[derive(Debug)] +pub struct Lock(()); + +impl Lock { + pub fn new(_p: &Path, _wait: bool, _create: bool, _exclusive: bool) -> io::Result<Lock> { + let msg = "file locks not supported on this platform"; + Err(io::Error::new(io::ErrorKind::Other, msg)) + } + + pub fn error_unsupported(_err: &io::Error) -> bool { + true + } +} diff --git a/compiler/rustc_data_structures/src/flock/windows.rs b/compiler/rustc_data_structures/src/flock/windows.rs new file mode 100644 index 000000000..43e6caaa1 --- /dev/null +++ b/compiler/rustc_data_structures/src/flock/windows.rs @@ -0,0 +1,77 @@ +use std::fs::{File, OpenOptions}; +use std::io; +use std::mem; +use std::os::windows::prelude::*; +use std::path::Path; + +use winapi::shared::winerror::ERROR_INVALID_FUNCTION; +use winapi::um::fileapi::LockFileEx; +use winapi::um::minwinbase::{LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY, OVERLAPPED}; +use winapi::um::winnt::{FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE}; + +#[derive(Debug)] +pub struct Lock { + _file: File, +} + +impl Lock { + pub fn new(p: &Path, wait: bool, create: bool, exclusive: bool) -> io::Result<Lock> { + assert!( + p.parent().unwrap().exists(), + "Parent directory of lock-file must exist: {}", + p.display() + ); + + let share_mode = FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE; + + let mut open_options = OpenOptions::new(); + open_options.read(true).share_mode(share_mode); + + if create { + open_options.create(true).write(true); + } + + debug!("attempting to open lock file `{}`", p.display()); + let file = match open_options.open(p) { + Ok(file) => { + debug!("lock file opened successfully"); + file + } + Err(err) => { + debug!("error opening lock file: {}", err); + return Err(err); + } + }; + + let ret = unsafe { + let mut overlapped: OVERLAPPED = mem::zeroed(); + + let mut dwFlags = 0; + if !wait { + dwFlags |= LOCKFILE_FAIL_IMMEDIATELY; + } + + if exclusive { + dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; + } + + debug!("attempting to acquire lock on lock file `{}`", p.display()); + LockFileEx(file.as_raw_handle(), dwFlags, 0, 0xFFFF_FFFF, 0xFFFF_FFFF, &mut overlapped) + }; + if ret == 0 { + let err = io::Error::last_os_error(); + debug!("failed acquiring file lock: {}", err); + Err(err) + } else { + debug!("successfully acquired lock"); + Ok(Lock { _file: file }) + } + } + + pub fn error_unsupported(err: &io::Error) -> bool { + err.raw_os_error() == Some(ERROR_INVALID_FUNCTION as i32) + } +} + +// Note that we don't need a Drop impl on Windows: The file is unlocked +// automatically when it's closed. diff --git a/compiler/rustc_data_structures/src/frozen.rs b/compiler/rustc_data_structures/src/frozen.rs new file mode 100644 index 000000000..c81e1b124 --- /dev/null +++ b/compiler/rustc_data_structures/src/frozen.rs @@ -0,0 +1,64 @@ +//! An immutable, owned value (except for interior mutability). +//! +//! The purpose of `Frozen` is to make a value immutable for the sake of defensive programming. For example, +//! suppose we have the following: +//! +//! ```rust +//! struct Bar { /* some data */ } +//! +//! struct Foo { +//! /// Some computed data that should never change after construction. +//! pub computed: Bar, +//! +//! /* some other fields */ +//! } +//! +//! impl Bar { +//! /// Mutate the `Bar`. +//! pub fn mutate(&mut self) { } +//! } +//! ``` +//! +//! Now suppose we want to pass around a mutable `Foo` instance but, we want to make sure that +//! `computed` does not change accidentally (e.g. somebody might accidentally call +//! `foo.computed.mutate()`). This is what `Frozen` is for. We can do the following: +//! +//! ``` +//! # struct Bar {} +//! use rustc_data_structures::frozen::Frozen; +//! +//! struct Foo { +//! /// Some computed data that should never change after construction. +//! pub computed: Frozen<Bar>, +//! +//! /* some other fields */ +//! } +//! ``` +//! +//! `Frozen` impls `Deref`, so we can ergonomically call methods on `Bar`, but it doesn't `impl +//! DerefMut`. Now calling `foo.compute.mutate()` will result in a compile-time error stating that +//! `mutate` requires a mutable reference but we don't have one. +//! +//! # Caveats +//! +//! - `Frozen` doesn't try to defend against interior mutability (e.g. `Frozen<RefCell<Bar>>`). +//! - `Frozen` doesn't pin it's contents (e.g. one could still do `foo.computed = +//! Frozen::freeze(new_bar)`). + +/// An owned immutable value. +#[derive(Debug)] +pub struct Frozen<T>(T); + +impl<T> Frozen<T> { + pub fn freeze(val: T) -> Self { + Frozen(val) + } +} + +impl<T> std::ops::Deref for Frozen<T> { + type Target = T; + + fn deref(&self) -> &T { + &self.0 + } +} diff --git a/compiler/rustc_data_structures/src/functor.rs b/compiler/rustc_data_structures/src/functor.rs new file mode 100644 index 000000000..a3d3f9883 --- /dev/null +++ b/compiler/rustc_data_structures/src/functor.rs @@ -0,0 +1,99 @@ +use rustc_index::vec::{Idx, IndexVec}; +use std::mem; + +pub trait IdFunctor: Sized { + type Inner; + + fn try_map_id<F, E>(self, f: F) -> Result<Self, E> + where + F: FnMut(Self::Inner) -> Result<Self::Inner, E>; +} + +impl<T> IdFunctor for Box<T> { + type Inner = T; + + #[inline] + fn try_map_id<F, E>(self, mut f: F) -> Result<Self, E> + where + F: FnMut(Self::Inner) -> Result<Self::Inner, E>, + { + let raw = Box::into_raw(self); + Ok(unsafe { + // SAFETY: The raw pointer points to a valid value of type `T`. + let value = raw.read(); + // SAFETY: Converts `Box<T>` to `Box<MaybeUninit<T>>` which is the + // inverse of `Box::assume_init()` and should be safe. + let raw: Box<mem::MaybeUninit<T>> = Box::from_raw(raw.cast()); + // SAFETY: Write the mapped value back into the `Box`. + Box::write(raw, f(value)?) + }) + } +} + +impl<T> IdFunctor for Vec<T> { + type Inner = T; + + #[inline] + fn try_map_id<F, E>(self, mut f: F) -> Result<Self, E> + where + F: FnMut(Self::Inner) -> Result<Self::Inner, E>, + { + struct HoleVec<T> { + vec: Vec<mem::ManuallyDrop<T>>, + hole: Option<usize>, + } + + impl<T> Drop for HoleVec<T> { + fn drop(&mut self) { + unsafe { + for (index, slot) in self.vec.iter_mut().enumerate() { + if self.hole != Some(index) { + mem::ManuallyDrop::drop(slot); + } + } + } + } + } + + unsafe { + let (ptr, length, capacity) = self.into_raw_parts(); + let vec = Vec::from_raw_parts(ptr.cast(), length, capacity); + let mut hole_vec = HoleVec { vec, hole: None }; + + for (index, slot) in hole_vec.vec.iter_mut().enumerate() { + hole_vec.hole = Some(index); + let original = mem::ManuallyDrop::take(slot); + let mapped = f(original)?; + *slot = mem::ManuallyDrop::new(mapped); + hole_vec.hole = None; + } + + mem::forget(hole_vec); + Ok(Vec::from_raw_parts(ptr, length, capacity)) + } + } +} + +impl<T> IdFunctor for Box<[T]> { + type Inner = T; + + #[inline] + fn try_map_id<F, E>(self, f: F) -> Result<Self, E> + where + F: FnMut(Self::Inner) -> Result<Self::Inner, E>, + { + Vec::from(self).try_map_id(f).map(Into::into) + } +} + +impl<I: Idx, T> IdFunctor for IndexVec<I, T> { + type Inner = T; + + #[inline] + fn try_map_id<F, E>(self, f: F) -> Result<Self, E> + where + F: FnMut(Self::Inner) -> Result<Self::Inner, E>, + { + self.raw.try_map_id(f).map(IndexVec::from_raw) + } +} diff --git a/compiler/rustc_data_structures/src/fx.rs b/compiler/rustc_data_structures/src/fx.rs new file mode 100644 index 000000000..bbeb193db --- /dev/null +++ b/compiler/rustc_data_structures/src/fx.rs @@ -0,0 +1,14 @@ +use std::hash::BuildHasherDefault; + +pub use rustc_hash::{FxHashMap, FxHashSet, FxHasher}; + +pub type FxIndexMap<K, V> = indexmap::IndexMap<K, V, BuildHasherDefault<FxHasher>>; +pub type FxIndexSet<V> = indexmap::IndexSet<V, BuildHasherDefault<FxHasher>>; + +#[macro_export] +macro_rules! define_id_collections { + ($map_name:ident, $set_name:ident, $key:ty) => { + pub type $map_name<T> = $crate::fx::FxHashMap<$key, T>; + pub type $set_name = $crate::fx::FxHashSet<$key>; + }; +} diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs new file mode 100644 index 000000000..00913a483 --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs @@ -0,0 +1,324 @@ +//! Finding the dominators in a control-flow graph. +//! +//! Algorithm based on Loukas Georgiadis, +//! "Linear-Time Algorithms for Dominators and Related Problems", +//! <ftp://ftp.cs.princeton.edu/techreports/2005/737.pdf> +//! +//! Additionally useful is the original Lengauer-Tarjan paper on this subject, +//! "A Fast Algorithm for Finding Dominators in a Flowgraph" +//! Thomas Lengauer and Robert Endre Tarjan. +//! <https://www.cs.princeton.edu/courses/archive/spr03/cs423/download/dominators.pdf> + +use super::ControlFlowGraph; +use rustc_index::vec::{Idx, IndexVec}; +use std::cmp::Ordering; + +#[cfg(test)] +mod tests; + +struct PreOrderFrame<Iter> { + pre_order_idx: PreorderIndex, + iter: Iter, +} + +rustc_index::newtype_index! { + struct PreorderIndex { .. } +} + +pub fn dominators<G: ControlFlowGraph>(graph: G) -> Dominators<G::Node> { + // compute the post order index (rank) for each node + let mut post_order_rank = IndexVec::from_elem_n(0, graph.num_nodes()); + + // We allocate capacity for the full set of nodes, because most of the time + // most of the nodes *are* reachable. + let mut parent: IndexVec<PreorderIndex, PreorderIndex> = + IndexVec::with_capacity(graph.num_nodes()); + + let mut stack = vec![PreOrderFrame { + pre_order_idx: PreorderIndex::new(0), + iter: graph.successors(graph.start_node()), + }]; + let mut pre_order_to_real: IndexVec<PreorderIndex, G::Node> = + IndexVec::with_capacity(graph.num_nodes()); + let mut real_to_pre_order: IndexVec<G::Node, Option<PreorderIndex>> = + IndexVec::from_elem_n(None, graph.num_nodes()); + pre_order_to_real.push(graph.start_node()); + parent.push(PreorderIndex::new(0)); // the parent of the root node is the root for now. + real_to_pre_order[graph.start_node()] = Some(PreorderIndex::new(0)); + let mut post_order_idx = 0; + + // Traverse the graph, collecting a number of things: + // + // * Preorder mapping (to it, and back to the actual ordering) + // * Postorder mapping (used exclusively for rank_partial_cmp on the final product) + // * Parents for each vertex in the preorder tree + // + // These are all done here rather than through one of the 'standard' + // graph traversals to help make this fast. + 'recurse: while let Some(frame) = stack.last_mut() { + while let Some(successor) = frame.iter.next() { + if real_to_pre_order[successor].is_none() { + let pre_order_idx = pre_order_to_real.push(successor); + real_to_pre_order[successor] = Some(pre_order_idx); + parent.push(frame.pre_order_idx); + stack.push(PreOrderFrame { pre_order_idx, iter: graph.successors(successor) }); + + continue 'recurse; + } + } + post_order_rank[pre_order_to_real[frame.pre_order_idx]] = post_order_idx; + post_order_idx += 1; + + stack.pop(); + } + + let reachable_vertices = pre_order_to_real.len(); + + let mut idom = IndexVec::from_elem_n(PreorderIndex::new(0), reachable_vertices); + let mut semi = IndexVec::from_fn_n(std::convert::identity, reachable_vertices); + let mut label = semi.clone(); + let mut bucket = IndexVec::from_elem_n(vec![], reachable_vertices); + let mut lastlinked = None; + + // We loop over vertices in reverse preorder. This implements the pseudocode + // of the simple Lengauer-Tarjan algorithm. A few key facts are noted here + // which are helpful for understanding the code (full proofs and such are + // found in various papers, including one cited at the top of this file). + // + // For each vertex w (which is not the root), + // * semi[w] is a proper ancestor of the vertex w (i.e., semi[w] != w) + // * idom[w] is an ancestor of semi[w] (i.e., idom[w] may equal semi[w]) + // + // An immediate dominator of w (idom[w]) is a vertex v where v dominates w + // and every other dominator of w dominates v. (Every vertex except the root has + // a unique immediate dominator.) + // + // A semidominator for a given vertex w (semi[w]) is the vertex v with minimum + // preorder number such that there exists a path from v to w in which all elements (other than w) have + // preorder numbers greater than w (i.e., this path is not the tree path to + // w). + for w in (PreorderIndex::new(1)..PreorderIndex::new(reachable_vertices)).rev() { + // Optimization: process buckets just once, at the start of the + // iteration. Do not explicitly empty the bucket (even though it will + // not be used again), to save some instructions. + // + // The bucket here contains the vertices whose semidominator is the + // vertex w, which we are guaranteed to have found: all vertices who can + // be semidominated by w must have a preorder number exceeding w, so + // they have been placed in the bucket. + // + // We compute a partial set of immediate dominators here. + let z = parent[w]; + for &v in bucket[z].iter() { + // This uses the result of Lemma 5 from section 2 from the original + // 1979 paper, to compute either the immediate or relative dominator + // for a given vertex v. + // + // eval returns a vertex y, for which semi[y] is minimum among + // vertices semi[v] +> y *> v. Note that semi[v] = z as we're in the + // z bucket. + // + // Given such a vertex y, semi[y] <= semi[v] and idom[y] = idom[v]. + // If semi[y] = semi[v], though, idom[v] = semi[v]. + // + // Using this, we can either set idom[v] to be: + // * semi[v] (i.e. z), if semi[y] is z + // * idom[y], otherwise + // + // We don't directly set to idom[y] though as it's not necessarily + // known yet. The second preorder traversal will cleanup by updating + // the idom for any that were missed in this pass. + let y = eval(&mut parent, lastlinked, &semi, &mut label, v); + idom[v] = if semi[y] < z { y } else { z }; + } + + // This loop computes the semi[w] for w. + semi[w] = w; + for v in graph.predecessors(pre_order_to_real[w]) { + let v = real_to_pre_order[v].unwrap(); + + // eval returns a vertex x from which semi[x] is minimum among + // vertices semi[v] +> x *> v. + // + // From Lemma 4 from section 2, we know that the semidominator of a + // vertex w is the minimum (by preorder number) vertex of the + // following: + // + // * direct predecessors of w with preorder number less than w + // * semidominators of u such that u > w and there exists (v, w) + // such that u *> v + // + // This loop therefore identifies such a minima. Note that any + // semidominator path to w must have all but the first vertex go + // through vertices numbered greater than w, so the reverse preorder + // traversal we are using guarantees that all of the information we + // might need is available at this point. + // + // The eval call will give us semi[x], which is either: + // + // * v itself, if v has not yet been processed + // * A possible 'best' semidominator for w. + let x = eval(&mut parent, lastlinked, &semi, &mut label, v); + semi[w] = std::cmp::min(semi[w], semi[x]); + } + // semi[w] is now semidominator(w) and won't change any more. + + // Optimization: Do not insert into buckets if parent[w] = semi[w], as + // we then immediately know the idom. + // + // If we don't yet know the idom directly, then push this vertex into + // our semidominator's bucket, where it will get processed at a later + // stage to compute its immediate dominator. + if parent[w] != semi[w] { + bucket[semi[w]].push(w); + } else { + idom[w] = parent[w]; + } + + // Optimization: We share the parent array between processed and not + // processed elements; lastlinked represents the divider. + lastlinked = Some(w); + } + + // Finalize the idoms for any that were not fully settable during initial + // traversal. + // + // If idom[w] != semi[w] then we know that we've stored vertex y from above + // into idom[w]. It is known to be our 'relative dominator', which means + // that it's one of w's ancestors and has the same immediate dominator as w, + // so use that idom. + for w in PreorderIndex::new(1)..PreorderIndex::new(reachable_vertices) { + if idom[w] != semi[w] { + idom[w] = idom[idom[w]]; + } + } + + let mut immediate_dominators = IndexVec::from_elem_n(None, graph.num_nodes()); + for (idx, node) in pre_order_to_real.iter_enumerated() { + immediate_dominators[*node] = Some(pre_order_to_real[idom[idx]]); + } + + Dominators { post_order_rank, immediate_dominators } +} + +/// Evaluate the link-eval virtual forest, providing the currently minimum semi +/// value for the passed `node` (which may be itself). +/// +/// This maintains that for every vertex v, `label[v]` is such that: +/// +/// ```text +/// semi[eval(v)] = min { semi[label[u]] | root_in_forest(v) +> u *> v } +/// ``` +/// +/// where `+>` is a proper ancestor and `*>` is just an ancestor. +#[inline] +fn eval( + ancestor: &mut IndexVec<PreorderIndex, PreorderIndex>, + lastlinked: Option<PreorderIndex>, + semi: &IndexVec<PreorderIndex, PreorderIndex>, + label: &mut IndexVec<PreorderIndex, PreorderIndex>, + node: PreorderIndex, +) -> PreorderIndex { + if is_processed(node, lastlinked) { + compress(ancestor, lastlinked, semi, label, node); + label[node] + } else { + node + } +} + +#[inline] +fn is_processed(v: PreorderIndex, lastlinked: Option<PreorderIndex>) -> bool { + if let Some(ll) = lastlinked { v >= ll } else { false } +} + +#[inline] +fn compress( + ancestor: &mut IndexVec<PreorderIndex, PreorderIndex>, + lastlinked: Option<PreorderIndex>, + semi: &IndexVec<PreorderIndex, PreorderIndex>, + label: &mut IndexVec<PreorderIndex, PreorderIndex>, + v: PreorderIndex, +) { + assert!(is_processed(v, lastlinked)); + // Compute the processed list of ancestors + // + // We use a heap stack here to avoid recursing too deeply, exhausting the + // stack space. + let mut stack: smallvec::SmallVec<[_; 8]> = smallvec::smallvec![v]; + let mut u = ancestor[v]; + while is_processed(u, lastlinked) { + stack.push(u); + u = ancestor[u]; + } + + // Then in reverse order, popping the stack + for &[v, u] in stack.array_windows().rev() { + if semi[label[u]] < semi[label[v]] { + label[v] = label[u]; + } + ancestor[v] = ancestor[u]; + } +} + +#[derive(Clone, Debug)] +pub struct Dominators<N: Idx> { + post_order_rank: IndexVec<N, usize>, + immediate_dominators: IndexVec<N, Option<N>>, +} + +impl<Node: Idx> Dominators<Node> { + pub fn dummy() -> Self { + Self { post_order_rank: IndexVec::new(), immediate_dominators: IndexVec::new() } + } + + pub fn is_reachable(&self, node: Node) -> bool { + self.immediate_dominators[node].is_some() + } + + pub fn immediate_dominator(&self, node: Node) -> Node { + assert!(self.is_reachable(node), "node {:?} is not reachable", node); + self.immediate_dominators[node].unwrap() + } + + pub fn dominators(&self, node: Node) -> Iter<'_, Node> { + assert!(self.is_reachable(node), "node {:?} is not reachable", node); + Iter { dominators: self, node: Some(node) } + } + + pub fn is_dominated_by(&self, node: Node, dom: Node) -> bool { + // FIXME -- could be optimized by using post-order-rank + self.dominators(node).any(|n| n == dom) + } + + /// Provide deterministic ordering of nodes such that, if any two nodes have a dominator + /// relationship, the dominator will always precede the dominated. (The relative ordering + /// of two unrelated nodes will also be consistent, but otherwise the order has no + /// meaning.) This method cannot be used to determine if either Node dominates the other. + pub fn rank_partial_cmp(&self, lhs: Node, rhs: Node) -> Option<Ordering> { + self.post_order_rank[lhs].partial_cmp(&self.post_order_rank[rhs]) + } +} + +pub struct Iter<'dom, Node: Idx> { + dominators: &'dom Dominators<Node>, + node: Option<Node>, +} + +impl<'dom, Node: Idx> Iterator for Iter<'dom, Node> { + type Item = Node; + + fn next(&mut self) -> Option<Self::Item> { + if let Some(node) = self.node { + let dom = self.dominators.immediate_dominator(node); + if dom == node { + self.node = None; // reached the root + } else { + self.node = Some(dom); + } + Some(node) + } else { + None + } + } +} diff --git a/compiler/rustc_data_structures/src/graph/dominators/tests.rs b/compiler/rustc_data_structures/src/graph/dominators/tests.rs new file mode 100644 index 000000000..ff31d8f7f --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/dominators/tests.rs @@ -0,0 +1,45 @@ +use super::*; + +use super::super::tests::TestGraph; + +#[test] +fn diamond() { + let graph = TestGraph::new(0, &[(0, 1), (0, 2), (1, 3), (2, 3)]); + + let dominators = dominators(&graph); + let immediate_dominators = &dominators.immediate_dominators; + assert_eq!(immediate_dominators[0], Some(0)); + assert_eq!(immediate_dominators[1], Some(0)); + assert_eq!(immediate_dominators[2], Some(0)); + assert_eq!(immediate_dominators[3], Some(0)); +} + +#[test] +fn paper() { + // example from the paper: + let graph = TestGraph::new( + 6, + &[(6, 5), (6, 4), (5, 1), (4, 2), (4, 3), (1, 2), (2, 3), (3, 2), (2, 1)], + ); + + let dominators = dominators(&graph); + let immediate_dominators = &dominators.immediate_dominators; + assert_eq!(immediate_dominators[0], None); // <-- note that 0 is not in graph + assert_eq!(immediate_dominators[1], Some(6)); + assert_eq!(immediate_dominators[2], Some(6)); + assert_eq!(immediate_dominators[3], Some(6)); + assert_eq!(immediate_dominators[4], Some(6)); + assert_eq!(immediate_dominators[5], Some(6)); + assert_eq!(immediate_dominators[6], Some(6)); +} + +#[test] +fn paper_slt() { + // example from the paper: + let graph = TestGraph::new( + 1, + &[(1, 2), (1, 3), (2, 3), (2, 7), (3, 4), (3, 6), (4, 5), (5, 4), (6, 7), (7, 8), (8, 5)], + ); + + dominators(&graph); +} diff --git a/compiler/rustc_data_structures/src/graph/implementation/mod.rs b/compiler/rustc_data_structures/src/graph/implementation/mod.rs new file mode 100644 index 000000000..1aa7ac024 --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/implementation/mod.rs @@ -0,0 +1,366 @@ +//! A graph module for use in dataflow, region resolution, and elsewhere. +//! +//! # Interface details +//! +//! You customize the graph by specifying a "node data" type `N` and an +//! "edge data" type `E`. You can then later gain access (mutable or +//! immutable) to these "user-data" bits. Currently, you can only add +//! nodes or edges to the graph. You cannot remove or modify them once +//! added. This could be changed if we have a need. +//! +//! # Implementation details +//! +//! The main tricky thing about this code is the way that edges are +//! stored. The edges are stored in a central array, but they are also +//! threaded onto two linked lists for each node, one for incoming edges +//! and one for outgoing edges. Note that every edge is a member of some +//! incoming list and some outgoing list. Basically you can load the +//! first index of the linked list from the node data structures (the +//! field `first_edge`) and then, for each edge, load the next index from +//! the field `next_edge`). Each of those fields is an array that should +//! be indexed by the direction (see the type `Direction`). + +use crate::snapshot_vec::{SnapshotVec, SnapshotVecDelegate}; +use rustc_index::bit_set::BitSet; +use std::fmt::Debug; + +#[cfg(test)] +mod tests; + +pub struct Graph<N, E> { + nodes: SnapshotVec<Node<N>>, + edges: SnapshotVec<Edge<E>>, +} + +pub struct Node<N> { + first_edge: [EdgeIndex; 2], // see module comment + pub data: N, +} + +#[derive(Debug)] +pub struct Edge<E> { + next_edge: [EdgeIndex; 2], // see module comment + source: NodeIndex, + target: NodeIndex, + pub data: E, +} + +impl<N> SnapshotVecDelegate for Node<N> { + type Value = Node<N>; + type Undo = (); + + fn reverse(_: &mut Vec<Node<N>>, _: ()) {} +} + +impl<N> SnapshotVecDelegate for Edge<N> { + type Value = Edge<N>; + type Undo = (); + + fn reverse(_: &mut Vec<Edge<N>>, _: ()) {} +} + +#[derive(Copy, Clone, PartialEq, Debug)] +pub struct NodeIndex(pub usize); + +#[derive(Copy, Clone, PartialEq, Debug)] +pub struct EdgeIndex(pub usize); + +pub const INVALID_EDGE_INDEX: EdgeIndex = EdgeIndex(usize::MAX); + +// Use a private field here to guarantee no more instances are created: +#[derive(Copy, Clone, Debug, PartialEq)] +pub struct Direction { + repr: usize, +} + +pub const OUTGOING: Direction = Direction { repr: 0 }; + +pub const INCOMING: Direction = Direction { repr: 1 }; + +impl NodeIndex { + /// Returns unique ID (unique with respect to the graph holding associated node). + pub fn node_id(self) -> usize { + self.0 + } +} + +impl<N: Debug, E: Debug> Graph<N, E> { + pub fn new() -> Graph<N, E> { + Graph { nodes: SnapshotVec::new(), edges: SnapshotVec::new() } + } + + pub fn with_capacity(nodes: usize, edges: usize) -> Graph<N, E> { + Graph { nodes: SnapshotVec::with_capacity(nodes), edges: SnapshotVec::with_capacity(edges) } + } + + // # Simple accessors + + #[inline] + pub fn all_nodes(&self) -> &[Node<N>] { + &self.nodes + } + + #[inline] + pub fn len_nodes(&self) -> usize { + self.nodes.len() + } + + #[inline] + pub fn all_edges(&self) -> &[Edge<E>] { + &self.edges + } + + #[inline] + pub fn len_edges(&self) -> usize { + self.edges.len() + } + + // # Node construction + + pub fn next_node_index(&self) -> NodeIndex { + NodeIndex(self.nodes.len()) + } + + pub fn add_node(&mut self, data: N) -> NodeIndex { + let idx = self.next_node_index(); + self.nodes.push(Node { first_edge: [INVALID_EDGE_INDEX, INVALID_EDGE_INDEX], data }); + idx + } + + pub fn mut_node_data(&mut self, idx: NodeIndex) -> &mut N { + &mut self.nodes[idx.0].data + } + + pub fn node_data(&self, idx: NodeIndex) -> &N { + &self.nodes[idx.0].data + } + + pub fn node(&self, idx: NodeIndex) -> &Node<N> { + &self.nodes[idx.0] + } + + // # Edge construction and queries + + pub fn next_edge_index(&self) -> EdgeIndex { + EdgeIndex(self.edges.len()) + } + + pub fn add_edge(&mut self, source: NodeIndex, target: NodeIndex, data: E) -> EdgeIndex { + debug!("graph: add_edge({:?}, {:?}, {:?})", source, target, data); + + let idx = self.next_edge_index(); + + // read current first of the list of edges from each node + let source_first = self.nodes[source.0].first_edge[OUTGOING.repr]; + let target_first = self.nodes[target.0].first_edge[INCOMING.repr]; + + // create the new edge, with the previous firsts from each node + // as the next pointers + self.edges.push(Edge { next_edge: [source_first, target_first], source, target, data }); + + // adjust the firsts for each node target be the next object. + self.nodes[source.0].first_edge[OUTGOING.repr] = idx; + self.nodes[target.0].first_edge[INCOMING.repr] = idx; + + idx + } + + pub fn edge(&self, idx: EdgeIndex) -> &Edge<E> { + &self.edges[idx.0] + } + + // # Iterating over nodes, edges + + pub fn enumerated_nodes(&self) -> impl Iterator<Item = (NodeIndex, &Node<N>)> { + self.nodes.iter().enumerate().map(|(idx, n)| (NodeIndex(idx), n)) + } + + pub fn enumerated_edges(&self) -> impl Iterator<Item = (EdgeIndex, &Edge<E>)> { + self.edges.iter().enumerate().map(|(idx, e)| (EdgeIndex(idx), e)) + } + + pub fn each_node<'a>(&'a self, mut f: impl FnMut(NodeIndex, &'a Node<N>) -> bool) -> bool { + //! Iterates over all edges defined in the graph. + self.enumerated_nodes().all(|(node_idx, node)| f(node_idx, node)) + } + + pub fn each_edge<'a>(&'a self, mut f: impl FnMut(EdgeIndex, &'a Edge<E>) -> bool) -> bool { + //! Iterates over all edges defined in the graph + self.enumerated_edges().all(|(edge_idx, edge)| f(edge_idx, edge)) + } + + pub fn outgoing_edges(&self, source: NodeIndex) -> AdjacentEdges<'_, N, E> { + self.adjacent_edges(source, OUTGOING) + } + + pub fn incoming_edges(&self, source: NodeIndex) -> AdjacentEdges<'_, N, E> { + self.adjacent_edges(source, INCOMING) + } + + pub fn adjacent_edges( + &self, + source: NodeIndex, + direction: Direction, + ) -> AdjacentEdges<'_, N, E> { + let first_edge = self.node(source).first_edge[direction.repr]; + AdjacentEdges { graph: self, direction, next: first_edge } + } + + pub fn successor_nodes<'a>( + &'a self, + source: NodeIndex, + ) -> impl Iterator<Item = NodeIndex> + 'a { + self.outgoing_edges(source).targets() + } + + pub fn predecessor_nodes<'a>( + &'a self, + target: NodeIndex, + ) -> impl Iterator<Item = NodeIndex> + 'a { + self.incoming_edges(target).sources() + } + + pub fn depth_traverse( + &self, + start: NodeIndex, + direction: Direction, + ) -> DepthFirstTraversal<'_, N, E> { + DepthFirstTraversal::with_start_node(self, start, direction) + } + + pub fn nodes_in_postorder( + &self, + direction: Direction, + entry_node: NodeIndex, + ) -> Vec<NodeIndex> { + let mut visited = BitSet::new_empty(self.len_nodes()); + let mut stack = vec![]; + let mut result = Vec::with_capacity(self.len_nodes()); + let mut push_node = |stack: &mut Vec<_>, node: NodeIndex| { + if visited.insert(node.0) { + stack.push((node, self.adjacent_edges(node, direction))); + } + }; + + for node in + Some(entry_node).into_iter().chain(self.enumerated_nodes().map(|(node, _)| node)) + { + push_node(&mut stack, node); + while let Some((node, mut iter)) = stack.pop() { + if let Some((_, child)) = iter.next() { + let target = child.source_or_target(direction); + // the current node needs more processing, so + // add it back to the stack + stack.push((node, iter)); + // and then push the new node + push_node(&mut stack, target); + } else { + result.push(node); + } + } + } + + assert_eq!(result.len(), self.len_nodes()); + result + } +} + +// # Iterators + +pub struct AdjacentEdges<'g, N, E> { + graph: &'g Graph<N, E>, + direction: Direction, + next: EdgeIndex, +} + +impl<'g, N: Debug, E: Debug> AdjacentEdges<'g, N, E> { + fn targets(self) -> impl Iterator<Item = NodeIndex> + 'g { + self.map(|(_, edge)| edge.target) + } + + fn sources(self) -> impl Iterator<Item = NodeIndex> + 'g { + self.map(|(_, edge)| edge.source) + } +} + +impl<'g, N: Debug, E: Debug> Iterator for AdjacentEdges<'g, N, E> { + type Item = (EdgeIndex, &'g Edge<E>); + + fn next(&mut self) -> Option<(EdgeIndex, &'g Edge<E>)> { + let edge_index = self.next; + if edge_index == INVALID_EDGE_INDEX { + return None; + } + + let edge = self.graph.edge(edge_index); + self.next = edge.next_edge[self.direction.repr]; + Some((edge_index, edge)) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + // At most, all the edges in the graph. + (0, Some(self.graph.len_edges())) + } +} + +pub struct DepthFirstTraversal<'g, N, E> { + graph: &'g Graph<N, E>, + stack: Vec<NodeIndex>, + visited: BitSet<usize>, + direction: Direction, +} + +impl<'g, N: Debug, E: Debug> DepthFirstTraversal<'g, N, E> { + pub fn with_start_node( + graph: &'g Graph<N, E>, + start_node: NodeIndex, + direction: Direction, + ) -> Self { + let mut visited = BitSet::new_empty(graph.len_nodes()); + visited.insert(start_node.node_id()); + DepthFirstTraversal { graph, stack: vec![start_node], visited, direction } + } + + fn visit(&mut self, node: NodeIndex) { + if self.visited.insert(node.node_id()) { + self.stack.push(node); + } + } +} + +impl<'g, N: Debug, E: Debug> Iterator for DepthFirstTraversal<'g, N, E> { + type Item = NodeIndex; + + fn next(&mut self) -> Option<NodeIndex> { + let next = self.stack.pop(); + if let Some(idx) = next { + for (_, edge) in self.graph.adjacent_edges(idx, self.direction) { + let target = edge.source_or_target(self.direction); + self.visit(target); + } + } + next + } + + fn size_hint(&self) -> (usize, Option<usize>) { + // We will visit every node in the graph exactly once. + let remaining = self.graph.len_nodes() - self.visited.count(); + (remaining, Some(remaining)) + } +} + +impl<'g, N: Debug, E: Debug> ExactSizeIterator for DepthFirstTraversal<'g, N, E> {} + +impl<E> Edge<E> { + pub fn source(&self) -> NodeIndex { + self.source + } + + pub fn target(&self) -> NodeIndex { + self.target + } + + pub fn source_or_target(&self, direction: Direction) -> NodeIndex { + if direction == OUTGOING { self.target } else { self.source } + } +} diff --git a/compiler/rustc_data_structures/src/graph/implementation/tests.rs b/compiler/rustc_data_structures/src/graph/implementation/tests.rs new file mode 100644 index 000000000..e4e4d0d44 --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/implementation/tests.rs @@ -0,0 +1,131 @@ +use crate::graph::implementation::*; +use std::fmt::Debug; + +type TestGraph = Graph<&'static str, &'static str>; + +fn create_graph() -> TestGraph { + let mut graph = Graph::new(); + + // Create a simple graph + // + // F + // | + // V + // A --> B --> C + // | ^ + // v | + // D --> E + + let a = graph.add_node("A"); + let b = graph.add_node("B"); + let c = graph.add_node("C"); + let d = graph.add_node("D"); + let e = graph.add_node("E"); + let f = graph.add_node("F"); + + graph.add_edge(a, b, "AB"); + graph.add_edge(b, c, "BC"); + graph.add_edge(b, d, "BD"); + graph.add_edge(d, e, "DE"); + graph.add_edge(e, c, "EC"); + graph.add_edge(f, b, "FB"); + + return graph; +} + +#[test] +fn each_node() { + let graph = create_graph(); + let expected = ["A", "B", "C", "D", "E", "F"]; + graph.each_node(|idx, node| { + assert_eq!(&expected[idx.0], graph.node_data(idx)); + assert_eq!(expected[idx.0], node.data); + true + }); +} + +#[test] +fn each_edge() { + let graph = create_graph(); + let expected = ["AB", "BC", "BD", "DE", "EC", "FB"]; + graph.each_edge(|idx, edge| { + assert_eq!(expected[idx.0], edge.data); + true + }); +} + +fn test_adjacent_edges<N: PartialEq + Debug, E: PartialEq + Debug>( + graph: &Graph<N, E>, + start_index: NodeIndex, + start_data: N, + expected_incoming: &[(E, N)], + expected_outgoing: &[(E, N)], +) { + assert!(graph.node_data(start_index) == &start_data); + + let mut counter = 0; + for (edge_index, edge) in graph.incoming_edges(start_index) { + assert!(counter < expected_incoming.len()); + debug!( + "counter={:?} expected={:?} edge_index={:?} edge={:?}", + counter, expected_incoming[counter], edge_index, edge + ); + match expected_incoming[counter] { + (ref e, ref n) => { + assert!(e == &edge.data); + assert!(n == graph.node_data(edge.source())); + assert!(start_index == edge.target); + } + } + counter += 1; + } + assert_eq!(counter, expected_incoming.len()); + + let mut counter = 0; + for (edge_index, edge) in graph.outgoing_edges(start_index) { + assert!(counter < expected_outgoing.len()); + debug!( + "counter={:?} expected={:?} edge_index={:?} edge={:?}", + counter, expected_outgoing[counter], edge_index, edge + ); + match expected_outgoing[counter] { + (ref e, ref n) => { + assert!(e == &edge.data); + assert!(start_index == edge.source); + assert!(n == graph.node_data(edge.target)); + } + } + counter += 1; + } + assert_eq!(counter, expected_outgoing.len()); +} + +#[test] +fn each_adjacent_from_a() { + let graph = create_graph(); + test_adjacent_edges(&graph, NodeIndex(0), "A", &[], &[("AB", "B")]); +} + +#[test] +fn each_adjacent_from_b() { + let graph = create_graph(); + test_adjacent_edges( + &graph, + NodeIndex(1), + "B", + &[("FB", "F"), ("AB", "A")], + &[("BD", "D"), ("BC", "C")], + ); +} + +#[test] +fn each_adjacent_from_c() { + let graph = create_graph(); + test_adjacent_edges(&graph, NodeIndex(2), "C", &[("EC", "E"), ("BC", "B")], &[]); +} + +#[test] +fn each_adjacent_from_d() { + let graph = create_graph(); + test_adjacent_edges(&graph, NodeIndex(3), "D", &[("BD", "B")], &[("DE", "E")]); +} diff --git a/compiler/rustc_data_structures/src/graph/iterate/mod.rs b/compiler/rustc_data_structures/src/graph/iterate/mod.rs new file mode 100644 index 000000000..57007611a --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/iterate/mod.rs @@ -0,0 +1,353 @@ +use super::{DirectedGraph, WithNumNodes, WithStartNode, WithSuccessors}; +use rustc_index::bit_set::BitSet; +use rustc_index::vec::IndexVec; +use std::ops::ControlFlow; + +#[cfg(test)] +mod tests; + +pub fn post_order_from<G: DirectedGraph + WithSuccessors + WithNumNodes>( + graph: &G, + start_node: G::Node, +) -> Vec<G::Node> { + post_order_from_to(graph, start_node, None) +} + +pub fn post_order_from_to<G: DirectedGraph + WithSuccessors + WithNumNodes>( + graph: &G, + start_node: G::Node, + end_node: Option<G::Node>, +) -> Vec<G::Node> { + let mut visited: IndexVec<G::Node, bool> = IndexVec::from_elem_n(false, graph.num_nodes()); + let mut result: Vec<G::Node> = Vec::with_capacity(graph.num_nodes()); + if let Some(end_node) = end_node { + visited[end_node] = true; + } + post_order_walk(graph, start_node, &mut result, &mut visited); + result +} + +fn post_order_walk<G: DirectedGraph + WithSuccessors + WithNumNodes>( + graph: &G, + node: G::Node, + result: &mut Vec<G::Node>, + visited: &mut IndexVec<G::Node, bool>, +) { + struct PostOrderFrame<Node, Iter> { + node: Node, + iter: Iter, + } + + if visited[node] { + return; + } + + let mut stack = vec![PostOrderFrame { node, iter: graph.successors(node) }]; + + 'recurse: while let Some(frame) = stack.last_mut() { + let node = frame.node; + visited[node] = true; + + while let Some(successor) = frame.iter.next() { + if !visited[successor] { + stack.push(PostOrderFrame { node: successor, iter: graph.successors(successor) }); + continue 'recurse; + } + } + + let _ = stack.pop(); + result.push(node); + } +} + +pub fn reverse_post_order<G: DirectedGraph + WithSuccessors + WithNumNodes>( + graph: &G, + start_node: G::Node, +) -> Vec<G::Node> { + let mut vec = post_order_from(graph, start_node); + vec.reverse(); + vec +} + +/// A "depth-first search" iterator for a directed graph. +pub struct DepthFirstSearch<'graph, G> +where + G: ?Sized + DirectedGraph + WithNumNodes + WithSuccessors, +{ + graph: &'graph G, + stack: Vec<G::Node>, + visited: BitSet<G::Node>, +} + +impl<'graph, G> DepthFirstSearch<'graph, G> +where + G: ?Sized + DirectedGraph + WithNumNodes + WithSuccessors, +{ + pub fn new(graph: &'graph G) -> Self { + Self { graph, stack: vec![], visited: BitSet::new_empty(graph.num_nodes()) } + } + + /// Version of `push_start_node` that is convenient for chained + /// use. + pub fn with_start_node(mut self, start_node: G::Node) -> Self { + self.push_start_node(start_node); + self + } + + /// Pushes another start node onto the stack. If the node + /// has not already been visited, then you will be able to + /// walk its successors (and so forth) after the current + /// contents of the stack are drained. If multiple start nodes + /// are added into the walk, then their mutual successors + /// will all be walked. You can use this method once the + /// iterator has been completely drained to add additional + /// start nodes. + pub fn push_start_node(&mut self, start_node: G::Node) { + if self.visited.insert(start_node) { + self.stack.push(start_node); + } + } + + /// Searches all nodes reachable from the current start nodes. + /// This is equivalent to just invoke `next` repeatedly until + /// you get a `None` result. + pub fn complete_search(&mut self) { + while let Some(_) = self.next() {} + } + + /// Returns true if node has been visited thus far. + /// A node is considered "visited" once it is pushed + /// onto the internal stack; it may not yet have been yielded + /// from the iterator. This method is best used after + /// the iterator is completely drained. + pub fn visited(&self, node: G::Node) -> bool { + self.visited.contains(node) + } +} + +impl<G> std::fmt::Debug for DepthFirstSearch<'_, G> +where + G: ?Sized + DirectedGraph + WithNumNodes + WithSuccessors, +{ + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut f = fmt.debug_set(); + for n in self.visited.iter() { + f.entry(&n); + } + f.finish() + } +} + +impl<G> Iterator for DepthFirstSearch<'_, G> +where + G: ?Sized + DirectedGraph + WithNumNodes + WithSuccessors, +{ + type Item = G::Node; + + fn next(&mut self) -> Option<G::Node> { + let DepthFirstSearch { stack, visited, graph } = self; + let n = stack.pop()?; + stack.extend(graph.successors(n).filter(|&m| visited.insert(m))); + Some(n) + } +} + +/// The status of a node in the depth-first search. +/// +/// See the documentation of `TriColorDepthFirstSearch` to see how a node's status is updated +/// during DFS. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NodeStatus { + /// This node has been examined by the depth-first search but is not yet `Settled`. + /// + /// Also referred to as "gray" or "discovered" nodes in [CLR]. + /// + /// [CLR]: https://en.wikipedia.org/wiki/Introduction_to_Algorithms + Visited, + + /// This node and all nodes reachable from it have been examined by the depth-first search. + /// + /// Also referred to as "black" or "finished" nodes in [CLR]. + /// + /// [CLR]: https://en.wikipedia.org/wiki/Introduction_to_Algorithms + Settled, +} + +struct Event<N> { + node: N, + becomes: NodeStatus, +} + +/// A depth-first search that also tracks when all successors of a node have been examined. +/// +/// This is based on the DFS described in [Introduction to Algorithms (1st ed.)][CLR], hereby +/// referred to as **CLR**. However, we use the terminology in [`NodeStatus`] above instead of +/// "discovered"/"finished" or "white"/"grey"/"black". Each node begins the search with no status, +/// becomes `Visited` when it is first examined by the DFS and is `Settled` when all nodes +/// reachable from it have been examined. This allows us to differentiate between "tree", "back" +/// and "forward" edges (see [`TriColorVisitor::node_examined`]). +/// +/// Unlike the pseudocode in [CLR], this implementation is iterative and does not use timestamps. +/// We accomplish this by storing `Event`s on the stack that result in a (possible) state change +/// for each node. A `Visited` event signifies that we should examine this node if it has not yet +/// been `Visited` or `Settled`. When a node is examined for the first time, we mark it as +/// `Visited` and push a `Settled` event for it on stack followed by `Visited` events for all of +/// its predecessors, scheduling them for examination. Multiple `Visited` events for a single node +/// may exist on the stack simultaneously if a node has multiple predecessors, but only one +/// `Settled` event will ever be created for each node. After all `Visited` events for a node's +/// successors have been popped off the stack (as well as any new events triggered by visiting +/// those successors), we will pop off that node's `Settled` event. +/// +/// [CLR]: https://en.wikipedia.org/wiki/Introduction_to_Algorithms +pub struct TriColorDepthFirstSearch<'graph, G> +where + G: ?Sized + DirectedGraph + WithNumNodes + WithSuccessors, +{ + graph: &'graph G, + stack: Vec<Event<G::Node>>, + visited: BitSet<G::Node>, + settled: BitSet<G::Node>, +} + +impl<'graph, G> TriColorDepthFirstSearch<'graph, G> +where + G: ?Sized + DirectedGraph + WithNumNodes + WithSuccessors, +{ + pub fn new(graph: &'graph G) -> Self { + TriColorDepthFirstSearch { + graph, + stack: vec![], + visited: BitSet::new_empty(graph.num_nodes()), + settled: BitSet::new_empty(graph.num_nodes()), + } + } + + /// Performs a depth-first search, starting from the given `root`. + /// + /// This won't visit nodes that are not reachable from `root`. + pub fn run_from<V>(mut self, root: G::Node, visitor: &mut V) -> Option<V::BreakVal> + where + V: TriColorVisitor<G>, + { + use NodeStatus::{Settled, Visited}; + + self.stack.push(Event { node: root, becomes: Visited }); + + loop { + match self.stack.pop()? { + Event { node, becomes: Settled } => { + let not_previously_settled = self.settled.insert(node); + assert!(not_previously_settled, "A node should be settled exactly once"); + if let ControlFlow::Break(val) = visitor.node_settled(node) { + return Some(val); + } + } + + Event { node, becomes: Visited } => { + let not_previously_visited = self.visited.insert(node); + let prior_status = if not_previously_visited { + None + } else if self.settled.contains(node) { + Some(Settled) + } else { + Some(Visited) + }; + + if let ControlFlow::Break(val) = visitor.node_examined(node, prior_status) { + return Some(val); + } + + // If this node has already been examined, we are done. + if prior_status.is_some() { + continue; + } + + // Otherwise, push a `Settled` event for this node onto the stack, then + // schedule its successors for examination. + self.stack.push(Event { node, becomes: Settled }); + for succ in self.graph.successors(node) { + if !visitor.ignore_edge(node, succ) { + self.stack.push(Event { node: succ, becomes: Visited }); + } + } + } + } + } + } +} + +impl<G> TriColorDepthFirstSearch<'_, G> +where + G: ?Sized + DirectedGraph + WithNumNodes + WithSuccessors + WithStartNode, +{ + /// Performs a depth-first search, starting from `G::start_node()`. + /// + /// This won't visit nodes that are not reachable from the start node. + pub fn run_from_start<V>(self, visitor: &mut V) -> Option<V::BreakVal> + where + V: TriColorVisitor<G>, + { + let root = self.graph.start_node(); + self.run_from(root, visitor) + } +} + +/// What to do when a node is examined or becomes `Settled` during DFS. +pub trait TriColorVisitor<G> +where + G: ?Sized + DirectedGraph, +{ + /// The value returned by this search. + type BreakVal; + + /// Called when a node is examined by the depth-first search. + /// + /// By checking the value of `prior_status`, this visitor can determine whether the edge + /// leading to this node was a tree edge (`None`), forward edge (`Some(Settled)`) or back edge + /// (`Some(Visited)`). For a full explanation of each edge type, see the "Depth-first Search" + /// chapter in [CLR] or [wikipedia]. + /// + /// If you want to know *both* nodes linked by each edge, you'll need to modify + /// `TriColorDepthFirstSearch` to store a `source` node for each `Visited` event. + /// + /// [wikipedia]: https://en.wikipedia.org/wiki/Depth-first_search#Output_of_a_depth-first_search + /// [CLR]: https://en.wikipedia.org/wiki/Introduction_to_Algorithms + fn node_examined( + &mut self, + _node: G::Node, + _prior_status: Option<NodeStatus>, + ) -> ControlFlow<Self::BreakVal> { + ControlFlow::CONTINUE + } + + /// Called after all nodes reachable from this one have been examined. + fn node_settled(&mut self, _node: G::Node) -> ControlFlow<Self::BreakVal> { + ControlFlow::CONTINUE + } + + /// Behave as if no edges exist from `source` to `target`. + fn ignore_edge(&mut self, _source: G::Node, _target: G::Node) -> bool { + false + } +} + +/// This `TriColorVisitor` looks for back edges in a graph, which indicate that a cycle exists. +pub struct CycleDetector; + +impl<G> TriColorVisitor<G> for CycleDetector +where + G: ?Sized + DirectedGraph, +{ + type BreakVal = (); + + fn node_examined( + &mut self, + _node: G::Node, + prior_status: Option<NodeStatus>, + ) -> ControlFlow<Self::BreakVal> { + match prior_status { + Some(NodeStatus::Visited) => ControlFlow::BREAK, + _ => ControlFlow::CONTINUE, + } + } +} diff --git a/compiler/rustc_data_structures/src/graph/iterate/tests.rs b/compiler/rustc_data_structures/src/graph/iterate/tests.rs new file mode 100644 index 000000000..c498c2893 --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/iterate/tests.rs @@ -0,0 +1,38 @@ +use super::super::tests::TestGraph; + +use super::*; + +#[test] +fn diamond_post_order() { + let graph = TestGraph::new(0, &[(0, 1), (0, 2), (1, 3), (2, 3)]); + + let result = post_order_from(&graph, 0); + assert_eq!(result, vec![3, 1, 2, 0]); +} + +#[test] +fn is_cyclic() { + use super::super::is_cyclic; + + let diamond_acyclic = TestGraph::new(0, &[(0, 1), (0, 2), (1, 3), (2, 3)]); + let diamond_cyclic = TestGraph::new(0, &[(0, 1), (1, 2), (2, 3), (3, 0)]); + + assert!(!is_cyclic(&diamond_acyclic)); + assert!(is_cyclic(&diamond_cyclic)); +} + +#[test] +fn dfs() { + let graph = TestGraph::new(0, &[(0, 1), (0, 2), (1, 3), (2, 3), (3, 0)]); + + let result: Vec<usize> = DepthFirstSearch::new(&graph).with_start_node(0).collect(); + assert_eq!(result, vec![0, 2, 3, 1]); +} + +#[test] +fn dfs_debug() { + let graph = TestGraph::new(0, &[(0, 1), (0, 2), (1, 3), (2, 3), (3, 0)]); + let mut dfs = DepthFirstSearch::new(&graph).with_start_node(0); + dfs.complete_search(); + assert_eq!(format!("{{0, 1, 2, 3}}"), format!("{:?}", dfs)); +} diff --git a/compiler/rustc_data_structures/src/graph/mod.rs b/compiler/rustc_data_structures/src/graph/mod.rs new file mode 100644 index 000000000..3560df6e5 --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/mod.rs @@ -0,0 +1,81 @@ +use rustc_index::vec::Idx; + +pub mod dominators; +pub mod implementation; +pub mod iterate; +mod reference; +pub mod scc; +pub mod vec_graph; + +#[cfg(test)] +mod tests; + +pub trait DirectedGraph { + type Node: Idx; +} + +pub trait WithNumNodes: DirectedGraph { + fn num_nodes(&self) -> usize; +} + +pub trait WithNumEdges: DirectedGraph { + fn num_edges(&self) -> usize; +} + +pub trait WithSuccessors: DirectedGraph +where + Self: for<'graph> GraphSuccessors<'graph, Item = <Self as DirectedGraph>::Node>, +{ + fn successors(&self, node: Self::Node) -> <Self as GraphSuccessors<'_>>::Iter; + + fn depth_first_search(&self, from: Self::Node) -> iterate::DepthFirstSearch<'_, Self> + where + Self: WithNumNodes, + { + iterate::DepthFirstSearch::new(self).with_start_node(from) + } +} + +#[allow(unused_lifetimes)] +pub trait GraphSuccessors<'graph> { + type Item; + type Iter: Iterator<Item = Self::Item>; +} + +pub trait WithPredecessors: DirectedGraph +where + Self: for<'graph> GraphPredecessors<'graph, Item = <Self as DirectedGraph>::Node>, +{ + fn predecessors(&self, node: Self::Node) -> <Self as GraphPredecessors<'_>>::Iter; +} + +#[allow(unused_lifetimes)] +pub trait GraphPredecessors<'graph> { + type Item; + type Iter: Iterator<Item = Self::Item>; +} + +pub trait WithStartNode: DirectedGraph { + fn start_node(&self) -> Self::Node; +} + +pub trait ControlFlowGraph: + DirectedGraph + WithStartNode + WithPredecessors + WithSuccessors + WithNumNodes +{ + // convenient trait +} + +impl<T> ControlFlowGraph for T where + T: DirectedGraph + WithStartNode + WithPredecessors + WithSuccessors + WithNumNodes +{ +} + +/// Returns `true` if the graph has a cycle that is reachable from the start node. +pub fn is_cyclic<G>(graph: &G) -> bool +where + G: ?Sized + DirectedGraph + WithStartNode + WithSuccessors + WithNumNodes, +{ + iterate::TriColorDepthFirstSearch::new(graph) + .run_from_start(&mut iterate::CycleDetector) + .is_some() +} diff --git a/compiler/rustc_data_structures/src/graph/reference.rs b/compiler/rustc_data_structures/src/graph/reference.rs new file mode 100644 index 000000000..c259fe56c --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/reference.rs @@ -0,0 +1,39 @@ +use super::*; + +impl<'graph, G: DirectedGraph> DirectedGraph for &'graph G { + type Node = G::Node; +} + +impl<'graph, G: WithNumNodes> WithNumNodes for &'graph G { + fn num_nodes(&self) -> usize { + (**self).num_nodes() + } +} + +impl<'graph, G: WithStartNode> WithStartNode for &'graph G { + fn start_node(&self) -> Self::Node { + (**self).start_node() + } +} + +impl<'graph, G: WithSuccessors> WithSuccessors for &'graph G { + fn successors(&self, node: Self::Node) -> <Self as GraphSuccessors<'_>>::Iter { + (**self).successors(node) + } +} + +impl<'graph, G: WithPredecessors> WithPredecessors for &'graph G { + fn predecessors(&self, node: Self::Node) -> <Self as GraphPredecessors<'_>>::Iter { + (**self).predecessors(node) + } +} + +impl<'iter, 'graph, G: WithPredecessors> GraphPredecessors<'iter> for &'graph G { + type Item = G::Node; + type Iter = <G as GraphPredecessors<'iter>>::Iter; +} + +impl<'iter, 'graph, G: WithSuccessors> GraphSuccessors<'iter> for &'graph G { + type Item = G::Node; + type Iter = <G as GraphSuccessors<'iter>>::Iter; +} diff --git a/compiler/rustc_data_structures/src/graph/scc/mod.rs b/compiler/rustc_data_structures/src/graph/scc/mod.rs new file mode 100644 index 000000000..7099ca7eb --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/scc/mod.rs @@ -0,0 +1,567 @@ +//! Routine to compute the strongly connected components (SCCs) of a graph. +//! +//! Also computes as the resulting DAG if each SCC is replaced with a +//! node in the graph. This uses [Tarjan's algorithm]( +//! https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm) +//! that completes in *O*(*n*) time. + +use crate::fx::FxHashSet; +use crate::graph::vec_graph::VecGraph; +use crate::graph::{DirectedGraph, GraphSuccessors, WithNumEdges, WithNumNodes, WithSuccessors}; +use rustc_index::vec::{Idx, IndexVec}; +use std::cmp::Ord; +use std::ops::Range; + +#[cfg(test)] +mod tests; + +/// Strongly connected components (SCC) of a graph. The type `N` is +/// the index type for the graph nodes and `S` is the index type for +/// the SCCs. We can map from each node to the SCC that it +/// participates in, and we also have the successors of each SCC. +pub struct Sccs<N: Idx, S: Idx> { + /// For each node, what is the SCC index of the SCC to which it + /// belongs. + scc_indices: IndexVec<N, S>, + + /// Data about each SCC. + scc_data: SccData<S>, +} + +struct SccData<S: Idx> { + /// For each SCC, the range of `all_successors` where its + /// successors can be found. + ranges: IndexVec<S, Range<usize>>, + + /// Contains the successors for all the Sccs, concatenated. The + /// range of indices corresponding to a given SCC is found in its + /// SccData. + all_successors: Vec<S>, +} + +impl<N: Idx, S: Idx + Ord> Sccs<N, S> { + pub fn new(graph: &(impl DirectedGraph<Node = N> + WithNumNodes + WithSuccessors)) -> Self { + SccsConstruction::construct(graph) + } + + /// Returns the number of SCCs in the graph. + pub fn num_sccs(&self) -> usize { + self.scc_data.len() + } + + /// Returns an iterator over the SCCs in the graph. + /// + /// The SCCs will be iterated in **dependency order** (or **post order**), + /// meaning that if `S1 -> S2`, we will visit `S2` first and `S1` after. + /// This is convenient when the edges represent dependencies: when you visit + /// `S1`, the value for `S2` will already have been computed. + pub fn all_sccs(&self) -> impl Iterator<Item = S> { + (0..self.scc_data.len()).map(S::new) + } + + /// Returns the SCC to which a node `r` belongs. + pub fn scc(&self, r: N) -> S { + self.scc_indices[r] + } + + /// Returns the successors of the given SCC. + pub fn successors(&self, scc: S) -> &[S] { + self.scc_data.successors(scc) + } + + /// Construct the reverse graph of the SCC graph. + pub fn reverse(&self) -> VecGraph<S> { + VecGraph::new( + self.num_sccs(), + self.all_sccs() + .flat_map(|source| { + self.successors(source).iter().map(move |&target| (target, source)) + }) + .collect(), + ) + } +} + +impl<N: Idx, S: Idx> DirectedGraph for Sccs<N, S> { + type Node = S; +} + +impl<N: Idx, S: Idx + Ord> WithNumNodes for Sccs<N, S> { + fn num_nodes(&self) -> usize { + self.num_sccs() + } +} + +impl<N: Idx, S: Idx> WithNumEdges for Sccs<N, S> { + fn num_edges(&self) -> usize { + self.scc_data.all_successors.len() + } +} + +impl<'graph, N: Idx, S: Idx> GraphSuccessors<'graph> for Sccs<N, S> { + type Item = S; + + type Iter = std::iter::Cloned<std::slice::Iter<'graph, S>>; +} + +impl<N: Idx, S: Idx + Ord> WithSuccessors for Sccs<N, S> { + fn successors(&self, node: S) -> <Self as GraphSuccessors<'_>>::Iter { + self.successors(node).iter().cloned() + } +} + +impl<S: Idx> SccData<S> { + /// Number of SCCs, + fn len(&self) -> usize { + self.ranges.len() + } + + /// Returns the successors of the given SCC. + fn successors(&self, scc: S) -> &[S] { + // Annoyingly, `range` does not implement `Copy`, so we have + // to do `range.start..range.end`: + let range = &self.ranges[scc]; + &self.all_successors[range.start..range.end] + } + + /// Creates a new SCC with `successors` as its successors and + /// returns the resulting index. + fn create_scc(&mut self, successors: impl IntoIterator<Item = S>) -> S { + // Store the successors on `scc_successors_vec`, remembering + // the range of indices. + let all_successors_start = self.all_successors.len(); + self.all_successors.extend(successors); + let all_successors_end = self.all_successors.len(); + + debug!( + "create_scc({:?}) successors={:?}", + self.ranges.len(), + &self.all_successors[all_successors_start..all_successors_end], + ); + + self.ranges.push(all_successors_start..all_successors_end) + } +} + +struct SccsConstruction<'c, G: DirectedGraph + WithNumNodes + WithSuccessors, S: Idx> { + graph: &'c G, + + /// The state of each node; used during walk to record the stack + /// and after walk to record what cycle each node ended up being + /// in. + node_states: IndexVec<G::Node, NodeState<G::Node, S>>, + + /// The stack of nodes that we are visiting as part of the DFS. + node_stack: Vec<G::Node>, + + /// The stack of successors: as we visit a node, we mark our + /// position in this stack, and when we encounter a successor SCC, + /// we push it on the stack. When we complete an SCC, we can pop + /// everything off the stack that was found along the way. + successors_stack: Vec<S>, + + /// A set used to strip duplicates. As we accumulate successors + /// into the successors_stack, we sometimes get duplicate entries. + /// We use this set to remove those -- we also keep its storage + /// around between successors to amortize memory allocation costs. + duplicate_set: FxHashSet<S>, + + scc_data: SccData<S>, +} + +#[derive(Copy, Clone, Debug)] +enum NodeState<N, S> { + /// This node has not yet been visited as part of the DFS. + /// + /// After SCC construction is complete, this state ought to be + /// impossible. + NotVisited, + + /// This node is currently being walk as part of our DFS. It is on + /// the stack at the depth `depth`. + /// + /// After SCC construction is complete, this state ought to be + /// impossible. + BeingVisited { depth: usize }, + + /// Indicates that this node is a member of the given cycle. + InCycle { scc_index: S }, + + /// Indicates that this node is a member of whatever cycle + /// `parent` is a member of. This state is transient: whenever we + /// see it, we try to overwrite it with the current state of + /// `parent` (this is the "path compression" step of a union-find + /// algorithm). + InCycleWith { parent: N }, +} + +#[derive(Copy, Clone, Debug)] +enum WalkReturn<S> { + Cycle { min_depth: usize }, + Complete { scc_index: S }, +} + +impl<'c, G, S> SccsConstruction<'c, G, S> +where + G: DirectedGraph + WithNumNodes + WithSuccessors, + S: Idx, +{ + /// Identifies SCCs in the graph `G` and computes the resulting + /// DAG. This uses a variant of [Tarjan's + /// algorithm][wikipedia]. The high-level summary of the algorithm + /// is that we do a depth-first search. Along the way, we keep a + /// stack of each node whose successors are being visited. We + /// track the depth of each node on this stack (there is no depth + /// if the node is not on the stack). When we find that some node + /// N with depth D can reach some other node N' with lower depth + /// D' (i.e., D' < D), we know that N, N', and all nodes in + /// between them on the stack are part of an SCC. + /// + /// [wikipedia]: https://bit.ly/2EZIx84 + fn construct(graph: &'c G) -> Sccs<G::Node, S> { + let num_nodes = graph.num_nodes(); + + let mut this = Self { + graph, + node_states: IndexVec::from_elem_n(NodeState::NotVisited, num_nodes), + node_stack: Vec::with_capacity(num_nodes), + successors_stack: Vec::new(), + scc_data: SccData { ranges: IndexVec::new(), all_successors: Vec::new() }, + duplicate_set: FxHashSet::default(), + }; + + let scc_indices = (0..num_nodes) + .map(G::Node::new) + .map(|node| match this.start_walk_from(node) { + WalkReturn::Complete { scc_index } => scc_index, + WalkReturn::Cycle { min_depth } => panic!( + "`start_walk_node({:?})` returned cycle with depth {:?}", + node, min_depth + ), + }) + .collect(); + + Sccs { scc_indices, scc_data: this.scc_data } + } + + fn start_walk_from(&mut self, node: G::Node) -> WalkReturn<S> { + if let Some(result) = self.inspect_node(node) { + result + } else { + self.walk_unvisited_node(node) + } + } + + /// Inspect a node during the DFS. We first examine its current + /// state -- if it is not yet visited (`NotVisited`), return `None` so + /// that the caller might push it onto the stack and start walking its + /// successors. + /// + /// If it is already on the DFS stack it will be in the state + /// `BeingVisited`. In that case, we have found a cycle and we + /// return the depth from the stack. + /// + /// Otherwise, we are looking at a node that has already been + /// completely visited. We therefore return `WalkReturn::Complete` + /// with its associated SCC index. + fn inspect_node(&mut self, node: G::Node) -> Option<WalkReturn<S>> { + Some(match self.find_state(node) { + NodeState::InCycle { scc_index } => WalkReturn::Complete { scc_index }, + + NodeState::BeingVisited { depth: min_depth } => WalkReturn::Cycle { min_depth }, + + NodeState::NotVisited => return None, + + NodeState::InCycleWith { parent } => panic!( + "`find_state` returned `InCycleWith({:?})`, which ought to be impossible", + parent + ), + }) + } + + /// Fetches the state of the node `r`. If `r` is recorded as being + /// in a cycle with some other node `r2`, then fetches the state + /// of `r2` (and updates `r` to reflect current result). This is + /// basically the "find" part of a standard union-find algorithm + /// (with path compression). + fn find_state(&mut self, mut node: G::Node) -> NodeState<G::Node, S> { + // To avoid recursion we temporarily reuse the `parent` of each + // InCycleWith link to encode a downwards link while compressing + // the path. After we have found the root or deepest node being + // visited, we traverse the reverse links and correct the node + // states on the way. + // + // **Note**: This mutation requires that this is a leaf function + // or at least that none of the called functions inspects the + // current node states. Luckily, we are a leaf. + + // Remember one previous link. The termination condition when + // following links downwards is then simply as soon as we have + // found the initial self-loop. + let mut previous_node = node; + + // Ultimately assigned by the parent when following + // `InCycleWith` upwards. + let node_state = loop { + debug!("find_state(r = {:?} in state {:?})", node, self.node_states[node]); + match self.node_states[node] { + NodeState::InCycle { scc_index } => break NodeState::InCycle { scc_index }, + NodeState::BeingVisited { depth } => break NodeState::BeingVisited { depth }, + NodeState::NotVisited => break NodeState::NotVisited, + NodeState::InCycleWith { parent } => { + // We test this, to be extremely sure that we never + // ever break our termination condition for the + // reverse iteration loop. + assert!(node != parent, "Node can not be in cycle with itself"); + // Store the previous node as an inverted list link + self.node_states[node] = NodeState::InCycleWith { parent: previous_node }; + // Update to parent node. + previous_node = node; + node = parent; + } + } + }; + + // The states form a graph where up to one outgoing link is stored at + // each node. Initially in general, + // + // E + // ^ + // | + // InCycleWith/BeingVisited/NotVisited + // | + // A-InCycleWith->B-InCycleWith…>C-InCycleWith->D-+ + // | + // = node, previous_node + // + // After the first loop, this will look like + // E + // ^ + // | + // InCycleWith/BeingVisited/NotVisited + // | + // +>A<-InCycleWith-B<…InCycleWith-C<-InCycleWith-D-+ + // | | | | + // | InCycleWith | = node + // +-+ =previous_node + // + // Note in particular that A will be linked to itself in a self-cycle + // and no other self-cycles occur due to how InCycleWith is assigned in + // the find phase implemented by `walk_unvisited_node`. + // + // We now want to compress the path, that is assign the state of the + // link D-E to all other links. + // + // We can then walk backwards, starting from `previous_node`, and assign + // each node in the list with the updated state. The loop terminates + // when we reach the self-cycle. + + // Move backwards until we found the node where we started. We + // will know when we hit the state where previous_node == node. + loop { + // Back at the beginning, we can return. + if previous_node == node { + return node_state; + } + // Update to previous node in the link. + match self.node_states[previous_node] { + NodeState::InCycleWith { parent: previous } => { + node = previous_node; + previous_node = previous; + } + // Only InCycleWith nodes were added to the reverse linked list. + other => panic!("Invalid previous link while compressing cycle: {:?}", other), + } + + debug!("find_state: parent_state = {:?}", node_state); + + // Update the node state from the parent state. The assigned + // state is actually a loop invariant but it will only be + // evaluated if there is at least one backlink to follow. + // Fully trusting llvm here to find this loop optimization. + match node_state { + // Path compression, make current node point to the same root. + NodeState::InCycle { .. } => { + self.node_states[node] = node_state; + } + // Still visiting nodes, compress to cycle to the node + // at that depth. + NodeState::BeingVisited { depth } => { + self.node_states[node] = + NodeState::InCycleWith { parent: self.node_stack[depth] }; + } + // These are never allowed as parent nodes. InCycleWith + // should have been followed to a real parent and + // NotVisited can not be part of a cycle since it should + // have instead gotten explored. + NodeState::NotVisited | NodeState::InCycleWith { .. } => { + panic!("invalid parent state: {:?}", node_state) + } + } + } + } + + /// Walks a node that has never been visited before. + /// + /// Call this method when `inspect_node` has returned `None`. Having the + /// caller decide avoids mutual recursion between the two methods and allows + /// us to maintain an allocated stack for nodes on the path between calls. + #[instrument(skip(self, initial), level = "debug")] + fn walk_unvisited_node(&mut self, initial: G::Node) -> WalkReturn<S> { + struct VisitingNodeFrame<G: DirectedGraph, Successors> { + node: G::Node, + iter: Option<Successors>, + depth: usize, + min_depth: usize, + successors_len: usize, + min_cycle_root: G::Node, + successor_node: G::Node, + } + + // Move the stack to a local variable. We want to utilize the existing allocation and + // mutably borrow it without borrowing self at the same time. + let mut successors_stack = core::mem::take(&mut self.successors_stack); + debug_assert_eq!(successors_stack.len(), 0); + + let mut stack: Vec<VisitingNodeFrame<G, _>> = vec![VisitingNodeFrame { + node: initial, + depth: 0, + min_depth: 0, + iter: None, + successors_len: 0, + min_cycle_root: initial, + successor_node: initial, + }]; + + let mut return_value = None; + + 'recurse: while let Some(frame) = stack.last_mut() { + let VisitingNodeFrame { + node, + depth, + iter, + successors_len, + min_depth, + min_cycle_root, + successor_node, + } = frame; + + let node = *node; + let depth = *depth; + + let successors = match iter { + Some(iter) => iter, + None => { + // This None marks that we still have the initialize this node's frame. + debug!(?depth, ?node); + + debug_assert!(matches!(self.node_states[node], NodeState::NotVisited)); + + // Push `node` onto the stack. + self.node_states[node] = NodeState::BeingVisited { depth }; + self.node_stack.push(node); + + // Walk each successor of the node, looking to see if any of + // them can reach a node that is presently on the stack. If + // so, that means they can also reach us. + *successors_len = successors_stack.len(); + // Set and return a reference, this is currently empty. + iter.get_or_insert(self.graph.successors(node)) + } + }; + + // Now that iter is initialized, this is a constant for this frame. + let successors_len = *successors_len; + + // Construct iterators for the nodes and walk results. There are two cases: + // * The walk of a successor node returned. + // * The remaining successor nodes. + let returned_walk = + return_value.take().into_iter().map(|walk| (*successor_node, Some(walk))); + + let successor_walk = successors.by_ref().map(|successor_node| { + debug!(?node, ?successor_node); + (successor_node, self.inspect_node(successor_node)) + }); + + for (successor_node, walk) in returned_walk.chain(successor_walk) { + match walk { + Some(WalkReturn::Cycle { min_depth: successor_min_depth }) => { + // Track the minimum depth we can reach. + assert!(successor_min_depth <= depth); + if successor_min_depth < *min_depth { + debug!(?node, ?successor_min_depth); + *min_depth = successor_min_depth; + *min_cycle_root = successor_node; + } + } + + Some(WalkReturn::Complete { scc_index: successor_scc_index }) => { + // Push the completed SCC indices onto + // the `successors_stack` for later. + debug!(?node, ?successor_scc_index); + successors_stack.push(successor_scc_index); + } + + None => { + let depth = depth + 1; + debug!(?depth, ?successor_node); + // Remember which node the return value will come from. + frame.successor_node = successor_node; + // Start a new stack frame the step into it. + stack.push(VisitingNodeFrame { + node: successor_node, + depth, + iter: None, + successors_len: 0, + min_depth: depth, + min_cycle_root: successor_node, + successor_node, + }); + continue 'recurse; + } + } + } + + // Completed walk, remove `node` from the stack. + let r = self.node_stack.pop(); + debug_assert_eq!(r, Some(node)); + + // Remove the frame, it's done. + let frame = stack.pop().unwrap(); + + // If `min_depth == depth`, then we are the root of the + // cycle: we can't reach anyone further down the stack. + + // Pass the 'return value' down the stack. + // We return one frame at a time so there can't be another return value. + debug_assert!(return_value.is_none()); + return_value = Some(if frame.min_depth == depth { + // Note that successor stack may have duplicates, so we + // want to remove those: + let deduplicated_successors = { + let duplicate_set = &mut self.duplicate_set; + duplicate_set.clear(); + successors_stack + .drain(successors_len..) + .filter(move |&i| duplicate_set.insert(i)) + }; + let scc_index = self.scc_data.create_scc(deduplicated_successors); + self.node_states[node] = NodeState::InCycle { scc_index }; + WalkReturn::Complete { scc_index } + } else { + // We are not the head of the cycle. Return back to our + // caller. They will take ownership of the + // `self.successors` data that we pushed. + self.node_states[node] = NodeState::InCycleWith { parent: frame.min_cycle_root }; + WalkReturn::Cycle { min_depth: frame.min_depth } + }); + } + + // Keep the allocation we used for successors_stack. + self.successors_stack = successors_stack; + debug_assert_eq!(self.successors_stack.len(), 0); + + return_value.unwrap() + } +} diff --git a/compiler/rustc_data_structures/src/graph/scc/tests.rs b/compiler/rustc_data_structures/src/graph/scc/tests.rs new file mode 100644 index 000000000..9940fee60 --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/scc/tests.rs @@ -0,0 +1,216 @@ +extern crate test; + +use super::*; +use crate::graph::tests::TestGraph; + +#[test] +fn diamond() { + let graph = TestGraph::new(0, &[(0, 1), (0, 2), (1, 3), (2, 3)]); + let sccs: Sccs<_, usize> = Sccs::new(&graph); + assert_eq!(sccs.num_sccs(), 4); + assert_eq!(sccs.num_sccs(), 4); +} + +#[test] +fn test_big_scc() { + // The order in which things will be visited is important to this + // test. + // + // We will visit: + // + // 0 -> 1 -> 2 -> 0 + // + // and at this point detect a cycle. 2 will return back to 1 which + // will visit 3. 3 will visit 2 before the cycle is complete, and + // hence it too will return a cycle. + + /* + +-> 0 + | | + | v + | 1 -> 3 + | | | + | v | + +-- 2 <--+ + */ + let graph = TestGraph::new(0, &[(0, 1), (1, 2), (1, 3), (2, 0), (3, 2)]); + let sccs: Sccs<_, usize> = Sccs::new(&graph); + assert_eq!(sccs.num_sccs(), 1); +} + +#[test] +fn test_three_sccs() { + /* + 0 + | + v + +-> 1 3 + | | | + | v | + +-- 2 <--+ + */ + let graph = TestGraph::new(0, &[(0, 1), (1, 2), (2, 1), (3, 2)]); + let sccs: Sccs<_, usize> = Sccs::new(&graph); + assert_eq!(sccs.num_sccs(), 3); + assert_eq!(sccs.scc(0), 1); + assert_eq!(sccs.scc(1), 0); + assert_eq!(sccs.scc(2), 0); + assert_eq!(sccs.scc(3), 2); + assert_eq!(sccs.successors(0), &[]); + assert_eq!(sccs.successors(1), &[0]); + assert_eq!(sccs.successors(2), &[0]); +} + +#[test] +fn test_find_state_2() { + // The order in which things will be visited is important to this + // test. It tests part of the `find_state` behavior. Here is the + // graph: + // + // + // /----+ + // 0 <--+ | + // | | | + // v | | + // +-> 1 -> 3 4 + // | | | + // | v | + // +-- 2 <----+ + + let graph = TestGraph::new(0, &[(0, 1), (0, 4), (1, 2), (1, 3), (2, 1), (3, 0), (4, 2)]); + + // For this graph, we will start in our DFS by visiting: + // + // 0 -> 1 -> 2 -> 1 + // + // and at this point detect a cycle. The state of 2 will thus be + // `InCycleWith { 1 }`. We will then visit the 1 -> 3 edge, which + // will attempt to visit 0 as well, thus going to the state + // `InCycleWith { 0 }`. Finally, node 1 will complete; the lowest + // depth of any successor was 3 which had depth 0, and thus it + // will be in the state `InCycleWith { 3 }`. + // + // When we finally traverse the `0 -> 4` edge and then visit node 2, + // the states of the nodes are: + // + // 0 BeingVisited { 0 } + // 1 InCycleWith { 3 } + // 2 InCycleWith { 1 } + // 3 InCycleWith { 0 } + // + // and hence 4 will traverse the links, finding an ultimate depth of 0. + // If will also collapse the states to the following: + // + // 0 BeingVisited { 0 } + // 1 InCycleWith { 3 } + // 2 InCycleWith { 1 } + // 3 InCycleWith { 0 } + + let sccs: Sccs<_, usize> = Sccs::new(&graph); + assert_eq!(sccs.num_sccs(), 1); + assert_eq!(sccs.scc(0), 0); + assert_eq!(sccs.scc(1), 0); + assert_eq!(sccs.scc(2), 0); + assert_eq!(sccs.scc(3), 0); + assert_eq!(sccs.scc(4), 0); + assert_eq!(sccs.successors(0), &[]); +} + +#[test] +fn test_find_state_3() { + /* + /----+ + 0 <--+ | + | | | + v | | + +-> 1 -> 3 4 5 + | | | | + | v | | + +-- 2 <----+-+ + */ + let graph = + TestGraph::new(0, &[(0, 1), (0, 4), (1, 2), (1, 3), (2, 1), (3, 0), (4, 2), (5, 2)]); + let sccs: Sccs<_, usize> = Sccs::new(&graph); + assert_eq!(sccs.num_sccs(), 2); + assert_eq!(sccs.scc(0), 0); + assert_eq!(sccs.scc(1), 0); + assert_eq!(sccs.scc(2), 0); + assert_eq!(sccs.scc(3), 0); + assert_eq!(sccs.scc(4), 0); + assert_eq!(sccs.scc(5), 1); + assert_eq!(sccs.successors(0), &[]); + assert_eq!(sccs.successors(1), &[0]); +} + +#[test] +fn test_deep_linear() { + /* + 0 + | + v + 1 + | + v + 2 + | + v + … + */ + #[cfg(not(miri))] + const NR_NODES: usize = 1 << 14; + #[cfg(miri)] + const NR_NODES: usize = 1 << 3; + let mut nodes = vec![]; + for i in 1..NR_NODES { + nodes.push((i - 1, i)); + } + let graph = TestGraph::new(0, nodes.as_slice()); + let sccs: Sccs<_, usize> = Sccs::new(&graph); + assert_eq!(sccs.num_sccs(), NR_NODES); + assert_eq!(sccs.scc(0), NR_NODES - 1); + assert_eq!(sccs.scc(NR_NODES - 1), 0); +} + +#[bench] +fn bench_sccc(b: &mut test::Bencher) { + // Like `test_three_sccs` but each state is replaced by a group of + // three or four to have some amount of test data. + /* + 0-3 + | + v + +->4-6 11-14 + | | | + | v | + +--7-10<-+ + */ + fn make_3_clique(slice: &mut [(usize, usize)], base: usize) { + slice[0] = (base + 0, base + 1); + slice[1] = (base + 1, base + 2); + slice[2] = (base + 2, base + 0); + } + // Not actually a clique but strongly connected. + fn make_4_clique(slice: &mut [(usize, usize)], base: usize) { + slice[0] = (base + 0, base + 1); + slice[1] = (base + 1, base + 2); + slice[2] = (base + 2, base + 3); + slice[3] = (base + 3, base + 0); + slice[4] = (base + 1, base + 3); + slice[5] = (base + 2, base + 1); + } + + let mut graph = [(0, 0); 6 + 3 + 6 + 3 + 4]; + make_4_clique(&mut graph[0..6], 0); + make_3_clique(&mut graph[6..9], 4); + make_4_clique(&mut graph[9..15], 7); + make_3_clique(&mut graph[15..18], 11); + graph[18] = (0, 4); + graph[19] = (5, 7); + graph[20] = (11, 10); + graph[21] = (7, 4); + let graph = TestGraph::new(0, &graph[..]); + b.iter(|| { + let sccs: Sccs<_, usize> = Sccs::new(&graph); + assert_eq!(sccs.num_sccs(), 3); + }); +} diff --git a/compiler/rustc_data_structures/src/graph/tests.rs b/compiler/rustc_data_structures/src/graph/tests.rs new file mode 100644 index 000000000..7f4ef906b --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/tests.rs @@ -0,0 +1,73 @@ +use crate::fx::FxHashMap; +use std::cmp::max; +use std::iter; +use std::slice; + +use super::*; + +pub struct TestGraph { + num_nodes: usize, + start_node: usize, + successors: FxHashMap<usize, Vec<usize>>, + predecessors: FxHashMap<usize, Vec<usize>>, +} + +impl TestGraph { + pub fn new(start_node: usize, edges: &[(usize, usize)]) -> Self { + let mut graph = TestGraph { + num_nodes: start_node + 1, + start_node, + successors: FxHashMap::default(), + predecessors: FxHashMap::default(), + }; + for &(source, target) in edges { + graph.num_nodes = max(graph.num_nodes, source + 1); + graph.num_nodes = max(graph.num_nodes, target + 1); + graph.successors.entry(source).or_default().push(target); + graph.predecessors.entry(target).or_default().push(source); + } + for node in 0..graph.num_nodes { + graph.successors.entry(node).or_default(); + graph.predecessors.entry(node).or_default(); + } + graph + } +} + +impl DirectedGraph for TestGraph { + type Node = usize; +} + +impl WithStartNode for TestGraph { + fn start_node(&self) -> usize { + self.start_node + } +} + +impl WithNumNodes for TestGraph { + fn num_nodes(&self) -> usize { + self.num_nodes + } +} + +impl WithPredecessors for TestGraph { + fn predecessors(&self, node: usize) -> <Self as GraphPredecessors<'_>>::Iter { + self.predecessors[&node].iter().cloned() + } +} + +impl WithSuccessors for TestGraph { + fn successors(&self, node: usize) -> <Self as GraphSuccessors<'_>>::Iter { + self.successors[&node].iter().cloned() + } +} + +impl<'graph> GraphPredecessors<'graph> for TestGraph { + type Item = usize; + type Iter = iter::Cloned<slice::Iter<'graph, usize>>; +} + +impl<'graph> GraphSuccessors<'graph> for TestGraph { + type Item = usize; + type Iter = iter::Cloned<slice::Iter<'graph, usize>>; +} diff --git a/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs b/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs new file mode 100644 index 000000000..3d91bcade --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs @@ -0,0 +1,109 @@ +use std::cmp::Ord; + +use crate::graph::{DirectedGraph, GraphSuccessors, WithNumEdges, WithNumNodes, WithSuccessors}; +use rustc_index::vec::{Idx, IndexVec}; + +#[cfg(test)] +mod tests; + +pub struct VecGraph<N: Idx> { + /// Maps from a given node to an index where the set of successors + /// for that node starts. The index indexes into the `edges` + /// vector. To find the range for a given node, we look up the + /// start for that node and then the start for the next node + /// (i.e., with an index 1 higher) and get the range between the + /// two. This vector always has an extra entry so that this works + /// even for the max element. + node_starts: IndexVec<N, usize>, + + edge_targets: Vec<N>, +} + +impl<N: Idx + Ord> VecGraph<N> { + pub fn new(num_nodes: usize, mut edge_pairs: Vec<(N, N)>) -> Self { + // Sort the edges by the source -- this is important. + edge_pairs.sort(); + + let num_edges = edge_pairs.len(); + + // Store the *target* of each edge into `edge_targets`. + let edge_targets: Vec<N> = edge_pairs.iter().map(|&(_, target)| target).collect(); + + // Create the *edge starts* array. We are iterating over over + // the (sorted) edge pairs. We maintain the invariant that the + // length of the `node_starts` array is enough to store the + // current source node -- so when we see that the source node + // for an edge is greater than the current length, we grow the + // edge-starts array by just enough. + let mut node_starts = IndexVec::with_capacity(num_edges); + for (index, &(source, _)) in edge_pairs.iter().enumerate() { + // If we have a list like `[(0, x), (2, y)]`: + // + // - Start out with `node_starts` of `[]` + // - Iterate to `(0, x)` at index 0: + // - Push one entry because `node_starts.len()` (0) is <= the source (0) + // - Leaving us with `node_starts` of `[0]` + // - Iterate to `(2, y)` at index 1: + // - Push one entry because `node_starts.len()` (1) is <= the source (2) + // - Push one entry because `node_starts.len()` (2) is <= the source (2) + // - Leaving us with `node_starts` of `[0, 1, 1]` + // - Loop terminates + while node_starts.len() <= source.index() { + node_starts.push(index); + } + } + + // Pad out the `node_starts` array so that it has `num_nodes + + // 1` entries. Continuing our example above, if `num_nodes` is + // be `3`, we would push one more index: `[0, 1, 1, 2]`. + // + // Interpretation of that vector: + // + // [0, 1, 1, 2] + // ---- range for N=2 + // ---- range for N=1 + // ---- range for N=0 + while node_starts.len() <= num_nodes { + node_starts.push(edge_targets.len()); + } + + assert_eq!(node_starts.len(), num_nodes + 1); + + Self { node_starts, edge_targets } + } + + /// Gets the successors for `source` as a slice. + pub fn successors(&self, source: N) -> &[N] { + let start_index = self.node_starts[source]; + let end_index = self.node_starts[source.plus(1)]; + &self.edge_targets[start_index..end_index] + } +} + +impl<N: Idx> DirectedGraph for VecGraph<N> { + type Node = N; +} + +impl<N: Idx> WithNumNodes for VecGraph<N> { + fn num_nodes(&self) -> usize { + self.node_starts.len() - 1 + } +} + +impl<N: Idx> WithNumEdges for VecGraph<N> { + fn num_edges(&self) -> usize { + self.edge_targets.len() + } +} + +impl<'graph, N: Idx> GraphSuccessors<'graph> for VecGraph<N> { + type Item = N; + + type Iter = std::iter::Cloned<std::slice::Iter<'graph, N>>; +} + +impl<N: Idx + Ord> WithSuccessors for VecGraph<N> { + fn successors(&self, node: N) -> <Self as GraphSuccessors<'_>>::Iter { + self.successors(node).iter().cloned() + } +} diff --git a/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs b/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs new file mode 100644 index 000000000..c8f979267 --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs @@ -0,0 +1,42 @@ +use super::*; + +fn create_graph() -> VecGraph<usize> { + // Create a simple graph + // + // 5 + // | + // V + // 0 --> 1 --> 2 + // | + // v + // 3 --> 4 + // + // 6 + + VecGraph::new(7, vec![(0, 1), (1, 2), (1, 3), (3, 4), (5, 1)]) +} + +#[test] +fn num_nodes() { + let graph = create_graph(); + assert_eq!(graph.num_nodes(), 7); +} + +#[test] +fn successors() { + let graph = create_graph(); + assert_eq!(graph.successors(0), &[1]); + assert_eq!(graph.successors(1), &[2, 3]); + assert_eq!(graph.successors(2), &[]); + assert_eq!(graph.successors(3), &[4]); + assert_eq!(graph.successors(4), &[]); + assert_eq!(graph.successors(5), &[1]); + assert_eq!(graph.successors(6), &[]); +} + +#[test] +fn dfs() { + let graph = create_graph(); + let dfs: Vec<_> = graph.depth_first_search(0).collect(); + assert_eq!(dfs, vec![0, 1, 3, 4, 2]); +} diff --git a/compiler/rustc_data_structures/src/intern.rs b/compiler/rustc_data_structures/src/intern.rs new file mode 100644 index 000000000..009b5d534 --- /dev/null +++ b/compiler/rustc_data_structures/src/intern.rs @@ -0,0 +1,196 @@ +use crate::stable_hasher::{HashStable, StableHasher}; +use std::cmp::Ordering; +use std::hash::{Hash, Hasher}; +use std::ops::Deref; +use std::ptr; + +use crate::fingerprint::Fingerprint; + +mod private { + #[derive(Clone, Copy, Debug)] + pub struct PrivateZst; +} + +/// A reference to a value that is interned, and is known to be unique. +/// +/// Note that it is possible to have a `T` and a `Interned<T>` that are (or +/// refer to) equal but different values. But if you have two different +/// `Interned<T>`s, they both refer to the same value, at a single location in +/// memory. This means that equality and hashing can be done on the value's +/// address rather than the value's contents, which can improve performance. +/// +/// The `PrivateZst` field means you can pattern match with `Interned(v, _)` +/// but you can only construct a `Interned` with `new_unchecked`, and not +/// directly. +#[derive(Debug)] +#[rustc_pass_by_value] +pub struct Interned<'a, T>(pub &'a T, pub private::PrivateZst); + +impl<'a, T> Interned<'a, T> { + /// Create a new `Interned` value. The value referred to *must* be interned + /// and thus be unique, and it *must* remain unique in the future. This + /// function has `_unchecked` in the name but is not `unsafe`, because if + /// the uniqueness condition is violated condition it will cause incorrect + /// behaviour but will not affect memory safety. + #[inline] + pub const fn new_unchecked(t: &'a T) -> Self { + Interned(t, private::PrivateZst) + } +} + +impl<'a, T> Clone for Interned<'a, T> { + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T> Copy for Interned<'a, T> {} + +impl<'a, T> Deref for Interned<'a, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &T { + self.0 + } +} + +impl<'a, T> PartialEq for Interned<'a, T> { + #[inline] + fn eq(&self, other: &Self) -> bool { + // Pointer equality implies equality, due to the uniqueness constraint. + ptr::eq(self.0, other.0) + } +} + +impl<'a, T> Eq for Interned<'a, T> {} + +impl<'a, T: PartialOrd> PartialOrd for Interned<'a, T> { + fn partial_cmp(&self, other: &Interned<'a, T>) -> Option<Ordering> { + // Pointer equality implies equality, due to the uniqueness constraint, + // but the contents must be compared otherwise. + if ptr::eq(self.0, other.0) { + Some(Ordering::Equal) + } else { + let res = self.0.partial_cmp(&other.0); + debug_assert_ne!(res, Some(Ordering::Equal)); + res + } + } +} + +impl<'a, T: Ord> Ord for Interned<'a, T> { + fn cmp(&self, other: &Interned<'a, T>) -> Ordering { + // Pointer equality implies equality, due to the uniqueness constraint, + // but the contents must be compared otherwise. + if ptr::eq(self.0, other.0) { + Ordering::Equal + } else { + let res = self.0.cmp(&other.0); + debug_assert_ne!(res, Ordering::Equal); + res + } + } +} + +impl<'a, T> Hash for Interned<'a, T> { + #[inline] + fn hash<H: Hasher>(&self, s: &mut H) { + // Pointer hashing is sufficient, due to the uniqueness constraint. + ptr::hash(self.0, s) + } +} + +impl<T, CTX> HashStable<CTX> for Interned<'_, T> +where + T: HashStable<CTX>, +{ + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + self.0.hash_stable(hcx, hasher); + } +} + +/// A helper trait so that `Interned` things can cache stable hashes reproducibly. +pub trait InternedHashingContext { + fn with_def_path_and_no_spans(&mut self, f: impl FnOnce(&mut Self)); +} + +/// A helper type that you can wrap round your own type in order to automatically +/// cache the stable hash on creation and not recompute it whenever the stable hash +/// of the type is computed. +/// This is only done in incremental mode. You can also opt out of caching by using +/// StableHash::ZERO for the hash, in which case the hash gets computed each time. +/// This is useful if you have values that you intern but never (can?) use for stable +/// hashing. +#[derive(Copy, Clone)] +pub struct WithStableHash<T> { + pub internee: T, + pub stable_hash: Fingerprint, +} + +impl<T: PartialEq> PartialEq for WithStableHash<T> { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.internee.eq(&other.internee) + } +} + +impl<T: Eq> Eq for WithStableHash<T> {} + +impl<T: Ord> PartialOrd for WithStableHash<T> { + fn partial_cmp(&self, other: &WithStableHash<T>) -> Option<Ordering> { + Some(self.internee.cmp(&other.internee)) + } +} + +impl<T: Ord> Ord for WithStableHash<T> { + fn cmp(&self, other: &WithStableHash<T>) -> Ordering { + self.internee.cmp(&other.internee) + } +} + +impl<T> Deref for WithStableHash<T> { + type Target = T; + + #[inline] + fn deref(&self) -> &T { + &self.internee + } +} + +impl<T: Hash> Hash for WithStableHash<T> { + #[inline] + fn hash<H: Hasher>(&self, s: &mut H) { + self.internee.hash(s) + } +} + +impl<T: HashStable<CTX>, CTX: InternedHashingContext> HashStable<CTX> for WithStableHash<T> { + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + if self.stable_hash == Fingerprint::ZERO || cfg!(debug_assertions) { + // No cached hash available. This can only mean that incremental is disabled. + // We don't cache stable hashes in non-incremental mode, because they are used + // so rarely that the performance actually suffers. + + // We need to build the hash as if we cached it and then hash that hash, as + // otherwise the hashes will differ between cached and non-cached mode. + let stable_hash: Fingerprint = { + let mut hasher = StableHasher::new(); + hcx.with_def_path_and_no_spans(|hcx| self.internee.hash_stable(hcx, &mut hasher)); + hasher.finish() + }; + if cfg!(debug_assertions) && self.stable_hash != Fingerprint::ZERO { + assert_eq!( + stable_hash, self.stable_hash, + "cached stable hash does not match freshly computed stable hash" + ); + } + stable_hash.hash_stable(hcx, hasher); + } else { + self.stable_hash.hash_stable(hcx, hasher); + } + } +} + +#[cfg(test)] +mod tests; diff --git a/compiler/rustc_data_structures/src/intern/tests.rs b/compiler/rustc_data_structures/src/intern/tests.rs new file mode 100644 index 000000000..09810a085 --- /dev/null +++ b/compiler/rustc_data_structures/src/intern/tests.rs @@ -0,0 +1,59 @@ +use super::*; +use std::cmp::Ordering; + +#[derive(Debug)] +struct S(u32); + +impl PartialEq for S { + fn eq(&self, _other: &Self) -> bool { + panic!("shouldn't be called"); + } +} + +impl Eq for S {} + +impl PartialOrd for S { + fn partial_cmp(&self, other: &S) -> Option<Ordering> { + // The `==` case should be handled by `Interned`. + assert_ne!(self.0, other.0); + self.0.partial_cmp(&other.0) + } +} + +impl Ord for S { + fn cmp(&self, other: &S) -> Ordering { + // The `==` case should be handled by `Interned`. + assert_ne!(self.0, other.0); + self.0.cmp(&other.0) + } +} + +#[test] +fn test_uniq() { + let s1 = S(1); + let s2 = S(2); + let s3 = S(3); + let s4 = S(1); // violates uniqueness + + let v1 = Interned::new_unchecked(&s1); + let v2 = Interned::new_unchecked(&s2); + let v3a = Interned::new_unchecked(&s3); + let v3b = Interned::new_unchecked(&s3); + let v4 = Interned::new_unchecked(&s4); // violates uniqueness + + assert_ne!(v1, v2); + assert_ne!(v2, v3a); + assert_eq!(v1, v1); + assert_eq!(v3a, v3b); + assert_ne!(v1, v4); // same content but different addresses: not equal + + assert_eq!(v1.cmp(&v2), Ordering::Less); + assert_eq!(v3a.cmp(&v2), Ordering::Greater); + assert_eq!(v1.cmp(&v1), Ordering::Equal); // only uses Interned::eq, not S::cmp + assert_eq!(v3a.cmp(&v3b), Ordering::Equal); // only uses Interned::eq, not S::cmp + + assert_eq!(v1.partial_cmp(&v2), Some(Ordering::Less)); + assert_eq!(v3a.partial_cmp(&v2), Some(Ordering::Greater)); + assert_eq!(v1.partial_cmp(&v1), Some(Ordering::Equal)); // only uses Interned::eq, not S::cmp + assert_eq!(v3a.partial_cmp(&v3b), Some(Ordering::Equal)); // only uses Interned::eq, not S::cmp +} diff --git a/compiler/rustc_data_structures/src/jobserver.rs b/compiler/rustc_data_structures/src/jobserver.rs new file mode 100644 index 000000000..09baa3095 --- /dev/null +++ b/compiler/rustc_data_structures/src/jobserver.rs @@ -0,0 +1,40 @@ +pub use jobserver_crate::Client; +use std::sync::LazyLock; + +// We can only call `from_env` once per process + +// Note that this is unsafe because it may misinterpret file descriptors +// on Unix as jobserver file descriptors. We hopefully execute this near +// the beginning of the process though to ensure we don't get false +// positives, or in other words we try to execute this before we open +// any file descriptors ourselves. +// +// Pick a "reasonable maximum" if we don't otherwise have +// a jobserver in our environment, capping out at 32 so we +// don't take everything down by hogging the process run queue. +// The fixed number is used to have deterministic compilation +// across machines. +// +// Also note that we stick this in a global because there could be +// multiple rustc instances in this process, and the jobserver is +// per-process. +static GLOBAL_CLIENT: LazyLock<Client> = LazyLock::new(|| unsafe { + Client::from_env().unwrap_or_else(|| { + let client = Client::new(32).expect("failed to create jobserver"); + // Acquire a token for the main thread which we can release later + client.acquire_raw().ok(); + client + }) +}); + +pub fn client() -> Client { + GLOBAL_CLIENT.clone() +} + +pub fn acquire_thread() { + GLOBAL_CLIENT.acquire_raw().ok(); +} + +pub fn release_thread() { + GLOBAL_CLIENT.release_raw().ok(); +} diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs new file mode 100644 index 000000000..265f45b72 --- /dev/null +++ b/compiler/rustc_data_structures/src/lib.rs @@ -0,0 +1,113 @@ +//! Various data structures used by the Rust compiler. The intention +//! is that code in here should be not be *specific* to rustc, so that +//! it can be easily unit tested and so forth. +//! +//! # Note +//! +//! This API is completely unstable and subject to change. + +#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] +#![feature(array_windows)] +#![feature(associated_type_bounds)] +#![feature(auto_traits)] +#![feature(cell_leak)] +#![feature(control_flow_enum)] +#![feature(extend_one)] +#![feature(let_else)] +#![feature(hash_raw_entry)] +#![feature(hasher_prefixfree_extras)] +#![feature(maybe_uninit_uninit_array)] +#![feature(min_specialization)] +#![feature(never_type)] +#![feature(type_alias_impl_trait)] +#![feature(new_uninit)] +#![feature(once_cell)] +#![feature(rustc_attrs)] +#![feature(test)] +#![feature(thread_id_value)] +#![feature(vec_into_raw_parts)] +#![allow(rustc::default_hash_types)] +#![allow(rustc::potential_query_instability)] + +#[macro_use] +extern crate tracing; +#[macro_use] +extern crate cfg_if; +#[macro_use] +extern crate rustc_macros; + +pub use rustc_index::static_assert_size; + +#[inline(never)] +#[cold] +pub fn cold_path<F: FnOnce() -> R, R>(f: F) -> R { + f() +} + +pub mod base_n; +pub mod binary_search_util; +pub mod captures; +pub mod flock; +pub mod functor; +pub mod fx; +pub mod graph; +pub mod intern; +pub mod jobserver; +pub mod macros; +pub mod map_in_place; +pub mod obligation_forest; +pub mod owning_ref; +pub mod sip128; +pub mod small_c_str; +pub mod small_str; +pub mod snapshot_map; +pub mod svh; +pub use ena::snapshot_vec; +pub mod memmap; +pub mod sorted_map; +#[macro_use] +pub mod stable_hasher; +mod atomic_ref; +pub mod fingerprint; +pub mod profiling; +pub mod sharded; +pub mod stack; +pub mod sync; +pub mod thin_vec; +pub mod tiny_list; +pub mod transitive_relation; +pub mod vec_linked_list; +pub mod vec_map; +pub mod work_queue; +pub use atomic_ref::AtomicRef; +pub mod frozen; +pub mod sso; +pub mod steal; +pub mod tagged_ptr; +pub mod temp_dir; +pub mod unhash; + +pub use ena::undo_log; +pub use ena::unify; + +pub struct OnDrop<F: Fn()>(pub F); + +impl<F: Fn()> OnDrop<F> { + /// Forgets the function which prevents it from running. + /// Ensure that the function owns no memory, otherwise it will be leaked. + #[inline] + pub fn disable(self) { + std::mem::forget(self); + } +} + +impl<F: Fn()> Drop for OnDrop<F> { + #[inline] + fn drop(&mut self) { + (self.0)(); + } +} + +// See comments in src/librustc_middle/lib.rs +#[doc(hidden)] +pub fn __noop_fix_for_27438() {} diff --git a/compiler/rustc_data_structures/src/macros.rs b/compiler/rustc_data_structures/src/macros.rs new file mode 100644 index 000000000..e05491f6f --- /dev/null +++ b/compiler/rustc_data_structures/src/macros.rs @@ -0,0 +1,37 @@ +#[macro_export] +macro_rules! enum_from_u32 { + ($(#[$attr:meta])* pub enum $name:ident { + $($(#[$var_attr:meta])* $variant:ident = $e:expr,)* + }) => { + $(#[$attr])* + pub enum $name { + $($(#[$var_attr])* $variant = $e),* + } + + impl $name { + pub fn from_u32(u: u32) -> Option<$name> { + $(if u == $name::$variant as u32 { + return Some($name::$variant) + })* + None + } + } + }; + ($(#[$attr:meta])* pub enum $name:ident { + $($(#[$var_attr:meta])* $variant:ident,)* + }) => { + $(#[$attr])* + pub enum $name { + $($(#[$var_attr])* $variant,)* + } + + impl $name { + pub fn from_u32(u: u32) -> Option<$name> { + $(if u == $name::$variant as u32 { + return Some($name::$variant) + })* + None + } + } + } +} diff --git a/compiler/rustc_data_structures/src/map_in_place.rs b/compiler/rustc_data_structures/src/map_in_place.rs new file mode 100644 index 000000000..874de03d3 --- /dev/null +++ b/compiler/rustc_data_structures/src/map_in_place.rs @@ -0,0 +1,108 @@ +use smallvec::{Array, SmallVec}; +use std::ptr; + +pub trait MapInPlace<T>: Sized { + fn map_in_place<F>(&mut self, mut f: F) + where + F: FnMut(T) -> T, + { + self.flat_map_in_place(|e| Some(f(e))) + } + + fn flat_map_in_place<F, I>(&mut self, f: F) + where + F: FnMut(T) -> I, + I: IntoIterator<Item = T>; +} + +impl<T> MapInPlace<T> for Vec<T> { + fn flat_map_in_place<F, I>(&mut self, mut f: F) + where + F: FnMut(T) -> I, + I: IntoIterator<Item = T>, + { + let mut read_i = 0; + let mut write_i = 0; + unsafe { + let mut old_len = self.len(); + self.set_len(0); // make sure we just leak elements in case of panic + + while read_i < old_len { + // move the read_i'th item out of the vector and map it + // to an iterator + let e = ptr::read(self.as_ptr().add(read_i)); + let iter = f(e).into_iter(); + read_i += 1; + + for e in iter { + if write_i < read_i { + ptr::write(self.as_mut_ptr().add(write_i), e); + write_i += 1; + } else { + // If this is reached we ran out of space + // in the middle of the vector. + // However, the vector is in a valid state here, + // so we just do a somewhat inefficient insert. + self.set_len(old_len); + self.insert(write_i, e); + + old_len = self.len(); + self.set_len(0); + + read_i += 1; + write_i += 1; + } + } + } + + // write_i tracks the number of actually written new items. + self.set_len(write_i); + } + } +} + +impl<T, A: Array<Item = T>> MapInPlace<T> for SmallVec<A> { + fn flat_map_in_place<F, I>(&mut self, mut f: F) + where + F: FnMut(T) -> I, + I: IntoIterator<Item = T>, + { + let mut read_i = 0; + let mut write_i = 0; + unsafe { + let mut old_len = self.len(); + self.set_len(0); // make sure we just leak elements in case of panic + + while read_i < old_len { + // move the read_i'th item out of the vector and map it + // to an iterator + let e = ptr::read(self.as_ptr().add(read_i)); + let iter = f(e).into_iter(); + read_i += 1; + + for e in iter { + if write_i < read_i { + ptr::write(self.as_mut_ptr().add(write_i), e); + write_i += 1; + } else { + // If this is reached we ran out of space + // in the middle of the vector. + // However, the vector is in a valid state here, + // so we just do a somewhat inefficient insert. + self.set_len(old_len); + self.insert(write_i, e); + + old_len = self.len(); + self.set_len(0); + + read_i += 1; + write_i += 1; + } + } + } + + // write_i tracks the number of actually written new items. + self.set_len(write_i); + } + } +} diff --git a/compiler/rustc_data_structures/src/memmap.rs b/compiler/rustc_data_structures/src/memmap.rs new file mode 100644 index 000000000..917416df6 --- /dev/null +++ b/compiler/rustc_data_structures/src/memmap.rs @@ -0,0 +1,108 @@ +use std::fs::File; +use std::io; +use std::ops::{Deref, DerefMut}; + +use crate::owning_ref::StableAddress; + +/// A trivial wrapper for [`memmap2::Mmap`] that implements [`StableAddress`]. +#[cfg(not(target_arch = "wasm32"))] +pub struct Mmap(memmap2::Mmap); + +#[cfg(target_arch = "wasm32")] +pub struct Mmap(Vec<u8>); + +#[cfg(not(target_arch = "wasm32"))] +impl Mmap { + #[inline] + pub unsafe fn map(file: File) -> io::Result<Self> { + memmap2::Mmap::map(&file).map(Mmap) + } +} + +#[cfg(target_arch = "wasm32")] +impl Mmap { + #[inline] + pub unsafe fn map(mut file: File) -> io::Result<Self> { + use std::io::Read; + + let mut data = Vec::new(); + file.read_to_end(&mut data)?; + Ok(Mmap(data)) + } +} + +impl Deref for Mmap { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &[u8] { + &*self.0 + } +} + +// SAFETY: On architectures other than WASM, mmap is used as backing storage. The address of this +// memory map is stable. On WASM, `Vec<u8>` is used as backing storage. The `Mmap` type doesn't +// export any function that can cause the `Vec` to be re-allocated. As such the address of the +// bytes inside this `Vec` is stable. +unsafe impl StableAddress for Mmap {} + +#[cfg(not(target_arch = "wasm32"))] +pub struct MmapMut(memmap2::MmapMut); + +#[cfg(target_arch = "wasm32")] +pub struct MmapMut(Vec<u8>); + +#[cfg(not(target_arch = "wasm32"))] +impl MmapMut { + #[inline] + pub fn map_anon(len: usize) -> io::Result<Self> { + let mmap = memmap2::MmapMut::map_anon(len)?; + Ok(MmapMut(mmap)) + } + + #[inline] + pub fn flush(&mut self) -> io::Result<()> { + self.0.flush() + } + + #[inline] + pub fn make_read_only(self) -> std::io::Result<Mmap> { + let mmap = self.0.make_read_only()?; + Ok(Mmap(mmap)) + } +} + +#[cfg(target_arch = "wasm32")] +impl MmapMut { + #[inline] + pub fn map_anon(len: usize) -> io::Result<Self> { + let data = Vec::with_capacity(len); + Ok(MmapMut(data)) + } + + #[inline] + pub fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + + #[inline] + pub fn make_read_only(self) -> std::io::Result<Mmap> { + Ok(Mmap(self.0)) + } +} + +impl Deref for MmapMut { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &[u8] { + &*self.0 + } +} + +impl DerefMut for MmapMut { + #[inline] + fn deref_mut(&mut self) -> &mut [u8] { + &mut *self.0 + } +} diff --git a/compiler/rustc_data_structures/src/obligation_forest/graphviz.rs b/compiler/rustc_data_structures/src/obligation_forest/graphviz.rs new file mode 100644 index 000000000..3a268e4b4 --- /dev/null +++ b/compiler/rustc_data_structures/src/obligation_forest/graphviz.rs @@ -0,0 +1,90 @@ +use crate::obligation_forest::{ForestObligation, ObligationForest}; +use rustc_graphviz as dot; +use std::env::var_os; +use std::fs::File; +use std::io::BufWriter; +use std::path::Path; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +impl<O: ForestObligation> ObligationForest<O> { + /// Creates a graphviz representation of the obligation forest. Given a directory this will + /// create files with name of the format `<counter>_<description>.gv`. The counter is + /// global and is maintained internally. + /// + /// Calling this will do nothing unless the environment variable + /// `DUMP_OBLIGATION_FOREST_GRAPHVIZ` is defined. + /// + /// A few post-processing that you might want to do make the forest easier to visualize: + /// + /// * `sed 's,std::[a-z]*::,,g'` — Deletes the `std::<package>::` prefix of paths. + /// * `sed 's,"Binder(TraitPredicate(<\(.*\)>)) (\([^)]*\))","\1 (\2)",'` — Transforms + /// `Binder(TraitPredicate(<predicate>))` into just `<predicate>`. + #[allow(dead_code)] + pub fn dump_graphviz<P: AsRef<Path>>(&self, dir: P, description: &str) { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + if var_os("DUMP_OBLIGATION_FOREST_GRAPHVIZ").is_none() { + return; + } + + let counter = COUNTER.fetch_add(1, Ordering::AcqRel); + + let file_path = dir.as_ref().join(format!("{:010}_{}.gv", counter, description)); + + let mut gv_file = BufWriter::new(File::create(file_path).unwrap()); + + dot::render(&self, &mut gv_file).unwrap(); + } +} + +impl<'a, O: ForestObligation + 'a> dot::Labeller<'a> for &'a ObligationForest<O> { + type Node = usize; + type Edge = (usize, usize); + + fn graph_id(&self) -> dot::Id<'_> { + dot::Id::new("trait_obligation_forest").unwrap() + } + + fn node_id(&self, index: &Self::Node) -> dot::Id<'_> { + dot::Id::new(format!("obligation_{}", index)).unwrap() + } + + fn node_label(&self, index: &Self::Node) -> dot::LabelText<'_> { + let node = &self.nodes[*index]; + let label = format!("{:?} ({:?})", node.obligation.as_cache_key(), node.state.get()); + + dot::LabelText::LabelStr(label.into()) + } + + fn edge_label(&self, (_index_source, _index_target): &Self::Edge) -> dot::LabelText<'_> { + dot::LabelText::LabelStr("".into()) + } +} + +impl<'a, O: ForestObligation + 'a> dot::GraphWalk<'a> for &'a ObligationForest<O> { + type Node = usize; + type Edge = (usize, usize); + + fn nodes(&self) -> dot::Nodes<'_, Self::Node> { + (0..self.nodes.len()).collect() + } + + fn edges(&self) -> dot::Edges<'_, Self::Edge> { + (0..self.nodes.len()) + .flat_map(|i| { + let node = &self.nodes[i]; + + node.dependents.iter().map(move |&d| (d, i)) + }) + .collect() + } + + fn source(&self, (s, _): &Self::Edge) -> Self::Node { + *s + } + + fn target(&self, (_, t): &Self::Edge) -> Self::Node { + *t + } +} diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs new file mode 100644 index 000000000..07a96dd7d --- /dev/null +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -0,0 +1,698 @@ +//! The `ObligationForest` is a utility data structure used in trait +//! matching to track the set of outstanding obligations (those not yet +//! resolved to success or error). It also tracks the "backtrace" of each +//! pending obligation (why we are trying to figure this out in the first +//! place). +//! +//! ### External view +//! +//! `ObligationForest` supports two main public operations (there are a +//! few others not discussed here): +//! +//! 1. Add a new root obligations (`register_obligation`). +//! 2. Process the pending obligations (`process_obligations`). +//! +//! When a new obligation `N` is added, it becomes the root of an +//! obligation tree. This tree can also carry some per-tree state `T`, +//! which is given at the same time. This tree is a singleton to start, so +//! `N` is both the root and the only leaf. Each time the +//! `process_obligations` method is called, it will invoke its callback +//! with every pending obligation (so that will include `N`, the first +//! time). The callback also receives a (mutable) reference to the +//! per-tree state `T`. The callback should process the obligation `O` +//! that it is given and return a `ProcessResult`: +//! +//! - `Unchanged` -> ambiguous result. Obligation was neither a success +//! nor a failure. It is assumed that further attempts to process the +//! obligation will yield the same result unless something in the +//! surrounding environment changes. +//! - `Changed(C)` - the obligation was *shallowly successful*. The +//! vector `C` is a list of subobligations. The meaning of this is that +//! `O` was successful on the assumption that all the obligations in `C` +//! are also successful. Therefore, `O` is only considered a "true" +//! success if `C` is empty. Otherwise, `O` is put into a suspended +//! state and the obligations in `C` become the new pending +//! obligations. They will be processed the next time you call +//! `process_obligations`. +//! - `Error(E)` -> obligation failed with error `E`. We will collect this +//! error and return it from `process_obligations`, along with the +//! "backtrace" of obligations (that is, the list of obligations up to +//! and including the root of the failed obligation). No further +//! obligations from that same tree will be processed, since the tree is +//! now considered to be in error. +//! +//! When the call to `process_obligations` completes, you get back an `Outcome`, +//! which includes two bits of information: +//! +//! - `completed`: a list of obligations where processing was fully +//! completed without error (meaning that all transitive subobligations +//! have also been completed). So, for example, if the callback from +//! `process_obligations` returns `Changed(C)` for some obligation `O`, +//! then `O` will be considered completed right away if `C` is the +//! empty vector. Otherwise it will only be considered completed once +//! all the obligations in `C` have been found completed. +//! - `errors`: a list of errors that occurred and associated backtraces +//! at the time of error, which can be used to give context to the user. +//! +//! Upon completion, none of the existing obligations were *shallowly +//! successful* (that is, no callback returned `Changed(_)`). This implies that +//! all obligations were either errors or returned an ambiguous result. +//! +//! ### Implementation details +//! +//! For the most part, comments specific to the implementation are in the +//! code. This file only contains a very high-level overview. Basically, +//! the forest is stored in a vector. Each element of the vector is a node +//! in some tree. Each node in the vector has the index of its dependents, +//! including the first dependent which is known as the parent. It also +//! has a current state, described by `NodeState`. After each processing +//! step, we compress the vector to remove completed and error nodes, which +//! aren't needed anymore. + +use crate::fx::{FxHashMap, FxHashSet}; + +use std::cell::Cell; +use std::collections::hash_map::Entry; +use std::fmt::Debug; +use std::hash; +use std::marker::PhantomData; + +mod graphviz; + +#[cfg(test)] +mod tests; + +pub trait ForestObligation: Clone + Debug { + type CacheKey: Clone + hash::Hash + Eq + Debug; + + /// Converts this `ForestObligation` suitable for use as a cache key. + /// If two distinct `ForestObligations`s return the same cache key, + /// then it must be sound to use the result of processing one obligation + /// (e.g. success for error) for the other obligation + fn as_cache_key(&self) -> Self::CacheKey; +} + +pub trait ObligationProcessor { + type Obligation: ForestObligation; + type Error: Debug; + + fn needs_process_obligation(&self, obligation: &Self::Obligation) -> bool; + + fn process_obligation( + &mut self, + obligation: &mut Self::Obligation, + ) -> ProcessResult<Self::Obligation, Self::Error>; + + /// As we do the cycle check, we invoke this callback when we + /// encounter an actual cycle. `cycle` is an iterator that starts + /// at the start of the cycle in the stack and walks **toward the + /// top**. + /// + /// In other words, if we had O1 which required O2 which required + /// O3 which required O1, we would give an iterator yielding O1, + /// O2, O3 (O1 is not yielded twice). + fn process_backedge<'c, I>(&mut self, cycle: I, _marker: PhantomData<&'c Self::Obligation>) + where + I: Clone + Iterator<Item = &'c Self::Obligation>; +} + +/// The result type used by `process_obligation`. +#[derive(Debug)] +pub enum ProcessResult<O, E> { + Unchanged, + Changed(Vec<O>), + Error(E), +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +struct ObligationTreeId(usize); + +type ObligationTreeIdGenerator = + std::iter::Map<std::ops::RangeFrom<usize>, fn(usize) -> ObligationTreeId>; + +pub struct ObligationForest<O: ForestObligation> { + /// The list of obligations. In between calls to [Self::process_obligations], + /// this list only contains nodes in the `Pending` or `Waiting` state. + /// + /// `usize` indices are used here and throughout this module, rather than + /// [`rustc_index::newtype_index!`] indices, because this code is hot enough + /// that the `u32`-to-`usize` conversions that would be required are + /// significant, and space considerations are not important. + nodes: Vec<Node<O>>, + + /// A cache of predicates that have been successfully completed. + done_cache: FxHashSet<O::CacheKey>, + + /// A cache of the nodes in `nodes`, indexed by predicate. Unfortunately, + /// its contents are not guaranteed to match those of `nodes`. See the + /// comments in `Self::process_obligation` for details. + active_cache: FxHashMap<O::CacheKey, usize>, + + /// A vector reused in [Self::compress()] and [Self::find_cycles_from_node()], + /// to avoid allocating new vectors. + reused_node_vec: Vec<usize>, + + obligation_tree_id_generator: ObligationTreeIdGenerator, + + /// Per tree error cache. This is used to deduplicate errors, + /// which is necessary to avoid trait resolution overflow in + /// some cases. + /// + /// See [this][details] for details. + /// + /// [details]: https://github.com/rust-lang/rust/pull/53255#issuecomment-421184780 + error_cache: FxHashMap<ObligationTreeId, FxHashSet<O::CacheKey>>, +} + +#[derive(Debug)] +struct Node<O> { + obligation: O, + state: Cell<NodeState>, + + /// Obligations that depend on this obligation for their completion. They + /// must all be in a non-pending state. + dependents: Vec<usize>, + + /// If true, `dependents[0]` points to a "parent" node, which requires + /// special treatment upon error but is otherwise treated the same. + /// (It would be more idiomatic to store the parent node in a separate + /// `Option<usize>` field, but that slows down the common case of + /// iterating over the parent and other descendants together.) + has_parent: bool, + + /// Identifier of the obligation tree to which this node belongs. + obligation_tree_id: ObligationTreeId, +} + +impl<O> Node<O> { + fn new(parent: Option<usize>, obligation: O, obligation_tree_id: ObligationTreeId) -> Node<O> { + Node { + obligation, + state: Cell::new(NodeState::Pending), + dependents: if let Some(parent_index) = parent { vec![parent_index] } else { vec![] }, + has_parent: parent.is_some(), + obligation_tree_id, + } + } +} + +/// The state of one node in some tree within the forest. This represents the +/// current state of processing for the obligation (of type `O`) associated +/// with this node. +/// +/// The non-`Error` state transitions are as follows. +/// ```text +/// (Pre-creation) +/// | +/// | register_obligation_at() (called by process_obligations() and +/// v from outside the crate) +/// Pending +/// | +/// | process_obligations() +/// v +/// Success +/// | ^ +/// | | mark_successes() +/// | v +/// | Waiting +/// | +/// | process_cycles() +/// v +/// Done +/// | +/// | compress() +/// v +/// (Removed) +/// ``` +/// The `Error` state can be introduced in several places, via `error_at()`. +/// +/// Outside of `ObligationForest` methods, nodes should be either `Pending` or +/// `Waiting`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum NodeState { + /// This obligation has not yet been selected successfully. Cannot have + /// subobligations. + Pending, + + /// This obligation was selected successfully, but may or may not have + /// subobligations. + Success, + + /// This obligation was selected successfully, but it has a pending + /// subobligation. + Waiting, + + /// This obligation, along with its subobligations, are complete, and will + /// be removed in the next collection. + Done, + + /// This obligation was resolved to an error. It will be removed by the + /// next compression step. + Error, +} + +/// This trait allows us to have two different Outcome types: +/// - the normal one that does as little as possible +/// - one for tests that does some additional work and checking +pub trait OutcomeTrait { + type Error; + type Obligation; + + fn new() -> Self; + fn record_completed(&mut self, outcome: &Self::Obligation); + fn record_error(&mut self, error: Self::Error); +} + +#[derive(Debug)] +pub struct Outcome<O, E> { + /// Backtrace of obligations that were found to be in error. + pub errors: Vec<Error<O, E>>, +} + +impl<O, E> OutcomeTrait for Outcome<O, E> { + type Error = Error<O, E>; + type Obligation = O; + + fn new() -> Self { + Self { errors: vec![] } + } + + fn record_completed(&mut self, _outcome: &Self::Obligation) { + // do nothing + } + + fn record_error(&mut self, error: Self::Error) { + self.errors.push(error) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Error<O, E> { + pub error: E, + pub backtrace: Vec<O>, +} + +impl<O: ForestObligation> ObligationForest<O> { + pub fn new() -> ObligationForest<O> { + ObligationForest { + nodes: vec![], + done_cache: Default::default(), + active_cache: Default::default(), + reused_node_vec: vec![], + obligation_tree_id_generator: (0..).map(ObligationTreeId), + error_cache: Default::default(), + } + } + + /// Returns the total number of nodes in the forest that have not + /// yet been fully resolved. + pub fn len(&self) -> usize { + self.nodes.len() + } + + /// Registers an obligation. + pub fn register_obligation(&mut self, obligation: O) { + // Ignore errors here - there is no guarantee of success. + let _ = self.register_obligation_at(obligation, None); + } + + // Returns Err(()) if we already know this obligation failed. + fn register_obligation_at(&mut self, obligation: O, parent: Option<usize>) -> Result<(), ()> { + let cache_key = obligation.as_cache_key(); + if self.done_cache.contains(&cache_key) { + debug!("register_obligation_at: ignoring already done obligation: {:?}", obligation); + return Ok(()); + } + + match self.active_cache.entry(cache_key) { + Entry::Occupied(o) => { + let node = &mut self.nodes[*o.get()]; + if let Some(parent_index) = parent { + // If the node is already in `active_cache`, it has already + // had its chance to be marked with a parent. So if it's + // not already present, just dump `parent` into the + // dependents as a non-parent. + if !node.dependents.contains(&parent_index) { + node.dependents.push(parent_index); + } + } + if let NodeState::Error = node.state.get() { Err(()) } else { Ok(()) } + } + Entry::Vacant(v) => { + let obligation_tree_id = match parent { + Some(parent_index) => self.nodes[parent_index].obligation_tree_id, + None => self.obligation_tree_id_generator.next().unwrap(), + }; + + let already_failed = parent.is_some() + && self + .error_cache + .get(&obligation_tree_id) + .map_or(false, |errors| errors.contains(v.key())); + + if already_failed { + Err(()) + } else { + let new_index = self.nodes.len(); + v.insert(new_index); + self.nodes.push(Node::new(parent, obligation, obligation_tree_id)); + Ok(()) + } + } + } + } + + /// Converts all remaining obligations to the given error. + pub fn to_errors<E: Clone>(&mut self, error: E) -> Vec<Error<O, E>> { + let errors = self + .nodes + .iter() + .enumerate() + .filter(|(_index, node)| node.state.get() == NodeState::Pending) + .map(|(index, _node)| Error { error: error.clone(), backtrace: self.error_at(index) }) + .collect(); + + self.compress(|_| assert!(false)); + errors + } + + /// Returns the set of obligations that are in a pending state. + pub fn map_pending_obligations<P, F>(&self, f: F) -> Vec<P> + where + F: Fn(&O) -> P, + { + self.nodes + .iter() + .filter(|node| node.state.get() == NodeState::Pending) + .map(|node| f(&node.obligation)) + .collect() + } + + fn insert_into_error_cache(&mut self, index: usize) { + let node = &self.nodes[index]; + self.error_cache + .entry(node.obligation_tree_id) + .or_default() + .insert(node.obligation.as_cache_key()); + } + + /// Performs a fixpoint computation over the obligation list. + #[inline(never)] + pub fn process_obligations<P, OUT>(&mut self, processor: &mut P) -> OUT + where + P: ObligationProcessor<Obligation = O>, + OUT: OutcomeTrait<Obligation = O, Error = Error<O, P::Error>>, + { + let mut outcome = OUT::new(); + + // Fixpoint computation: we repeat until the inner loop stalls. + loop { + let mut has_changed = false; + + // Note that the loop body can append new nodes, and those new nodes + // will then be processed by subsequent iterations of the loop. + // + // We can't use an iterator for the loop because `self.nodes` is + // appended to and the borrow checker would complain. We also can't use + // `for index in 0..self.nodes.len() { ... }` because the range would + // be computed with the initial length, and we would miss the appended + // nodes. Therefore we use a `while` loop. + let mut index = 0; + while let Some(node) = self.nodes.get_mut(index) { + if node.state.get() != NodeState::Pending + || !processor.needs_process_obligation(&node.obligation) + { + index += 1; + continue; + } + + // `processor.process_obligation` can modify the predicate within + // `node.obligation`, and that predicate is the key used for + // `self.active_cache`. This means that `self.active_cache` can get + // out of sync with `nodes`. It's not very common, but it does + // happen, and code in `compress` has to allow for it. + + match processor.process_obligation(&mut node.obligation) { + ProcessResult::Unchanged => { + // No change in state. + } + ProcessResult::Changed(children) => { + // We are not (yet) stalled. + has_changed = true; + node.state.set(NodeState::Success); + + for child in children { + let st = self.register_obligation_at(child, Some(index)); + if let Err(()) = st { + // Error already reported - propagate it + // to our node. + self.error_at(index); + } + } + } + ProcessResult::Error(err) => { + has_changed = true; + outcome.record_error(Error { error: err, backtrace: self.error_at(index) }); + } + } + index += 1; + } + + // If unchanged, then we saw no successful obligations, which means + // there is no point in further iteration. This is based on the + // assumption that when trait matching returns `Error` or + // `Unchanged`, those results do not affect environmental inference + // state. (Note that this will occur if we invoke + // `process_obligations` with no pending obligations.) + if !has_changed { + break; + } + + self.mark_successes(); + self.process_cycles(processor); + self.compress(|obl| outcome.record_completed(obl)); + } + + outcome + } + + /// Returns a vector of obligations for `p` and all of its + /// ancestors, putting them into the error state in the process. + fn error_at(&self, mut index: usize) -> Vec<O> { + let mut error_stack: Vec<usize> = vec![]; + let mut trace = vec![]; + + loop { + let node = &self.nodes[index]; + node.state.set(NodeState::Error); + trace.push(node.obligation.clone()); + if node.has_parent { + // The first dependent is the parent, which is treated + // specially. + error_stack.extend(node.dependents.iter().skip(1)); + index = node.dependents[0]; + } else { + // No parent; treat all dependents non-specially. + error_stack.extend(node.dependents.iter()); + break; + } + } + + while let Some(index) = error_stack.pop() { + let node = &self.nodes[index]; + if node.state.get() != NodeState::Error { + node.state.set(NodeState::Error); + error_stack.extend(node.dependents.iter()); + } + } + + trace + } + + /// Mark all `Waiting` nodes as `Success`, except those that depend on a + /// pending node. + fn mark_successes(&self) { + // Convert all `Waiting` nodes to `Success`. + for node in &self.nodes { + if node.state.get() == NodeState::Waiting { + node.state.set(NodeState::Success); + } + } + + // Convert `Success` nodes that depend on a pending node back to + // `Waiting`. + for node in &self.nodes { + if node.state.get() == NodeState::Pending { + // This call site is hot. + self.inlined_mark_dependents_as_waiting(node); + } + } + } + + // This always-inlined function is for the hot call site. + #[inline(always)] + fn inlined_mark_dependents_as_waiting(&self, node: &Node<O>) { + for &index in node.dependents.iter() { + let node = &self.nodes[index]; + let state = node.state.get(); + if state == NodeState::Success { + // This call site is cold. + self.uninlined_mark_dependents_as_waiting(node); + } else { + debug_assert!(state == NodeState::Waiting || state == NodeState::Error) + } + } + } + + // This never-inlined function is for the cold call site. + #[inline(never)] + fn uninlined_mark_dependents_as_waiting(&self, node: &Node<O>) { + // Mark node Waiting in the cold uninlined code instead of the hot inlined + node.state.set(NodeState::Waiting); + self.inlined_mark_dependents_as_waiting(node) + } + + /// Report cycles between all `Success` nodes, and convert all `Success` + /// nodes to `Done`. This must be called after `mark_successes`. + fn process_cycles<P>(&mut self, processor: &mut P) + where + P: ObligationProcessor<Obligation = O>, + { + let mut stack = std::mem::take(&mut self.reused_node_vec); + for (index, node) in self.nodes.iter().enumerate() { + // For some benchmarks this state test is extremely hot. It's a win + // to handle the no-op cases immediately to avoid the cost of the + // function call. + if node.state.get() == NodeState::Success { + self.find_cycles_from_node(&mut stack, processor, index); + } + } + + debug_assert!(stack.is_empty()); + self.reused_node_vec = stack; + } + + fn find_cycles_from_node<P>(&self, stack: &mut Vec<usize>, processor: &mut P, index: usize) + where + P: ObligationProcessor<Obligation = O>, + { + let node = &self.nodes[index]; + if node.state.get() == NodeState::Success { + match stack.iter().rposition(|&n| n == index) { + None => { + stack.push(index); + for &dep_index in node.dependents.iter() { + self.find_cycles_from_node(stack, processor, dep_index); + } + stack.pop(); + node.state.set(NodeState::Done); + } + Some(rpos) => { + // Cycle detected. + processor.process_backedge( + stack[rpos..].iter().map(|&i| &self.nodes[i].obligation), + PhantomData, + ); + } + } + } + } + + /// Compresses the vector, removing all popped nodes. This adjusts the + /// indices and hence invalidates any outstanding indices. `process_cycles` + /// must be run beforehand to remove any cycles on `Success` nodes. + #[inline(never)] + fn compress(&mut self, mut outcome_cb: impl FnMut(&O)) { + let orig_nodes_len = self.nodes.len(); + let mut node_rewrites: Vec<_> = std::mem::take(&mut self.reused_node_vec); + debug_assert!(node_rewrites.is_empty()); + node_rewrites.extend(0..orig_nodes_len); + let mut dead_nodes = 0; + + // Move removable nodes to the end, preserving the order of the + // remaining nodes. + // + // LOOP INVARIANT: + // self.nodes[0..index - dead_nodes] are the first remaining nodes + // self.nodes[index - dead_nodes..index] are all dead + // self.nodes[index..] are unchanged + for index in 0..orig_nodes_len { + let node = &self.nodes[index]; + match node.state.get() { + NodeState::Pending | NodeState::Waiting => { + if dead_nodes > 0 { + self.nodes.swap(index, index - dead_nodes); + node_rewrites[index] -= dead_nodes; + } + } + NodeState::Done => { + // The removal lookup might fail because the contents of + // `self.active_cache` are not guaranteed to match those of + // `self.nodes`. See the comment in `process_obligation` + // for more details. + let cache_key = node.obligation.as_cache_key(); + self.active_cache.remove(&cache_key); + self.done_cache.insert(cache_key); + + // Extract the success stories. + outcome_cb(&node.obligation); + node_rewrites[index] = orig_nodes_len; + dead_nodes += 1; + } + NodeState::Error => { + // We *intentionally* remove the node from the cache at this point. Otherwise + // tests must come up with a different type on every type error they + // check against. + self.active_cache.remove(&node.obligation.as_cache_key()); + self.insert_into_error_cache(index); + node_rewrites[index] = orig_nodes_len; + dead_nodes += 1; + } + NodeState::Success => unreachable!(), + } + } + + if dead_nodes > 0 { + // Remove the dead nodes and rewrite indices. + self.nodes.truncate(orig_nodes_len - dead_nodes); + self.apply_rewrites(&node_rewrites); + } + + node_rewrites.truncate(0); + self.reused_node_vec = node_rewrites; + } + + #[inline(never)] + fn apply_rewrites(&mut self, node_rewrites: &[usize]) { + let orig_nodes_len = node_rewrites.len(); + + for node in &mut self.nodes { + let mut i = 0; + while let Some(dependent) = node.dependents.get_mut(i) { + let new_index = node_rewrites[*dependent]; + if new_index >= orig_nodes_len { + node.dependents.swap_remove(i); + if i == 0 && node.has_parent { + // We just removed the parent. + node.has_parent = false; + } + } else { + *dependent = new_index; + i += 1; + } + } + } + + // This updating of `self.active_cache` is necessary because the + // removal of nodes within `compress` can fail. See above. + self.active_cache.retain(|_predicate, index| { + let new_index = node_rewrites[*index]; + if new_index >= orig_nodes_len { + false + } else { + *index = new_index; + true + } + }); + } +} diff --git a/compiler/rustc_data_structures/src/obligation_forest/tests.rs b/compiler/rustc_data_structures/src/obligation_forest/tests.rs new file mode 100644 index 000000000..e2991aae1 --- /dev/null +++ b/compiler/rustc_data_structures/src/obligation_forest/tests.rs @@ -0,0 +1,479 @@ +use super::*; + +use std::fmt; +use std::marker::PhantomData; + +impl<'a> super::ForestObligation for &'a str { + type CacheKey = &'a str; + + fn as_cache_key(&self) -> Self::CacheKey { + self + } +} + +struct ClosureObligationProcessor<OF, BF, O, E> { + process_obligation: OF, + _process_backedge: BF, + marker: PhantomData<(O, E)>, +} + +struct TestOutcome<O, E> { + pub completed: Vec<O>, + pub errors: Vec<Error<O, E>>, +} + +impl<O, E> OutcomeTrait for TestOutcome<O, E> +where + O: Clone, +{ + type Error = Error<O, E>; + type Obligation = O; + + fn new() -> Self { + Self { errors: vec![], completed: vec![] } + } + + fn record_completed(&mut self, outcome: &Self::Obligation) { + self.completed.push(outcome.clone()) + } + + fn record_error(&mut self, error: Self::Error) { + self.errors.push(error) + } +} + +#[allow(non_snake_case)] +fn C<OF, BF, O>(of: OF, bf: BF) -> ClosureObligationProcessor<OF, BF, O, &'static str> +where + OF: FnMut(&mut O) -> ProcessResult<O, &'static str>, + BF: FnMut(&[O]), +{ + ClosureObligationProcessor { + process_obligation: of, + _process_backedge: bf, + marker: PhantomData, + } +} + +impl<OF, BF, O, E> ObligationProcessor for ClosureObligationProcessor<OF, BF, O, E> +where + O: super::ForestObligation + fmt::Debug, + E: fmt::Debug, + OF: FnMut(&mut O) -> ProcessResult<O, E>, + BF: FnMut(&[O]), +{ + type Obligation = O; + type Error = E; + + fn needs_process_obligation(&self, _obligation: &Self::Obligation) -> bool { + true + } + + fn process_obligation( + &mut self, + obligation: &mut Self::Obligation, + ) -> ProcessResult<Self::Obligation, Self::Error> { + (self.process_obligation)(obligation) + } + + fn process_backedge<'c, I>(&mut self, _cycle: I, _marker: PhantomData<&'c Self::Obligation>) + where + I: Clone + Iterator<Item = &'c Self::Obligation>, + { + } +} + +#[test] +fn push_pop() { + let mut forest = ObligationForest::new(); + forest.register_obligation("A"); + forest.register_obligation("B"); + forest.register_obligation("C"); + + // first round, B errors out, A has subtasks, and C completes, creating this: + // A |-> A.1 + // |-> A.2 + // |-> A.3 + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]), + "B" => ProcessResult::Error("B is for broken"), + "C" => ProcessResult::Changed(vec![]), + "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok, vec!["C"]); + assert_eq!(err, vec![Error { error: "B is for broken", backtrace: vec!["B"] }]); + + // second round: two delays, one success, creating an uneven set of subtasks: + // A |-> A.1 + // |-> A.2 + // |-> A.3 |-> A.3.i + // D |-> D.1 + // |-> D.2 + forest.register_obligation("D"); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.1" => ProcessResult::Unchanged, + "A.2" => ProcessResult::Unchanged, + "A.3" => ProcessResult::Changed(vec!["A.3.i"]), + "D" => ProcessResult::Changed(vec!["D.1", "D.2"]), + "A.3.i" | "D.1" | "D.2" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok, Vec::<&'static str>::new()); + assert_eq!(err, Vec::new()); + + // third round: ok in A.1 but trigger an error in A.2. Check that it + // propagates to A, but not D.1 or D.2. + // D |-> D.1 |-> D.1.i + // |-> D.2 |-> D.2.i + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.1" => ProcessResult::Changed(vec![]), + "A.2" => ProcessResult::Error("A is for apple"), + "A.3.i" => ProcessResult::Changed(vec![]), + "D.1" => ProcessResult::Changed(vec!["D.1.i"]), + "D.2" => ProcessResult::Changed(vec!["D.2.i"]), + "D.1.i" | "D.2.i" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + let mut ok = ok; + ok.sort(); + assert_eq!(ok, vec!["A.1", "A.3", "A.3.i"]); + assert_eq!(err, vec![Error { error: "A is for apple", backtrace: vec!["A.2", "A"] }]); + + // fourth round: error in D.1.i + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D.1.i" => ProcessResult::Error("D is for dumb"), + "D.2.i" => ProcessResult::Changed(vec![]), + _ => panic!("unexpected obligation {:?}", obligation), + }, + |_| {}, + )); + let mut ok = ok; + ok.sort(); + assert_eq!(ok, vec!["D.2", "D.2.i"]); + assert_eq!(err, vec![Error { error: "D is for dumb", backtrace: vec!["D.1.i", "D.1", "D"] }]); +} + +// Test that if a tree with grandchildren succeeds, everything is +// reported as expected: +// A +// A.1 +// A.2 +// A.2.i +// A.2.ii +// A.3 +#[test] +fn success_in_grandchildren() { + let mut forest = ObligationForest::new(); + forest.register_obligation("A"); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]), + "A.1" => ProcessResult::Changed(vec![]), + "A.2" => ProcessResult::Changed(vec!["A.2.i", "A.2.ii"]), + "A.3" => ProcessResult::Changed(vec![]), + "A.2.i" | "A.2.ii" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + let mut ok = ok; + ok.sort(); + assert_eq!(ok, vec!["A.1", "A.3"]); + assert!(err.is_empty()); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.2.i" => ProcessResult::Unchanged, + "A.2.ii" => ProcessResult::Changed(vec![]), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok, vec!["A.2.ii"]); + assert!(err.is_empty()); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.2.i" => ProcessResult::Changed(vec!["A.2.i.a"]), + "A.2.i.a" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert!(ok.is_empty()); + assert!(err.is_empty()); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.2.i.a" => ProcessResult::Changed(vec![]), + _ => unreachable!(), + }, + |_| {}, + )); + let mut ok = ok; + ok.sort(); + assert_eq!(ok, vec!["A", "A.2", "A.2.i", "A.2.i.a"]); + assert!(err.is_empty()); + + let TestOutcome { completed: ok, errors: err, .. } = + forest.process_obligations(&mut C(|_| unreachable!(), |_| {})); + + assert!(ok.is_empty()); + assert!(err.is_empty()); +} + +#[test] +fn to_errors_no_throw() { + // check that converting multiple children with common parent (A) + // yields to correct errors (and does not panic, in particular). + let mut forest = ObligationForest::new(); + forest.register_obligation("A"); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]), + "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); + assert_eq!(err.len(), 0); + let errors = forest.to_errors(()); + assert_eq!(errors[0].backtrace, vec!["A.1", "A"]); + assert_eq!(errors[1].backtrace, vec!["A.2", "A"]); + assert_eq!(errors[2].backtrace, vec!["A.3", "A"]); + assert_eq!(errors.len(), 3); +} + +#[test] +fn diamond() { + // check that diamond dependencies are handled correctly + let mut forest = ObligationForest::new(); + forest.register_obligation("A"); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Changed(vec!["A.1", "A.2"]), + "A.1" | "A.2" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); + assert_eq!(err.len(), 0); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.1" => ProcessResult::Changed(vec!["D"]), + "A.2" => ProcessResult::Changed(vec!["D"]), + "D" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); + assert_eq!(err.len(), 0); + + let mut d_count = 0; + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D" => { + d_count += 1; + ProcessResult::Changed(vec![]) + } + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(d_count, 1); + let mut ok = ok; + ok.sort(); + assert_eq!(ok, vec!["A", "A.1", "A.2", "D"]); + assert_eq!(err.len(), 0); + + let errors = forest.to_errors(()); + assert_eq!(errors.len(), 0); + + forest.register_obligation("A'"); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A'" => ProcessResult::Changed(vec!["A'.1", "A'.2"]), + "A'.1" | "A'.2" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); + assert_eq!(err.len(), 0); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A'.1" => ProcessResult::Changed(vec!["D'", "A'"]), + "A'.2" => ProcessResult::Changed(vec!["D'"]), + "D'" | "A'" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); + assert_eq!(err.len(), 0); + + let mut d_count = 0; + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D'" => { + d_count += 1; + ProcessResult::Error("operation failed") + } + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(d_count, 1); + assert_eq!(ok.len(), 0); + assert_eq!( + err, + vec![super::Error { error: "operation failed", backtrace: vec!["D'", "A'.1", "A'"] }] + ); + + let errors = forest.to_errors(()); + assert_eq!(errors.len(), 0); +} + +#[test] +fn done_dependency() { + // check that the local cache works + let mut forest = ObligationForest::new(); + forest.register_obligation("A: Sized"); + forest.register_obligation("B: Sized"); + forest.register_obligation("C: Sized"); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A: Sized" | "B: Sized" | "C: Sized" => ProcessResult::Changed(vec![]), + _ => unreachable!(), + }, + |_| {}, + )); + let mut ok = ok; + ok.sort(); + assert_eq!(ok, vec!["A: Sized", "B: Sized", "C: Sized"]); + assert_eq!(err.len(), 0); + + forest.register_obligation("(A,B,C): Sized"); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "(A,B,C): Sized" => ProcessResult::Changed(vec!["A: Sized", "B: Sized", "C: Sized"]), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok, vec!["(A,B,C): Sized"]); + assert_eq!(err.len(), 0); +} + +#[test] +fn orphan() { + // check that orphaned nodes are handled correctly + let mut forest = ObligationForest::new(); + forest.register_obligation("A"); + forest.register_obligation("B"); + forest.register_obligation("C1"); + forest.register_obligation("C2"); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Changed(vec!["D", "E"]), + "B" => ProcessResult::Unchanged, + "C1" => ProcessResult::Changed(vec![]), + "C2" => ProcessResult::Changed(vec![]), + "D" | "E" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + let mut ok = ok; + ok.sort(); + assert_eq!(ok, vec!["C1", "C2"]); + assert_eq!(err.len(), 0); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D" | "E" => ProcessResult::Unchanged, + "B" => ProcessResult::Changed(vec!["D"]), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); + assert_eq!(err.len(), 0); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D" => ProcessResult::Unchanged, + "E" => ProcessResult::Error("E is for error"), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); + assert_eq!(err, vec![super::Error { error: "E is for error", backtrace: vec!["E", "A"] }]); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D" => ProcessResult::Error("D is dead"), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); + assert_eq!(err, vec![super::Error { error: "D is dead", backtrace: vec!["D"] }]); + + let errors = forest.to_errors(()); + assert_eq!(errors.len(), 0); +} + +#[test] +fn simultaneous_register_and_error() { + // check that registering a failed obligation works correctly + let mut forest = ObligationForest::new(); + forest.register_obligation("A"); + forest.register_obligation("B"); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Error("An error"), + "B" => ProcessResult::Changed(vec!["A"]), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); + assert_eq!(err, vec![super::Error { error: "An error", backtrace: vec!["A"] }]); + + let mut forest = ObligationForest::new(); + forest.register_obligation("B"); + forest.register_obligation("A"); + + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Error("An error"), + "B" => ProcessResult::Changed(vec!["A"]), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); + assert_eq!(err, vec![super::Error { error: "An error", backtrace: vec!["A"] }]); +} diff --git a/compiler/rustc_data_structures/src/owning_ref/LICENSE b/compiler/rustc_data_structures/src/owning_ref/LICENSE new file mode 100644 index 000000000..dff72d1e4 --- /dev/null +++ b/compiler/rustc_data_structures/src/owning_ref/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Marvin Löbel + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/compiler/rustc_data_structures/src/owning_ref/mod.rs b/compiler/rustc_data_structures/src/owning_ref/mod.rs new file mode 100644 index 000000000..ed5e56618 --- /dev/null +++ b/compiler/rustc_data_structures/src/owning_ref/mod.rs @@ -0,0 +1,1214 @@ +#![warn(missing_docs)] + +/*! +# An owning reference. + +This crate provides the _owning reference_ types `OwningRef` and `OwningRefMut` +that enables it to bundle a reference together with the owner of the data it points to. +This allows moving and dropping of an `OwningRef` without needing to recreate the reference. + +This can sometimes be useful because Rust borrowing rules normally prevent +moving a type that has been moved from. For example, this kind of code gets rejected: + +```compile_fail,E0515 +fn return_owned_and_referenced<'a>() -> (Vec<u8>, &'a [u8]) { + let v = vec![1, 2, 3, 4]; + let s = &v[1..3]; + (v, s) +} +``` + +Even though, from a memory-layout point of view, this can be entirely safe +if the new location of the vector still lives longer than the lifetime `'a` +of the reference because the backing allocation of the vector does not change. + +This library enables this safe usage by keeping the owner and the reference +bundled together in a wrapper type that ensure that lifetime constraint: + +``` +# use rustc_data_structures::owning_ref::OwningRef; +# fn main() { +fn return_owned_and_referenced() -> OwningRef<Vec<u8>, [u8]> { + let v = vec![1, 2, 3, 4]; + let or = OwningRef::new(v); + let or = or.map(|v| &v[1..3]); + or +} +# } +``` + +It works by requiring owner types to dereference to stable memory locations +and preventing mutable access to root containers, which in practice requires heap allocation +as provided by `Box<T>`, `Rc<T>`, etc. + +Also provided are typedefs for common owner type combinations, +which allow for less verbose type signatures. +For example, `BoxRef<T>` instead of `OwningRef<Box<T>, T>`. + +The crate also provides the more advanced `OwningHandle` type, +which allows more freedom in bundling a dependent handle object +along with the data it depends on, at the cost of some unsafe needed in the API. +See the documentation around `OwningHandle` for more details. + +# Examples + +## Basics + +``` +use rustc_data_structures::owning_ref::BoxRef; + +fn main() { + // Create an array owned by a Box. + let arr = Box::new([1, 2, 3, 4]) as Box<[i32]>; + + // Transfer into a BoxRef. + let arr: BoxRef<[i32]> = BoxRef::new(arr); + assert_eq!(&*arr, &[1, 2, 3, 4]); + + // We can slice the array without losing ownership or changing type. + let arr: BoxRef<[i32]> = arr.map(|arr| &arr[1..3]); + assert_eq!(&*arr, &[2, 3]); + + // Also works for Arc, Rc, String and Vec! +} +``` + +## Caching a reference to a struct field + +``` +use rustc_data_structures::owning_ref::BoxRef; + +fn main() { + struct Foo { + tag: u32, + x: u16, + y: u16, + z: u16, + } + let foo = Foo { tag: 1, x: 100, y: 200, z: 300 }; + + let or = BoxRef::new(Box::new(foo)).map(|foo| { + match foo.tag { + 0 => &foo.x, + 1 => &foo.y, + 2 => &foo.z, + _ => panic!(), + } + }); + + assert_eq!(*or, 200); +} +``` + +## Caching a reference to an entry in a vector + +``` +use rustc_data_structures::owning_ref::VecRef; + +fn main() { + let v = VecRef::new(vec![1, 2, 3, 4, 5]).map(|v| &v[3]); + assert_eq!(*v, 4); +} +``` + +## Caching a subslice of a String + +``` +use rustc_data_structures::owning_ref::StringRef; + +fn main() { + let s = StringRef::new("hello world".to_owned()) + .map(|s| s.split(' ').nth(1).unwrap()); + + assert_eq!(&*s, "world"); +} +``` + +## Reference counted slices that share ownership of the backing storage + +``` +use rustc_data_structures::owning_ref::RcRef; +use std::rc::Rc; + +fn main() { + let rc: RcRef<[i32]> = RcRef::new(Rc::new([1, 2, 3, 4]) as Rc<[i32]>); + assert_eq!(&*rc, &[1, 2, 3, 4]); + + let rc_a: RcRef<[i32]> = rc.clone().map(|s| &s[0..2]); + let rc_b = rc.clone().map(|s| &s[1..3]); + let rc_c = rc.clone().map(|s| &s[2..4]); + assert_eq!(&*rc_a, &[1, 2]); + assert_eq!(&*rc_b, &[2, 3]); + assert_eq!(&*rc_c, &[3, 4]); + + let rc_c_a = rc_c.clone().map(|s| &s[1]); + assert_eq!(&*rc_c_a, &4); +} +``` + +## Atomic reference counted slices that share ownership of the backing storage + +``` +use rustc_data_structures::owning_ref::ArcRef; +use std::sync::Arc; + +fn main() { + use std::thread; + + fn par_sum(rc: ArcRef<[i32]>) -> i32 { + if rc.len() == 0 { + return 0; + } else if rc.len() == 1 { + return rc[0]; + } + let mid = rc.len() / 2; + let left = rc.clone().map(|s| &s[..mid]); + let right = rc.map(|s| &s[mid..]); + + let left = thread::spawn(move || par_sum(left)); + let right = thread::spawn(move || par_sum(right)); + + left.join().unwrap() + right.join().unwrap() + } + + let rc: Arc<[i32]> = Arc::new([1, 2, 3, 4]); + let rc: ArcRef<[i32]> = rc.into(); + + assert_eq!(par_sum(rc), 10); +} +``` + +## References into RAII locks + +``` +use rustc_data_structures::owning_ref::RefRef; +use std::cell::{RefCell, Ref}; + +fn main() { + let refcell = RefCell::new((1, 2, 3, 4)); + // Also works with Mutex and RwLock + + let refref = { + let refref = RefRef::new(refcell.borrow()).map(|x| &x.3); + assert_eq!(*refref, 4); + + // We move the RAII lock and the reference to one of + // the subfields in the data it guards here: + refref + }; + + assert_eq!(*refref, 4); + + drop(refref); + + assert_eq!(*refcell.borrow(), (1, 2, 3, 4)); +} +``` + +## Mutable reference + +When the owned container implements `DerefMut`, it is also possible to make +a _mutable owning reference_. (e.g., with `Box`, `RefMut`, `MutexGuard`) + +``` +use rustc_data_structures::owning_ref::RefMutRefMut; +use std::cell::{RefCell, RefMut}; + +fn main() { + let refcell = RefCell::new((1, 2, 3, 4)); + + let mut refmut_refmut = { + let mut refmut_refmut = RefMutRefMut::new(refcell.borrow_mut()).map_mut(|x| &mut x.3); + assert_eq!(*refmut_refmut, 4); + *refmut_refmut *= 2; + + refmut_refmut + }; + + assert_eq!(*refmut_refmut, 8); + *refmut_refmut *= 2; + + drop(refmut_refmut); + + assert_eq!(*refcell.borrow(), (1, 2, 3, 16)); +} +``` +*/ + +pub use stable_deref_trait::{ + CloneStableDeref as CloneStableAddress, StableDeref as StableAddress, +}; +use std::mem; + +/// An owning reference. +/// +/// This wraps an owner `O` and a reference `&T` pointing +/// at something reachable from `O::Target` while keeping +/// the ability to move `self` around. +/// +/// The owner is usually a pointer that points at some base type. +/// +/// For more details and examples, see the module and method docs. +pub struct OwningRef<O, T: ?Sized> { + owner: O, + reference: *const T, +} + +/// An mutable owning reference. +/// +/// This wraps an owner `O` and a reference `&mut T` pointing +/// at something reachable from `O::Target` while keeping +/// the ability to move `self` around. +/// +/// The owner is usually a pointer that points at some base type. +/// +/// For more details and examples, see the module and method docs. +pub struct OwningRefMut<O, T: ?Sized> { + owner: O, + reference: *mut T, +} + +/// Helper trait for an erased concrete type an owner dereferences to. +/// This is used in form of a trait object for keeping +/// something around to (virtually) call the destructor. +pub trait Erased {} +impl<T> Erased for T {} + +/// Helper trait for erasing the concrete type of what an owner dereferences to, +/// for example `Box<T> -> Box<Erased>`. This would be unneeded with +/// higher kinded types support in the language. +#[allow(unused_lifetimes)] +pub unsafe trait IntoErased<'a> { + /// Owner with the dereference type substituted to `Erased`. + type Erased; + /// Performs the type erasure. + fn into_erased(self) -> Self::Erased; +} + +/// Helper trait for erasing the concrete type of what an owner dereferences to, +/// for example `Box<T> -> Box<Erased + Send>`. This would be unneeded with +/// higher kinded types support in the language. +#[allow(unused_lifetimes)] +pub unsafe trait IntoErasedSend<'a> { + /// Owner with the dereference type substituted to `Erased + Send`. + type Erased: Send; + /// Performs the type erasure. + fn into_erased_send(self) -> Self::Erased; +} + +/// Helper trait for erasing the concrete type of what an owner dereferences to, +/// for example `Box<T> -> Box<Erased + Send + Sync>`. This would be unneeded with +/// higher kinded types support in the language. +#[allow(unused_lifetimes)] +pub unsafe trait IntoErasedSendSync<'a> { + /// Owner with the dereference type substituted to `Erased + Send + Sync`. + type Erased: Send + Sync; + /// Performs the type erasure. + fn into_erased_send_sync(self) -> Self::Erased; +} + +///////////////////////////////////////////////////////////////////////////// +// OwningRef +///////////////////////////////////////////////////////////////////////////// + +impl<O, T: ?Sized> OwningRef<O, T> { + /// Creates a new owning reference from an owner + /// initialized to the direct dereference of it. + /// + /// # Example + /// ``` + /// use rustc_data_structures::owning_ref::OwningRef; + /// + /// fn main() { + /// let owning_ref = OwningRef::new(Box::new(42)); + /// assert_eq!(*owning_ref, 42); + /// } + /// ``` + pub fn new(o: O) -> Self + where + O: StableAddress, + O: Deref<Target = T>, + { + OwningRef { reference: &*o, owner: o } + } + + /// Like `new`, but doesn’t require `O` to implement the `StableAddress` trait. + /// Instead, the caller is responsible to make the same promises as implementing the trait. + /// + /// This is useful for cases where coherence rules prevents implementing the trait + /// without adding a dependency to this crate in a third-party library. + pub unsafe fn new_assert_stable_address(o: O) -> Self + where + O: Deref<Target = T>, + { + OwningRef { reference: &*o, owner: o } + } + + /// Converts `self` into a new owning reference that points at something reachable + /// from the previous one. + /// + /// This can be a reference to a field of `U`, something reachable from a field of + /// `U`, or even something unrelated with a `'static` lifetime. + /// + /// # Example + /// ``` + /// use rustc_data_structures::owning_ref::OwningRef; + /// + /// fn main() { + /// let owning_ref = OwningRef::new(Box::new([1, 2, 3, 4])); + /// + /// // create an owning reference that points at the + /// // third element of the array. + /// let owning_ref = owning_ref.map(|array| &array[2]); + /// assert_eq!(*owning_ref, 3); + /// } + /// ``` + pub fn map<F, U: ?Sized>(self, f: F) -> OwningRef<O, U> + where + O: StableAddress, + F: FnOnce(&T) -> &U, + { + OwningRef { reference: f(&self), owner: self.owner } + } + + /// Tries to convert `self` into a new owning reference that points + /// at something reachable from the previous one. + /// + /// This can be a reference to a field of `U`, something reachable from a field of + /// `U`, or even something unrelated with a `'static` lifetime. + /// + /// # Example + /// ``` + /// use rustc_data_structures::owning_ref::OwningRef; + /// + /// fn main() { + /// let owning_ref = OwningRef::new(Box::new([1, 2, 3, 4])); + /// + /// // create an owning reference that points at the + /// // third element of the array. + /// let owning_ref = owning_ref.try_map(|array| { + /// if array[2] == 3 { Ok(&array[2]) } else { Err(()) } + /// }); + /// assert_eq!(*owning_ref.unwrap(), 3); + /// } + /// ``` + pub fn try_map<F, U: ?Sized, E>(self, f: F) -> Result<OwningRef<O, U>, E> + where + O: StableAddress, + F: FnOnce(&T) -> Result<&U, E>, + { + Ok(OwningRef { reference: f(&self)?, owner: self.owner }) + } + + /// Converts `self` into a new owning reference with a different owner type. + /// + /// The new owner type needs to still contain the original owner in some way + /// so that the reference into it remains valid. This function is marked unsafe + /// because the user needs to manually uphold this guarantee. + pub unsafe fn map_owner<F, P>(self, f: F) -> OwningRef<P, T> + where + O: StableAddress, + P: StableAddress, + F: FnOnce(O) -> P, + { + OwningRef { reference: self.reference, owner: f(self.owner) } + } + + /// Converts `self` into a new owning reference where the owner is wrapped + /// in an additional `Box<O>`. + /// + /// This can be used to safely erase the owner of any `OwningRef<O, T>` + /// to an `OwningRef<Box<Erased>, T>`. + pub fn map_owner_box(self) -> OwningRef<Box<O>, T> { + OwningRef { reference: self.reference, owner: Box::new(self.owner) } + } + + /// Erases the concrete base type of the owner with a trait object. + /// + /// This allows mixing of owned references with different owner base types. + /// + /// # Example + /// ``` + /// use rustc_data_structures::owning_ref::{OwningRef, Erased}; + /// + /// fn main() { + /// // N.B., using the concrete types here for explicitness. + /// // For less verbose code type aliases like `BoxRef` are provided. + /// + /// let owning_ref_a: OwningRef<Box<[i32; 4]>, [i32; 4]> + /// = OwningRef::new(Box::new([1, 2, 3, 4])); + /// + /// let owning_ref_b: OwningRef<Box<Vec<(i32, bool)>>, Vec<(i32, bool)>> + /// = OwningRef::new(Box::new(vec![(0, false), (1, true)])); + /// + /// let owning_ref_a: OwningRef<Box<[i32; 4]>, i32> + /// = owning_ref_a.map(|a| &a[0]); + /// + /// let owning_ref_b: OwningRef<Box<Vec<(i32, bool)>>, i32> + /// = owning_ref_b.map(|a| &a[1].0); + /// + /// let owning_refs: [OwningRef<Box<dyn Erased>, i32>; 2] + /// = [owning_ref_a.erase_owner(), owning_ref_b.erase_owner()]; + /// + /// assert_eq!(*owning_refs[0], 1); + /// assert_eq!(*owning_refs[1], 1); + /// } + /// ``` + pub fn erase_owner<'a>(self) -> OwningRef<O::Erased, T> + where + O: IntoErased<'a>, + { + OwningRef { reference: self.reference, owner: self.owner.into_erased() } + } + + /// Erases the concrete base type of the owner with a trait object which implements `Send`. + /// + /// This allows mixing of owned references with different owner base types. + pub fn erase_send_owner<'a>(self) -> OwningRef<O::Erased, T> + where + O: IntoErasedSend<'a>, + { + OwningRef { reference: self.reference, owner: self.owner.into_erased_send() } + } + + /// Erases the concrete base type of the owner with a trait object + /// which implements `Send` and `Sync`. + /// + /// This allows mixing of owned references with different owner base types. + pub fn erase_send_sync_owner<'a>(self) -> OwningRef<O::Erased, T> + where + O: IntoErasedSendSync<'a>, + { + OwningRef { reference: self.reference, owner: self.owner.into_erased_send_sync() } + } + + // UNIMPLEMENTED: wrap_owner + + // FIXME: Naming convention? + /// A getter for the underlying owner. + pub fn owner(&self) -> &O { + &self.owner + } + + // FIXME: Naming convention? + /// Discards the reference and retrieves the owner. + pub fn into_inner(self) -> O { + self.owner + } +} + +impl<O, T: ?Sized> OwningRefMut<O, T> { + /// Creates a new owning reference from an owner + /// initialized to the direct dereference of it. + /// + /// # Example + /// ``` + /// use rustc_data_structures::owning_ref::OwningRefMut; + /// + /// fn main() { + /// let owning_ref_mut = OwningRefMut::new(Box::new(42)); + /// assert_eq!(*owning_ref_mut, 42); + /// } + /// ``` + pub fn new(mut o: O) -> Self + where + O: StableAddress, + O: DerefMut<Target = T>, + { + OwningRefMut { reference: &mut *o, owner: o } + } + + /// Like `new`, but doesn’t require `O` to implement the `StableAddress` trait. + /// Instead, the caller is responsible to make the same promises as implementing the trait. + /// + /// This is useful for cases where coherence rules prevents implementing the trait + /// without adding a dependency to this crate in a third-party library. + pub unsafe fn new_assert_stable_address(mut o: O) -> Self + where + O: DerefMut<Target = T>, + { + OwningRefMut { reference: &mut *o, owner: o } + } + + /// Converts `self` into a new _shared_ owning reference that points at + /// something reachable from the previous one. + /// + /// This can be a reference to a field of `U`, something reachable from a field of + /// `U`, or even something unrelated with a `'static` lifetime. + /// + /// # Example + /// ``` + /// use rustc_data_structures::owning_ref::OwningRefMut; + /// + /// fn main() { + /// let owning_ref_mut = OwningRefMut::new(Box::new([1, 2, 3, 4])); + /// + /// // create an owning reference that points at the + /// // third element of the array. + /// let owning_ref = owning_ref_mut.map(|array| &array[2]); + /// assert_eq!(*owning_ref, 3); + /// } + /// ``` + pub fn map<F, U: ?Sized>(mut self, f: F) -> OwningRef<O, U> + where + O: StableAddress, + F: FnOnce(&mut T) -> &U, + { + OwningRef { reference: f(&mut self), owner: self.owner } + } + + /// Converts `self` into a new _mutable_ owning reference that points at + /// something reachable from the previous one. + /// + /// This can be a reference to a field of `U`, something reachable from a field of + /// `U`, or even something unrelated with a `'static` lifetime. + /// + /// # Example + /// ``` + /// use rustc_data_structures::owning_ref::OwningRefMut; + /// + /// fn main() { + /// let owning_ref_mut = OwningRefMut::new(Box::new([1, 2, 3, 4])); + /// + /// // create an owning reference that points at the + /// // third element of the array. + /// let owning_ref_mut = owning_ref_mut.map_mut(|array| &mut array[2]); + /// assert_eq!(*owning_ref_mut, 3); + /// } + /// ``` + pub fn map_mut<F, U: ?Sized>(mut self, f: F) -> OwningRefMut<O, U> + where + O: StableAddress, + F: FnOnce(&mut T) -> &mut U, + { + OwningRefMut { reference: f(&mut self), owner: self.owner } + } + + /// Tries to convert `self` into a new _shared_ owning reference that points + /// at something reachable from the previous one. + /// + /// This can be a reference to a field of `U`, something reachable from a field of + /// `U`, or even something unrelated with a `'static` lifetime. + /// + /// # Example + /// ``` + /// use rustc_data_structures::owning_ref::OwningRefMut; + /// + /// fn main() { + /// let owning_ref_mut = OwningRefMut::new(Box::new([1, 2, 3, 4])); + /// + /// // create an owning reference that points at the + /// // third element of the array. + /// let owning_ref = owning_ref_mut.try_map(|array| { + /// if array[2] == 3 { Ok(&array[2]) } else { Err(()) } + /// }); + /// assert_eq!(*owning_ref.unwrap(), 3); + /// } + /// ``` + pub fn try_map<F, U: ?Sized, E>(mut self, f: F) -> Result<OwningRef<O, U>, E> + where + O: StableAddress, + F: FnOnce(&mut T) -> Result<&U, E>, + { + Ok(OwningRef { reference: f(&mut self)?, owner: self.owner }) + } + + /// Tries to convert `self` into a new _mutable_ owning reference that points + /// at something reachable from the previous one. + /// + /// This can be a reference to a field of `U`, something reachable from a field of + /// `U`, or even something unrelated with a `'static` lifetime. + /// + /// # Example + /// ``` + /// use rustc_data_structures::owning_ref::OwningRefMut; + /// + /// fn main() { + /// let owning_ref_mut = OwningRefMut::new(Box::new([1, 2, 3, 4])); + /// + /// // create an owning reference that points at the + /// // third element of the array. + /// let owning_ref_mut = owning_ref_mut.try_map_mut(|array| { + /// if array[2] == 3 { Ok(&mut array[2]) } else { Err(()) } + /// }); + /// assert_eq!(*owning_ref_mut.unwrap(), 3); + /// } + /// ``` + pub fn try_map_mut<F, U: ?Sized, E>(mut self, f: F) -> Result<OwningRefMut<O, U>, E> + where + O: StableAddress, + F: FnOnce(&mut T) -> Result<&mut U, E>, + { + Ok(OwningRefMut { reference: f(&mut self)?, owner: self.owner }) + } + + /// Converts `self` into a new owning reference with a different owner type. + /// + /// The new owner type needs to still contain the original owner in some way + /// so that the reference into it remains valid. This function is marked unsafe + /// because the user needs to manually uphold this guarantee. + pub unsafe fn map_owner<F, P>(self, f: F) -> OwningRefMut<P, T> + where + O: StableAddress, + P: StableAddress, + F: FnOnce(O) -> P, + { + OwningRefMut { reference: self.reference, owner: f(self.owner) } + } + + /// Converts `self` into a new owning reference where the owner is wrapped + /// in an additional `Box<O>`. + /// + /// This can be used to safely erase the owner of any `OwningRefMut<O, T>` + /// to an `OwningRefMut<Box<Erased>, T>`. + pub fn map_owner_box(self) -> OwningRefMut<Box<O>, T> { + OwningRefMut { reference: self.reference, owner: Box::new(self.owner) } + } + + /// Erases the concrete base type of the owner with a trait object. + /// + /// This allows mixing of owned references with different owner base types. + /// + /// # Example + /// ``` + /// use rustc_data_structures::owning_ref::{OwningRefMut, Erased}; + /// + /// fn main() { + /// // N.B., using the concrete types here for explicitness. + /// // For less verbose code type aliases like `BoxRef` are provided. + /// + /// let owning_ref_mut_a: OwningRefMut<Box<[i32; 4]>, [i32; 4]> + /// = OwningRefMut::new(Box::new([1, 2, 3, 4])); + /// + /// let owning_ref_mut_b: OwningRefMut<Box<Vec<(i32, bool)>>, Vec<(i32, bool)>> + /// = OwningRefMut::new(Box::new(vec![(0, false), (1, true)])); + /// + /// let owning_ref_mut_a: OwningRefMut<Box<[i32; 4]>, i32> + /// = owning_ref_mut_a.map_mut(|a| &mut a[0]); + /// + /// let owning_ref_mut_b: OwningRefMut<Box<Vec<(i32, bool)>>, i32> + /// = owning_ref_mut_b.map_mut(|a| &mut a[1].0); + /// + /// let owning_refs_mut: [OwningRefMut<Box<dyn Erased>, i32>; 2] + /// = [owning_ref_mut_a.erase_owner(), owning_ref_mut_b.erase_owner()]; + /// + /// assert_eq!(*owning_refs_mut[0], 1); + /// assert_eq!(*owning_refs_mut[1], 1); + /// } + /// ``` + pub fn erase_owner<'a>(self) -> OwningRefMut<O::Erased, T> + where + O: IntoErased<'a>, + { + OwningRefMut { reference: self.reference, owner: self.owner.into_erased() } + } + + // UNIMPLEMENTED: wrap_owner + + // FIXME: Naming convention? + /// A getter for the underlying owner. + pub fn owner(&self) -> &O { + &self.owner + } + + // FIXME: Naming convention? + /// Discards the reference and retrieves the owner. + pub fn into_inner(self) -> O { + self.owner + } +} + +///////////////////////////////////////////////////////////////////////////// +// OwningHandle +///////////////////////////////////////////////////////////////////////////// + +use std::ops::{Deref, DerefMut}; + +/// `OwningHandle` is a complement to `OwningRef`. Where `OwningRef` allows +/// consumers to pass around an owned object and a dependent reference, +/// `OwningHandle` contains an owned object and a dependent _object_. +/// +/// `OwningHandle` can encapsulate a `RefMut` along with its associated +/// `RefCell`, or an `RwLockReadGuard` along with its associated `RwLock`. +/// However, the API is completely generic and there are no restrictions on +/// what types of owning and dependent objects may be used. +/// +/// `OwningHandle` is created by passing an owner object (which dereferences +/// to a stable address) along with a callback which receives a pointer to +/// that stable location. The callback may then dereference the pointer and +/// mint a dependent object, with the guarantee that the returned object will +/// not outlive the referent of the pointer. +/// +/// Since the callback needs to dereference a raw pointer, it requires `unsafe` +/// code. To avoid forcing this unsafety on most callers, the `ToHandle` trait is +/// implemented for common data structures. Types that implement `ToHandle` can +/// be wrapped into an `OwningHandle` without passing a callback. +pub struct OwningHandle<O, H> +where + O: StableAddress, + H: Deref, +{ + handle: H, + _owner: O, +} + +impl<O, H> Deref for OwningHandle<O, H> +where + O: StableAddress, + H: Deref, +{ + type Target = H::Target; + fn deref(&self) -> &H::Target { + self.handle.deref() + } +} + +unsafe impl<O, H> StableAddress for OwningHandle<O, H> +where + O: StableAddress, + H: StableAddress, +{ +} + +impl<O, H> DerefMut for OwningHandle<O, H> +where + O: StableAddress, + H: DerefMut, +{ + fn deref_mut(&mut self) -> &mut H::Target { + self.handle.deref_mut() + } +} + +/// Trait to implement the conversion of owner to handle for common types. +pub trait ToHandle { + /// The type of handle to be encapsulated by the OwningHandle. + type Handle: Deref; + + /// Given an appropriately-long-lived pointer to ourselves, create a + /// handle to be encapsulated by the `OwningHandle`. + unsafe fn to_handle(x: *const Self) -> Self::Handle; +} + +/// Trait to implement the conversion of owner to mutable handle for common types. +pub trait ToHandleMut { + /// The type of handle to be encapsulated by the OwningHandle. + type HandleMut: DerefMut; + + /// Given an appropriately-long-lived pointer to ourselves, create a + /// mutable handle to be encapsulated by the `OwningHandle`. + unsafe fn to_handle_mut(x: *const Self) -> Self::HandleMut; +} + +impl<O, H> OwningHandle<O, H> +where + O: StableAddress<Target: ToHandle<Handle = H>>, + H: Deref, +{ + /// Creates a new `OwningHandle` for a type that implements `ToHandle`. For types + /// that don't implement `ToHandle`, callers may invoke `new_with_fn`, which accepts + /// a callback to perform the conversion. + pub fn new(o: O) -> Self { + OwningHandle::new_with_fn(o, |x| unsafe { O::Target::to_handle(x) }) + } +} + +impl<O, H> OwningHandle<O, H> +where + O: StableAddress<Target: ToHandleMut<HandleMut = H>>, + H: DerefMut, +{ + /// Creates a new mutable `OwningHandle` for a type that implements `ToHandleMut`. + pub fn new_mut(o: O) -> Self { + OwningHandle::new_with_fn(o, |x| unsafe { O::Target::to_handle_mut(x) }) + } +} + +impl<O, H> OwningHandle<O, H> +where + O: StableAddress, + H: Deref, +{ + /// Creates a new OwningHandle. The provided callback will be invoked with + /// a pointer to the object owned by `o`, and the returned value is stored + /// as the object to which this `OwningHandle` will forward `Deref` and + /// `DerefMut`. + pub fn new_with_fn<F>(o: O, f: F) -> Self + where + F: FnOnce(*const O::Target) -> H, + { + let h: H; + { + h = f(o.deref() as *const O::Target); + } + + OwningHandle { handle: h, _owner: o } + } + + /// Creates a new OwningHandle. The provided callback will be invoked with + /// a pointer to the object owned by `o`, and the returned value is stored + /// as the object to which this `OwningHandle` will forward `Deref` and + /// `DerefMut`. + pub fn try_new<F, E>(o: O, f: F) -> Result<Self, E> + where + F: FnOnce(*const O::Target) -> Result<H, E>, + { + let h: H; + { + h = f(o.deref() as *const O::Target)?; + } + + Ok(OwningHandle { handle: h, _owner: o }) + } +} + +///////////////////////////////////////////////////////////////////////////// +// std traits +///////////////////////////////////////////////////////////////////////////// + +use std::borrow::Borrow; +use std::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; +use std::convert::From; +use std::fmt::{self, Debug}; +use std::hash::{Hash, Hasher}; +use std::marker::{Send, Sync}; + +impl<O, T: ?Sized> Deref for OwningRef<O, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { &*self.reference } + } +} + +impl<O, T: ?Sized> Deref for OwningRefMut<O, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { &*self.reference } + } +} + +impl<O, T: ?Sized> DerefMut for OwningRefMut<O, T> { + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.reference } + } +} + +unsafe impl<O, T: ?Sized> StableAddress for OwningRef<O, T> {} + +impl<O, T: ?Sized> AsRef<T> for OwningRef<O, T> { + fn as_ref(&self) -> &T { + &*self + } +} + +impl<O, T: ?Sized> AsRef<T> for OwningRefMut<O, T> { + fn as_ref(&self) -> &T { + &*self + } +} + +impl<O, T: ?Sized> AsMut<T> for OwningRefMut<O, T> { + fn as_mut(&mut self) -> &mut T { + &mut *self + } +} + +impl<O, T: ?Sized> Borrow<T> for OwningRef<O, T> { + fn borrow(&self) -> &T { + &*self + } +} + +impl<O, T: ?Sized> From<O> for OwningRef<O, T> +where + O: StableAddress, + O: Deref<Target = T>, +{ + fn from(owner: O) -> Self { + OwningRef::new(owner) + } +} + +impl<O, T: ?Sized> From<O> for OwningRefMut<O, T> +where + O: StableAddress, + O: DerefMut<Target = T>, +{ + fn from(owner: O) -> Self { + OwningRefMut::new(owner) + } +} + +impl<O, T: ?Sized> From<OwningRefMut<O, T>> for OwningRef<O, T> +where + O: StableAddress, + O: DerefMut<Target = T>, +{ + fn from(other: OwningRefMut<O, T>) -> Self { + OwningRef { owner: other.owner, reference: other.reference } + } +} + +// ^ FIXME: Is an Into impl for calling into_inner() possible as well? + +impl<O, T: ?Sized> Debug for OwningRef<O, T> +where + O: Debug, + T: Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "OwningRef {{ owner: {:?}, reference: {:?} }}", self.owner(), &**self) + } +} + +impl<O, T: ?Sized> Debug for OwningRefMut<O, T> +where + O: Debug, + T: Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "OwningRefMut {{ owner: {:?}, reference: {:?} }}", self.owner(), &**self) + } +} + +impl<O, T: ?Sized> Clone for OwningRef<O, T> +where + O: CloneStableAddress, +{ + fn clone(&self) -> Self { + OwningRef { owner: self.owner.clone(), reference: self.reference } + } +} + +unsafe impl<O, T: ?Sized> CloneStableAddress for OwningRef<O, T> where O: CloneStableAddress {} + +unsafe impl<O, T: ?Sized> Send for OwningRef<O, T> +where + O: Send, + for<'a> &'a T: Send, +{ +} +unsafe impl<O, T: ?Sized> Sync for OwningRef<O, T> +where + O: Sync, + for<'a> &'a T: Sync, +{ +} + +unsafe impl<O, T: ?Sized> Send for OwningRefMut<O, T> +where + O: Send, + for<'a> &'a mut T: Send, +{ +} +unsafe impl<O, T: ?Sized> Sync for OwningRefMut<O, T> +where + O: Sync, + for<'a> &'a mut T: Sync, +{ +} + +impl Debug for dyn Erased { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "<Erased>",) + } +} + +impl<O, T: ?Sized> PartialEq for OwningRef<O, T> +where + T: PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + (&*self as &T).eq(&*other as &T) + } +} + +impl<O, T: ?Sized> Eq for OwningRef<O, T> where T: Eq {} + +impl<O, T: ?Sized> PartialOrd for OwningRef<O, T> +where + T: PartialOrd, +{ + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + (&*self as &T).partial_cmp(&*other as &T) + } +} + +impl<O, T: ?Sized> Ord for OwningRef<O, T> +where + T: Ord, +{ + fn cmp(&self, other: &Self) -> Ordering { + (&*self as &T).cmp(&*other as &T) + } +} + +impl<O, T: ?Sized> Hash for OwningRef<O, T> +where + T: Hash, +{ + fn hash<H: Hasher>(&self, state: &mut H) { + (&*self as &T).hash(state); + } +} + +impl<O, T: ?Sized> PartialEq for OwningRefMut<O, T> +where + T: PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + (&*self as &T).eq(&*other as &T) + } +} + +impl<O, T: ?Sized> Eq for OwningRefMut<O, T> where T: Eq {} + +impl<O, T: ?Sized> PartialOrd for OwningRefMut<O, T> +where + T: PartialOrd, +{ + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + (&*self as &T).partial_cmp(&*other as &T) + } +} + +impl<O, T: ?Sized> Ord for OwningRefMut<O, T> +where + T: Ord, +{ + fn cmp(&self, other: &Self) -> Ordering { + (&*self as &T).cmp(&*other as &T) + } +} + +impl<O, T: ?Sized> Hash for OwningRefMut<O, T> +where + T: Hash, +{ + fn hash<H: Hasher>(&self, state: &mut H) { + (&*self as &T).hash(state); + } +} + +///////////////////////////////////////////////////////////////////////////// +// std types integration and convenience type defs +///////////////////////////////////////////////////////////////////////////// + +use std::boxed::Box; +use std::cell::{Ref, RefCell, RefMut}; +use std::rc::Rc; +use std::sync::Arc; +use std::sync::{MutexGuard, RwLockReadGuard, RwLockWriteGuard}; + +impl<T: 'static> ToHandle for RefCell<T> { + type Handle = Ref<'static, T>; + unsafe fn to_handle(x: *const Self) -> Self::Handle { + (*x).borrow() + } +} + +impl<T: 'static> ToHandleMut for RefCell<T> { + type HandleMut = RefMut<'static, T>; + unsafe fn to_handle_mut(x: *const Self) -> Self::HandleMut { + (*x).borrow_mut() + } +} + +// N.B., implementing ToHandle{,Mut} for Mutex and RwLock requires a decision +// about which handle creation to use (i.e., read() vs try_read()) as well as +// what to do with error results. + +/// Typedef of an owning reference that uses a `Box` as the owner. +pub type BoxRef<T, U = T> = OwningRef<Box<T>, U>; +/// Typedef of an owning reference that uses a `Vec` as the owner. +pub type VecRef<T, U = T> = OwningRef<Vec<T>, U>; +/// Typedef of an owning reference that uses a `String` as the owner. +pub type StringRef = OwningRef<String, str>; + +/// Typedef of an owning reference that uses an `Rc` as the owner. +pub type RcRef<T, U = T> = OwningRef<Rc<T>, U>; +/// Typedef of an owning reference that uses an `Arc` as the owner. +pub type ArcRef<T, U = T> = OwningRef<Arc<T>, U>; + +/// Typedef of an owning reference that uses a `Ref` as the owner. +pub type RefRef<'a, T, U = T> = OwningRef<Ref<'a, T>, U>; +/// Typedef of an owning reference that uses a `RefMut` as the owner. +pub type RefMutRef<'a, T, U = T> = OwningRef<RefMut<'a, T>, U>; +/// Typedef of an owning reference that uses a `MutexGuard` as the owner. +pub type MutexGuardRef<'a, T, U = T> = OwningRef<MutexGuard<'a, T>, U>; +/// Typedef of an owning reference that uses an `RwLockReadGuard` as the owner. +pub type RwLockReadGuardRef<'a, T, U = T> = OwningRef<RwLockReadGuard<'a, T>, U>; +/// Typedef of an owning reference that uses an `RwLockWriteGuard` as the owner. +pub type RwLockWriteGuardRef<'a, T, U = T> = OwningRef<RwLockWriteGuard<'a, T>, U>; + +/// Typedef of a mutable owning reference that uses a `Box` as the owner. +pub type BoxRefMut<T, U = T> = OwningRefMut<Box<T>, U>; +/// Typedef of a mutable owning reference that uses a `Vec` as the owner. +pub type VecRefMut<T, U = T> = OwningRefMut<Vec<T>, U>; +/// Typedef of a mutable owning reference that uses a `String` as the owner. +pub type StringRefMut = OwningRefMut<String, str>; + +/// Typedef of a mutable owning reference that uses a `RefMut` as the owner. +pub type RefMutRefMut<'a, T, U = T> = OwningRefMut<RefMut<'a, T>, U>; +/// Typedef of a mutable owning reference that uses a `MutexGuard` as the owner. +pub type MutexGuardRefMut<'a, T, U = T> = OwningRefMut<MutexGuard<'a, T>, U>; +/// Typedef of a mutable owning reference that uses an `RwLockWriteGuard` as the owner. +pub type RwLockWriteGuardRefMut<'a, T, U = T> = OwningRef<RwLockWriteGuard<'a, T>, U>; + +unsafe impl<'a, T: 'a> IntoErased<'a> for Box<T> { + type Erased = Box<dyn Erased + 'a>; + fn into_erased(self) -> Self::Erased { + self + } +} +unsafe impl<'a, T: 'a> IntoErased<'a> for Rc<T> { + type Erased = Rc<dyn Erased + 'a>; + fn into_erased(self) -> Self::Erased { + self + } +} +unsafe impl<'a, T: 'a> IntoErased<'a> for Arc<T> { + type Erased = Arc<dyn Erased + 'a>; + fn into_erased(self) -> Self::Erased { + self + } +} + +unsafe impl<'a, T: Send + 'a> IntoErasedSend<'a> for Box<T> { + type Erased = Box<dyn Erased + Send + 'a>; + fn into_erased_send(self) -> Self::Erased { + self + } +} + +unsafe impl<'a, T: Send + 'a> IntoErasedSendSync<'a> for Box<T> { + type Erased = Box<dyn Erased + Sync + Send + 'a>; + fn into_erased_send_sync(self) -> Self::Erased { + let result: Box<dyn Erased + Send + 'a> = self; + // This is safe since Erased can always implement Sync + // Only the destructor is available and it takes &mut self + unsafe { mem::transmute(result) } + } +} + +unsafe impl<'a, T: Send + Sync + 'a> IntoErasedSendSync<'a> for Arc<T> { + type Erased = Arc<dyn Erased + Send + Sync + 'a>; + fn into_erased_send_sync(self) -> Self::Erased { + self + } +} + +/// Typedef of an owning reference that uses an erased `Box` as the owner. +pub type ErasedBoxRef<U> = OwningRef<Box<dyn Erased>, U>; +/// Typedef of an owning reference that uses an erased `Rc` as the owner. +pub type ErasedRcRef<U> = OwningRef<Rc<dyn Erased>, U>; +/// Typedef of an owning reference that uses an erased `Arc` as the owner. +pub type ErasedArcRef<U> = OwningRef<Arc<dyn Erased>, U>; + +/// Typedef of a mutable owning reference that uses an erased `Box` as the owner. +pub type ErasedBoxRefMut<U> = OwningRefMut<Box<dyn Erased>, U>; + +#[cfg(test)] +mod tests; diff --git a/compiler/rustc_data_structures/src/owning_ref/tests.rs b/compiler/rustc_data_structures/src/owning_ref/tests.rs new file mode 100644 index 000000000..320c03d51 --- /dev/null +++ b/compiler/rustc_data_structures/src/owning_ref/tests.rs @@ -0,0 +1,711 @@ +// FIXME: owning_ref is not sound under stacked borrows. Preferably, get rid of it. +#[cfg(not(miri))] +mod owning_ref { + use super::super::OwningRef; + use super::super::{BoxRef, Erased, ErasedBoxRef, RcRef}; + use std::cmp::{Ord, Ordering, PartialEq, PartialOrd}; + use std::collections::hash_map::DefaultHasher; + use std::collections::HashMap; + use std::hash::{Hash, Hasher}; + use std::rc::Rc; + + #[derive(Debug, PartialEq)] + struct Example(u32, String, [u8; 3]); + fn example() -> Example { + Example(42, "hello world".to_string(), [1, 2, 3]) + } + + #[test] + fn new_deref() { + let or: OwningRef<Box<()>, ()> = OwningRef::new(Box::new(())); + assert_eq!(&*or, &()); + } + + #[test] + fn into() { + let or: OwningRef<Box<()>, ()> = Box::new(()).into(); + assert_eq!(&*or, &()); + } + + #[test] + fn map_offset_ref() { + let or: BoxRef<Example> = Box::new(example()).into(); + let or: BoxRef<_, u32> = or.map(|x| &x.0); + assert_eq!(&*or, &42); + + let or: BoxRef<Example> = Box::new(example()).into(); + let or: BoxRef<_, u8> = or.map(|x| &x.2[1]); + assert_eq!(&*or, &2); + } + + #[test] + fn map_heap_ref() { + let or: BoxRef<Example> = Box::new(example()).into(); + let or: BoxRef<_, str> = or.map(|x| &x.1[..5]); + assert_eq!(&*or, "hello"); + } + + #[test] + fn map_static_ref() { + let or: BoxRef<()> = Box::new(()).into(); + let or: BoxRef<_, str> = or.map(|_| "hello"); + assert_eq!(&*or, "hello"); + } + + #[test] + fn map_chained() { + let or: BoxRef<String> = Box::new(example().1).into(); + let or: BoxRef<_, str> = or.map(|x| &x[1..5]); + let or: BoxRef<_, str> = or.map(|x| &x[..2]); + assert_eq!(&*or, "el"); + } + + #[test] + fn map_chained_inference() { + let or = BoxRef::new(Box::new(example().1)).map(|x| &x[..5]).map(|x| &x[1..3]); + assert_eq!(&*or, "el"); + } + + #[test] + fn owner() { + let or: BoxRef<String> = Box::new(example().1).into(); + let or = or.map(|x| &x[..5]); + assert_eq!(&*or, "hello"); + assert_eq!(&**or.owner(), "hello world"); + } + + #[test] + fn into_inner() { + let or: BoxRef<String> = Box::new(example().1).into(); + let or = or.map(|x| &x[..5]); + assert_eq!(&*or, "hello"); + let s = *or.into_inner(); + assert_eq!(&s, "hello world"); + } + + #[test] + fn fmt_debug() { + let or: BoxRef<String> = Box::new(example().1).into(); + let or = or.map(|x| &x[..5]); + let s = format!("{:?}", or); + assert_eq!(&s, "OwningRef { owner: \"hello world\", reference: \"hello\" }"); + } + + #[test] + fn erased_owner() { + let o1: BoxRef<Example, str> = BoxRef::new(Box::new(example())).map(|x| &x.1[..]); + + let o2: BoxRef<String, str> = BoxRef::new(Box::new(example().1)).map(|x| &x[..]); + + let os: Vec<ErasedBoxRef<str>> = vec![o1.erase_owner(), o2.erase_owner()]; + assert!(os.iter().all(|e| &e[..] == "hello world")); + } + + #[test] + fn raii_locks() { + use super::super::{MutexGuardRef, RwLockReadGuardRef, RwLockWriteGuardRef}; + use super::super::{RefMutRef, RefRef}; + use std::cell::RefCell; + use std::sync::{Mutex, RwLock}; + + { + let a = RefCell::new(1); + let a = { + let a = RefRef::new(a.borrow()); + assert_eq!(*a, 1); + a + }; + assert_eq!(*a, 1); + drop(a); + } + { + let a = RefCell::new(1); + let a = { + let a = RefMutRef::new(a.borrow_mut()); + assert_eq!(*a, 1); + a + }; + assert_eq!(*a, 1); + drop(a); + } + { + let a = Mutex::new(1); + let a = { + let a = MutexGuardRef::new(a.lock().unwrap()); + assert_eq!(*a, 1); + a + }; + assert_eq!(*a, 1); + drop(a); + } + { + let a = RwLock::new(1); + let a = { + let a = RwLockReadGuardRef::new(a.read().unwrap()); + assert_eq!(*a, 1); + a + }; + assert_eq!(*a, 1); + drop(a); + } + { + let a = RwLock::new(1); + let a = { + let a = RwLockWriteGuardRef::new(a.write().unwrap()); + assert_eq!(*a, 1); + a + }; + assert_eq!(*a, 1); + drop(a); + } + } + + #[test] + fn eq() { + let or1: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice()); + let or2: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice()); + assert_eq!(or1.eq(&or2), true); + } + + #[test] + fn cmp() { + let or1: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice()); + let or2: BoxRef<[u8]> = BoxRef::new(vec![4, 5, 6].into_boxed_slice()); + assert_eq!(or1.cmp(&or2), Ordering::Less); + } + + #[test] + fn partial_cmp() { + let or1: BoxRef<[u8]> = BoxRef::new(vec![4, 5, 6].into_boxed_slice()); + let or2: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice()); + assert_eq!(or1.partial_cmp(&or2), Some(Ordering::Greater)); + } + + #[test] + fn hash() { + let mut h1 = DefaultHasher::new(); + let mut h2 = DefaultHasher::new(); + + let or1: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice()); + let or2: BoxRef<[u8]> = BoxRef::new(vec![1, 2, 3].into_boxed_slice()); + + or1.hash(&mut h1); + or2.hash(&mut h2); + + assert_eq!(h1.finish(), h2.finish()); + } + + #[test] + fn borrow() { + let mut hash = HashMap::new(); + let key = RcRef::<String>::new(Rc::new("foo-bar".to_string())).map(|s| &s[..]); + + hash.insert(key.clone().map(|s| &s[..3]), 42); + hash.insert(key.clone().map(|s| &s[4..]), 23); + + assert_eq!(hash.get("foo"), Some(&42)); + assert_eq!(hash.get("bar"), Some(&23)); + } + + #[test] + fn total_erase() { + let a: OwningRef<Vec<u8>, [u8]> = OwningRef::new(vec![]).map(|x| &x[..]); + let b: OwningRef<Box<[u8]>, [u8]> = + OwningRef::new(vec![].into_boxed_slice()).map(|x| &x[..]); + + let c: OwningRef<Rc<Vec<u8>>, [u8]> = unsafe { a.map_owner(Rc::new) }; + let d: OwningRef<Rc<Box<[u8]>>, [u8]> = unsafe { b.map_owner(Rc::new) }; + + let e: OwningRef<Rc<dyn Erased>, [u8]> = c.erase_owner(); + let f: OwningRef<Rc<dyn Erased>, [u8]> = d.erase_owner(); + + let _g = e.clone(); + let _h = f.clone(); + } + + #[test] + fn total_erase_box() { + let a: OwningRef<Vec<u8>, [u8]> = OwningRef::new(vec![]).map(|x| &x[..]); + let b: OwningRef<Box<[u8]>, [u8]> = + OwningRef::new(vec![].into_boxed_slice()).map(|x| &x[..]); + + let c: OwningRef<Box<Vec<u8>>, [u8]> = a.map_owner_box(); + let d: OwningRef<Box<Box<[u8]>>, [u8]> = b.map_owner_box(); + + let _e: OwningRef<Box<dyn Erased>, [u8]> = c.erase_owner(); + let _f: OwningRef<Box<dyn Erased>, [u8]> = d.erase_owner(); + } + + #[test] + fn try_map1() { + use std::any::Any; + + let x = Box::new(123_i32); + let y: Box<dyn Any> = x; + + assert!(OwningRef::new(y).try_map(|x| x.downcast_ref::<i32>().ok_or(())).is_ok()); + } + + #[test] + fn try_map2() { + use std::any::Any; + + let x = Box::new(123_i32); + let y: Box<dyn Any> = x; + + assert!(!OwningRef::new(y).try_map(|x| x.downcast_ref::<i32>().ok_or(())).is_err()); + } +} + +mod owning_handle { + use super::super::OwningHandle; + use super::super::RcRef; + use std::cell::RefCell; + use std::rc::Rc; + use std::sync::Arc; + use std::sync::RwLock; + + #[test] + fn owning_handle() { + use std::cell::RefCell; + let cell = Rc::new(RefCell::new(2)); + let cell_ref = RcRef::new(cell); + let mut handle = + OwningHandle::new_with_fn(cell_ref, |x| unsafe { x.as_ref() }.unwrap().borrow_mut()); + assert_eq!(*handle, 2); + *handle = 3; + assert_eq!(*handle, 3); + } + + #[test] + fn try_owning_handle_ok() { + use std::cell::RefCell; + let cell = Rc::new(RefCell::new(2)); + let cell_ref = RcRef::new(cell); + let mut handle = OwningHandle::try_new::<_, ()>(cell_ref, |x| { + Ok(unsafe { x.as_ref() }.unwrap().borrow_mut()) + }) + .unwrap(); + assert_eq!(*handle, 2); + *handle = 3; + assert_eq!(*handle, 3); + } + + #[test] + fn try_owning_handle_err() { + use std::cell::RefCell; + let cell = Rc::new(RefCell::new(2)); + let cell_ref = RcRef::new(cell); + let handle = OwningHandle::try_new::<_, ()>(cell_ref, |x| { + if false { + return Ok(unsafe { x.as_ref() }.unwrap().borrow_mut()); + } + Err(()) + }); + assert!(handle.is_err()); + } + + #[test] + fn nested() { + use std::cell::RefCell; + use std::sync::{Arc, RwLock}; + + let result = { + let complex = Rc::new(RefCell::new(Arc::new(RwLock::new("someString")))); + let curr = RcRef::new(complex); + let curr = + OwningHandle::new_with_fn(curr, |x| unsafe { x.as_ref() }.unwrap().borrow_mut()); + let mut curr = OwningHandle::new_with_fn(curr, |x| { + unsafe { x.as_ref() }.unwrap().try_write().unwrap() + }); + assert_eq!(*curr, "someString"); + *curr = "someOtherString"; + curr + }; + assert_eq!(*result, "someOtherString"); + } + + #[test] + fn owning_handle_safe() { + use std::cell::RefCell; + let cell = Rc::new(RefCell::new(2)); + let cell_ref = RcRef::new(cell); + let handle = OwningHandle::new(cell_ref); + assert_eq!(*handle, 2); + } + + #[test] + fn owning_handle_mut_safe() { + use std::cell::RefCell; + let cell = Rc::new(RefCell::new(2)); + let cell_ref = RcRef::new(cell); + let mut handle = OwningHandle::new_mut(cell_ref); + assert_eq!(*handle, 2); + *handle = 3; + assert_eq!(*handle, 3); + } + + #[test] + fn owning_handle_safe_2() { + let result = { + let complex = Rc::new(RefCell::new(Arc::new(RwLock::new("someString")))); + let curr = RcRef::new(complex); + let curr = + OwningHandle::new_with_fn(curr, |x| unsafe { x.as_ref() }.unwrap().borrow_mut()); + let mut curr = OwningHandle::new_with_fn(curr, |x| { + unsafe { x.as_ref() }.unwrap().try_write().unwrap() + }); + assert_eq!(*curr, "someString"); + *curr = "someOtherString"; + curr + }; + assert_eq!(*result, "someOtherString"); + } +} + +// FIXME: owning_ref is not sound under stacked borrows. Preferably, get rid of it. +#[cfg(not(miri))] +mod owning_ref_mut { + use super::super::BoxRef; + use super::super::{BoxRefMut, Erased, ErasedBoxRefMut, OwningRefMut}; + use std::cmp::{Ord, Ordering, PartialEq, PartialOrd}; + use std::collections::hash_map::DefaultHasher; + use std::collections::HashMap; + use std::hash::{Hash, Hasher}; + + #[derive(Debug, PartialEq)] + struct Example(u32, String, [u8; 3]); + fn example() -> Example { + Example(42, "hello world".to_string(), [1, 2, 3]) + } + + #[test] + fn new_deref() { + let or: OwningRefMut<Box<()>, ()> = OwningRefMut::new(Box::new(())); + assert_eq!(&*or, &()); + } + + #[test] + fn new_deref_mut() { + let mut or: OwningRefMut<Box<()>, ()> = OwningRefMut::new(Box::new(())); + assert_eq!(&mut *or, &mut ()); + } + + #[test] + fn mutate() { + let mut or: OwningRefMut<Box<usize>, usize> = OwningRefMut::new(Box::new(0)); + assert_eq!(&*or, &0); + *or = 1; + assert_eq!(&*or, &1); + } + + #[test] + fn into() { + let or: OwningRefMut<Box<()>, ()> = Box::new(()).into(); + assert_eq!(&*or, &()); + } + + #[test] + fn map_offset_ref() { + let or: BoxRefMut<Example> = Box::new(example()).into(); + let or: BoxRef<_, u32> = or.map(|x| &mut x.0); + assert_eq!(&*or, &42); + + let or: BoxRefMut<Example> = Box::new(example()).into(); + let or: BoxRef<_, u8> = or.map(|x| &mut x.2[1]); + assert_eq!(&*or, &2); + } + + #[test] + fn map_heap_ref() { + let or: BoxRefMut<Example> = Box::new(example()).into(); + let or: BoxRef<_, str> = or.map(|x| &mut x.1[..5]); + assert_eq!(&*or, "hello"); + } + + #[test] + fn map_static_ref() { + let or: BoxRefMut<()> = Box::new(()).into(); + let or: BoxRef<_, str> = or.map(|_| "hello"); + assert_eq!(&*or, "hello"); + } + + #[test] + fn map_mut_offset_ref() { + let or: BoxRefMut<Example> = Box::new(example()).into(); + let or: BoxRefMut<_, u32> = or.map_mut(|x| &mut x.0); + assert_eq!(&*or, &42); + + let or: BoxRefMut<Example> = Box::new(example()).into(); + let or: BoxRefMut<_, u8> = or.map_mut(|x| &mut x.2[1]); + assert_eq!(&*or, &2); + } + + #[test] + fn map_mut_heap_ref() { + let or: BoxRefMut<Example> = Box::new(example()).into(); + let or: BoxRefMut<_, str> = or.map_mut(|x| &mut x.1[..5]); + assert_eq!(&*or, "hello"); + } + + #[test] + fn map_mut_static_ref() { + static mut MUT_S: [u8; 5] = *b"hello"; + + let mut_s: &'static mut [u8] = unsafe { &mut MUT_S }; + + let or: BoxRefMut<()> = Box::new(()).into(); + let or: BoxRefMut<_, [u8]> = or.map_mut(move |_| mut_s); + assert_eq!(&*or, b"hello"); + } + + #[test] + fn map_mut_chained() { + let or: BoxRefMut<String> = Box::new(example().1).into(); + let or: BoxRefMut<_, str> = or.map_mut(|x| &mut x[1..5]); + let or: BoxRefMut<_, str> = or.map_mut(|x| &mut x[..2]); + assert_eq!(&*or, "el"); + } + + #[test] + fn map_chained_inference() { + let or = BoxRefMut::new(Box::new(example().1)) + .map_mut(|x| &mut x[..5]) + .map_mut(|x| &mut x[1..3]); + assert_eq!(&*or, "el"); + } + + #[test] + fn try_map_mut() { + let or: BoxRefMut<String> = Box::new(example().1).into(); + let or: Result<BoxRefMut<_, str>, ()> = or.try_map_mut(|x| Ok(&mut x[1..5])); + assert_eq!(&*or.unwrap(), "ello"); + + let or: BoxRefMut<String> = Box::new(example().1).into(); + let or: Result<BoxRefMut<_, str>, ()> = or.try_map_mut(|_| Err(())); + assert!(or.is_err()); + } + + #[test] + fn owner() { + let or: BoxRefMut<String> = Box::new(example().1).into(); + let or = or.map_mut(|x| &mut x[..5]); + assert_eq!(&*or, "hello"); + assert_eq!(&**or.owner(), "hello world"); + } + + #[test] + fn into_inner() { + let or: BoxRefMut<String> = Box::new(example().1).into(); + let or = or.map_mut(|x| &mut x[..5]); + assert_eq!(&*or, "hello"); + let s = *or.into_inner(); + assert_eq!(&s, "hello world"); + } + + #[test] + fn fmt_debug() { + let or: BoxRefMut<String> = Box::new(example().1).into(); + let or = or.map_mut(|x| &mut x[..5]); + let s = format!("{:?}", or); + assert_eq!(&s, "OwningRefMut { owner: \"hello world\", reference: \"hello\" }"); + } + + #[test] + fn erased_owner() { + let o1: BoxRefMut<Example, str> = + BoxRefMut::new(Box::new(example())).map_mut(|x| &mut x.1[..]); + + let o2: BoxRefMut<String, str> = + BoxRefMut::new(Box::new(example().1)).map_mut(|x| &mut x[..]); + + let os: Vec<ErasedBoxRefMut<str>> = vec![o1.erase_owner(), o2.erase_owner()]; + assert!(os.iter().all(|e| &e[..] == "hello world")); + } + + #[test] + fn raii_locks() { + use super::super::RefMutRefMut; + use super::super::{MutexGuardRefMut, RwLockWriteGuardRefMut}; + use std::cell::RefCell; + use std::sync::{Mutex, RwLock}; + + { + let a = RefCell::new(1); + let a = { + let a = RefMutRefMut::new(a.borrow_mut()); + assert_eq!(*a, 1); + a + }; + assert_eq!(*a, 1); + drop(a); + } + { + let a = Mutex::new(1); + let a = { + let a = MutexGuardRefMut::new(a.lock().unwrap()); + assert_eq!(*a, 1); + a + }; + assert_eq!(*a, 1); + drop(a); + } + { + let a = RwLock::new(1); + let a = { + let a = RwLockWriteGuardRefMut::new(a.write().unwrap()); + assert_eq!(*a, 1); + a + }; + assert_eq!(*a, 1); + drop(a); + } + } + + #[test] + fn eq() { + let or1: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice()); + let or2: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice()); + assert_eq!(or1.eq(&or2), true); + } + + #[test] + fn cmp() { + let or1: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice()); + let or2: BoxRefMut<[u8]> = BoxRefMut::new(vec![4, 5, 6].into_boxed_slice()); + assert_eq!(or1.cmp(&or2), Ordering::Less); + } + + #[test] + fn partial_cmp() { + let or1: BoxRefMut<[u8]> = BoxRefMut::new(vec![4, 5, 6].into_boxed_slice()); + let or2: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice()); + assert_eq!(or1.partial_cmp(&or2), Some(Ordering::Greater)); + } + + #[test] + fn hash() { + let mut h1 = DefaultHasher::new(); + let mut h2 = DefaultHasher::new(); + + let or1: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice()); + let or2: BoxRefMut<[u8]> = BoxRefMut::new(vec![1, 2, 3].into_boxed_slice()); + + or1.hash(&mut h1); + or2.hash(&mut h2); + + assert_eq!(h1.finish(), h2.finish()); + } + + #[test] + fn borrow() { + let mut hash = HashMap::new(); + let key1 = BoxRefMut::<String>::new(Box::new("foo".to_string())).map(|s| &s[..]); + let key2 = BoxRefMut::<String>::new(Box::new("bar".to_string())).map(|s| &s[..]); + + hash.insert(key1, 42); + hash.insert(key2, 23); + + assert_eq!(hash.get("foo"), Some(&42)); + assert_eq!(hash.get("bar"), Some(&23)); + } + + #[test] + fn total_erase() { + let a: OwningRefMut<Vec<u8>, [u8]> = OwningRefMut::new(vec![]).map_mut(|x| &mut x[..]); + let b: OwningRefMut<Box<[u8]>, [u8]> = + OwningRefMut::new(vec![].into_boxed_slice()).map_mut(|x| &mut x[..]); + + let c: OwningRefMut<Box<Vec<u8>>, [u8]> = unsafe { a.map_owner(Box::new) }; + let d: OwningRefMut<Box<Box<[u8]>>, [u8]> = unsafe { b.map_owner(Box::new) }; + + let _e: OwningRefMut<Box<dyn Erased>, [u8]> = c.erase_owner(); + let _f: OwningRefMut<Box<dyn Erased>, [u8]> = d.erase_owner(); + } + + #[test] + fn total_erase_box() { + let a: OwningRefMut<Vec<u8>, [u8]> = OwningRefMut::new(vec![]).map_mut(|x| &mut x[..]); + let b: OwningRefMut<Box<[u8]>, [u8]> = + OwningRefMut::new(vec![].into_boxed_slice()).map_mut(|x| &mut x[..]); + + let c: OwningRefMut<Box<Vec<u8>>, [u8]> = a.map_owner_box(); + let d: OwningRefMut<Box<Box<[u8]>>, [u8]> = b.map_owner_box(); + + let _e: OwningRefMut<Box<dyn Erased>, [u8]> = c.erase_owner(); + let _f: OwningRefMut<Box<dyn Erased>, [u8]> = d.erase_owner(); + } + + #[test] + fn try_map1() { + use std::any::Any; + + let x = Box::new(123_i32); + let y: Box<dyn Any> = x; + + assert!(OwningRefMut::new(y).try_map_mut(|x| x.downcast_mut::<i32>().ok_or(())).is_ok()); + } + + #[test] + fn try_map2() { + use std::any::Any; + + let x = Box::new(123_i32); + let y: Box<dyn Any> = x; + + assert!(!OwningRefMut::new(y).try_map_mut(|x| x.downcast_mut::<i32>().ok_or(())).is_err()); + } + + #[test] + fn try_map3() { + use std::any::Any; + + let x = Box::new(123_i32); + let y: Box<dyn Any> = x; + + assert!(OwningRefMut::new(y).try_map(|x| x.downcast_ref::<i32>().ok_or(())).is_ok()); + } + + #[test] + fn try_map4() { + use std::any::Any; + + let x = Box::new(123_i32); + let y: Box<dyn Any> = x; + + assert!(!OwningRefMut::new(y).try_map(|x| x.downcast_ref::<i32>().ok_or(())).is_err()); + } + + #[test] + fn into_owning_ref() { + use super::super::BoxRef; + + let or: BoxRefMut<()> = Box::new(()).into(); + let or: BoxRef<()> = or.into(); + assert_eq!(&*or, &()); + } + + struct Foo { + u: u32, + } + struct Bar { + f: Foo, + } + + #[test] + fn ref_mut() { + use std::cell::RefCell; + + let a = RefCell::new(Bar { f: Foo { u: 42 } }); + let mut b = OwningRefMut::new(a.borrow_mut()); + assert_eq!(b.f.u, 42); + b.f.u = 43; + let mut c = b.map_mut(|x| &mut x.f); + assert_eq!(c.u, 43); + c.u = 44; + let mut d = c.map_mut(|x| &mut x.u); + assert_eq!(*d, 44); + *d = 45; + assert_eq!(*d, 45); + } +} diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs new file mode 100644 index 000000000..d8b26f984 --- /dev/null +++ b/compiler/rustc_data_structures/src/profiling.rs @@ -0,0 +1,861 @@ +//! # Rust Compiler Self-Profiling +//! +//! This module implements the basic framework for the compiler's self- +//! profiling support. It provides the `SelfProfiler` type which enables +//! recording "events". An event is something that starts and ends at a given +//! point in time and has an ID and a kind attached to it. This allows for +//! tracing the compiler's activity. +//! +//! Internally this module uses the custom tailored [measureme][mm] crate for +//! efficiently recording events to disk in a compact format that can be +//! post-processed and analyzed by the suite of tools in the `measureme` +//! project. The highest priority for the tracing framework is on incurring as +//! little overhead as possible. +//! +//! +//! ## Event Overview +//! +//! Events have a few properties: +//! +//! - The `event_kind` designates the broad category of an event (e.g. does it +//! correspond to the execution of a query provider or to loading something +//! from the incr. comp. on-disk cache, etc). +//! - The `event_id` designates the query invocation or function call it +//! corresponds to, possibly including the query key or function arguments. +//! - Each event stores the ID of the thread it was recorded on. +//! - The timestamp stores beginning and end of the event, or the single point +//! in time it occurred at for "instant" events. +//! +//! +//! ## Event Filtering +//! +//! Event generation can be filtered by event kind. Recording all possible +//! events generates a lot of data, much of which is not needed for most kinds +//! of analysis. So, in order to keep overhead as low as possible for a given +//! use case, the `SelfProfiler` will only record the kinds of events that +//! pass the filter specified as a command line argument to the compiler. +//! +//! +//! ## `event_id` Assignment +//! +//! As far as `measureme` is concerned, `event_id`s are just strings. However, +//! it would incur too much overhead to generate and persist each `event_id` +//! string at the point where the event is recorded. In order to make this more +//! efficient `measureme` has two features: +//! +//! - Strings can share their content, so that re-occurring parts don't have to +//! be copied over and over again. One allocates a string in `measureme` and +//! gets back a `StringId`. This `StringId` is then used to refer to that +//! string. `measureme` strings are actually DAGs of string components so that +//! arbitrary sharing of substrings can be done efficiently. This is useful +//! because `event_id`s contain lots of redundant text like query names or +//! def-path components. +//! +//! - `StringId`s can be "virtual" which means that the client picks a numeric +//! ID according to some application-specific scheme and can later make that +//! ID be mapped to an actual string. This is used to cheaply generate +//! `event_id`s while the events actually occur, causing little timing +//! distortion, and then later map those `StringId`s, in bulk, to actual +//! `event_id` strings. This way the largest part of the tracing overhead is +//! localized to one contiguous chunk of time. +//! +//! How are these `event_id`s generated in the compiler? For things that occur +//! infrequently (e.g. "generic activities"), we just allocate the string the +//! first time it is used and then keep the `StringId` in a hash table. This +//! is implemented in `SelfProfiler::get_or_alloc_cached_string()`. +//! +//! For queries it gets more interesting: First we need a unique numeric ID for +//! each query invocation (the `QueryInvocationId`). This ID is used as the +//! virtual `StringId` we use as `event_id` for a given event. This ID has to +//! be available both when the query is executed and later, together with the +//! query key, when we allocate the actual `event_id` strings in bulk. +//! +//! We could make the compiler generate and keep track of such an ID for each +//! query invocation but luckily we already have something that fits all the +//! the requirements: the query's `DepNodeIndex`. So we use the numeric value +//! of the `DepNodeIndex` as `event_id` when recording the event and then, +//! just before the query context is dropped, we walk the entire query cache +//! (which stores the `DepNodeIndex` along with the query key for each +//! invocation) and allocate the corresponding strings together with a mapping +//! for `DepNodeIndex as StringId`. +//! +//! [mm]: https://github.com/rust-lang/measureme/ + +use crate::cold_path; +use crate::fx::FxHashMap; + +use std::borrow::Borrow; +use std::collections::hash_map::Entry; +use std::convert::Into; +use std::error::Error; +use std::fs; +use std::path::Path; +use std::process; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +pub use measureme::EventId; +use measureme::{EventIdBuilder, Profiler, SerializableString, StringId}; +use parking_lot::RwLock; +use smallvec::SmallVec; + +bitflags::bitflags! { + struct EventFilter: u32 { + const GENERIC_ACTIVITIES = 1 << 0; + const QUERY_PROVIDERS = 1 << 1; + const QUERY_CACHE_HITS = 1 << 2; + const QUERY_BLOCKED = 1 << 3; + const INCR_CACHE_LOADS = 1 << 4; + + const QUERY_KEYS = 1 << 5; + const FUNCTION_ARGS = 1 << 6; + const LLVM = 1 << 7; + const INCR_RESULT_HASHING = 1 << 8; + const ARTIFACT_SIZES = 1 << 9; + + const DEFAULT = Self::GENERIC_ACTIVITIES.bits | + Self::QUERY_PROVIDERS.bits | + Self::QUERY_BLOCKED.bits | + Self::INCR_CACHE_LOADS.bits | + Self::INCR_RESULT_HASHING.bits | + Self::ARTIFACT_SIZES.bits; + + const ARGS = Self::QUERY_KEYS.bits | Self::FUNCTION_ARGS.bits; + } +} + +// keep this in sync with the `-Z self-profile-events` help message in rustc_session/options.rs +const EVENT_FILTERS_BY_NAME: &[(&str, EventFilter)] = &[ + ("none", EventFilter::empty()), + ("all", EventFilter::all()), + ("default", EventFilter::DEFAULT), + ("generic-activity", EventFilter::GENERIC_ACTIVITIES), + ("query-provider", EventFilter::QUERY_PROVIDERS), + ("query-cache-hit", EventFilter::QUERY_CACHE_HITS), + ("query-blocked", EventFilter::QUERY_BLOCKED), + ("incr-cache-load", EventFilter::INCR_CACHE_LOADS), + ("query-keys", EventFilter::QUERY_KEYS), + ("function-args", EventFilter::FUNCTION_ARGS), + ("args", EventFilter::ARGS), + ("llvm", EventFilter::LLVM), + ("incr-result-hashing", EventFilter::INCR_RESULT_HASHING), + ("artifact-sizes", EventFilter::ARTIFACT_SIZES), +]; + +/// Something that uniquely identifies a query invocation. +pub struct QueryInvocationId(pub u32); + +/// A reference to the SelfProfiler. It can be cloned and sent across thread +/// boundaries at will. +#[derive(Clone)] +pub struct SelfProfilerRef { + // This field is `None` if self-profiling is disabled for the current + // compilation session. + profiler: Option<Arc<SelfProfiler>>, + + // We store the filter mask directly in the reference because that doesn't + // cost anything and allows for filtering with checking if the profiler is + // actually enabled. + event_filter_mask: EventFilter, + + // Print verbose generic activities to stdout + print_verbose_generic_activities: bool, + + // Print extra verbose generic activities to stdout + print_extra_verbose_generic_activities: bool, +} + +impl SelfProfilerRef { + pub fn new( + profiler: Option<Arc<SelfProfiler>>, + print_verbose_generic_activities: bool, + print_extra_verbose_generic_activities: bool, + ) -> SelfProfilerRef { + // If there is no SelfProfiler then the filter mask is set to NONE, + // ensuring that nothing ever tries to actually access it. + let event_filter_mask = + profiler.as_ref().map_or(EventFilter::empty(), |p| p.event_filter_mask); + + SelfProfilerRef { + profiler, + event_filter_mask, + print_verbose_generic_activities, + print_extra_verbose_generic_activities, + } + } + + /// This shim makes sure that calls only get executed if the filter mask + /// lets them pass. It also contains some trickery to make sure that + /// code is optimized for non-profiling compilation sessions, i.e. anything + /// past the filter check is never inlined so it doesn't clutter the fast + /// path. + #[inline(always)] + fn exec<F>(&self, event_filter: EventFilter, f: F) -> TimingGuard<'_> + where + F: for<'a> FnOnce(&'a SelfProfiler) -> TimingGuard<'a>, + { + #[inline(never)] + #[cold] + fn cold_call<F>(profiler_ref: &SelfProfilerRef, f: F) -> TimingGuard<'_> + where + F: for<'a> FnOnce(&'a SelfProfiler) -> TimingGuard<'a>, + { + let profiler = profiler_ref.profiler.as_ref().unwrap(); + f(&**profiler) + } + + if self.event_filter_mask.contains(event_filter) { + cold_call(self, f) + } else { + TimingGuard::none() + } + } + + /// Start profiling a verbose generic activity. Profiling continues until the + /// VerboseTimingGuard returned from this call is dropped. In addition to recording + /// a measureme event, "verbose" generic activities also print a timing entry to + /// stdout if the compiler is invoked with -Ztime or -Ztime-passes. + pub fn verbose_generic_activity<'a>( + &'a self, + event_label: &'static str, + ) -> VerboseTimingGuard<'a> { + let message = + if self.print_verbose_generic_activities { Some(event_label.to_owned()) } else { None }; + + VerboseTimingGuard::start(message, self.generic_activity(event_label)) + } + + /// Start profiling an extra verbose generic activity. Profiling continues until the + /// VerboseTimingGuard returned from this call is dropped. In addition to recording + /// a measureme event, "extra verbose" generic activities also print a timing entry to + /// stdout if the compiler is invoked with -Ztime-passes. + pub fn extra_verbose_generic_activity<'a, A>( + &'a self, + event_label: &'static str, + event_arg: A, + ) -> VerboseTimingGuard<'a> + where + A: Borrow<str> + Into<String>, + { + let message = if self.print_extra_verbose_generic_activities { + Some(format!("{}({})", event_label, event_arg.borrow())) + } else { + None + }; + + VerboseTimingGuard::start(message, self.generic_activity_with_arg(event_label, event_arg)) + } + + /// Start profiling a generic activity. Profiling continues until the + /// TimingGuard returned from this call is dropped. + #[inline(always)] + pub fn generic_activity(&self, event_label: &'static str) -> TimingGuard<'_> { + self.exec(EventFilter::GENERIC_ACTIVITIES, |profiler| { + let event_label = profiler.get_or_alloc_cached_string(event_label); + let event_id = EventId::from_label(event_label); + TimingGuard::start(profiler, profiler.generic_activity_event_kind, event_id) + }) + } + + /// Start profiling with some event filter for a given event. Profiling continues until the + /// TimingGuard returned from this call is dropped. + #[inline(always)] + pub fn generic_activity_with_event_id(&self, event_id: EventId) -> TimingGuard<'_> { + self.exec(EventFilter::GENERIC_ACTIVITIES, |profiler| { + TimingGuard::start(profiler, profiler.generic_activity_event_kind, event_id) + }) + } + + /// Start profiling a generic activity. Profiling continues until the + /// TimingGuard returned from this call is dropped. + #[inline(always)] + pub fn generic_activity_with_arg<A>( + &self, + event_label: &'static str, + event_arg: A, + ) -> TimingGuard<'_> + where + A: Borrow<str> + Into<String>, + { + self.exec(EventFilter::GENERIC_ACTIVITIES, |profiler| { + let builder = EventIdBuilder::new(&profiler.profiler); + let event_label = profiler.get_or_alloc_cached_string(event_label); + let event_id = if profiler.event_filter_mask.contains(EventFilter::FUNCTION_ARGS) { + let event_arg = profiler.get_or_alloc_cached_string(event_arg); + builder.from_label_and_arg(event_label, event_arg) + } else { + builder.from_label(event_label) + }; + TimingGuard::start(profiler, profiler.generic_activity_event_kind, event_id) + }) + } + + /// Start profiling a generic activity, allowing costly arguments to be recorded. Profiling + /// continues until the `TimingGuard` returned from this call is dropped. + /// + /// If the arguments to a generic activity are cheap to create, use `generic_activity_with_arg` + /// or `generic_activity_with_args` for their simpler API. However, if they are costly or + /// require allocation in sufficiently hot contexts, then this allows for a closure to be called + /// only when arguments were asked to be recorded via `-Z self-profile-events=args`. + /// + /// In this case, the closure will be passed a `&mut EventArgRecorder`, to help with recording + /// one or many arguments within the generic activity being profiled, by calling its + /// `record_arg` method for example. + /// + /// This `EventArgRecorder` may implement more specific traits from other rustc crates, e.g. for + /// richer handling of rustc-specific argument types, while keeping this single entry-point API + /// for recording arguments. + /// + /// Note: recording at least one argument is *required* for the self-profiler to create the + /// `TimingGuard`. A panic will be triggered if that doesn't happen. This function exists + /// explicitly to record arguments, so it fails loudly when there are none to record. + /// + #[inline(always)] + pub fn generic_activity_with_arg_recorder<F>( + &self, + event_label: &'static str, + mut f: F, + ) -> TimingGuard<'_> + where + F: FnMut(&mut EventArgRecorder<'_>), + { + // Ensure this event will only be recorded when self-profiling is turned on. + self.exec(EventFilter::GENERIC_ACTIVITIES, |profiler| { + let builder = EventIdBuilder::new(&profiler.profiler); + let event_label = profiler.get_or_alloc_cached_string(event_label); + + // Ensure the closure to create event arguments will only be called when argument + // recording is turned on. + let event_id = if profiler.event_filter_mask.contains(EventFilter::FUNCTION_ARGS) { + // Set up the builder and call the user-provided closure to record potentially + // costly event arguments. + let mut recorder = EventArgRecorder { profiler, args: SmallVec::new() }; + f(&mut recorder); + + // It is expected that the closure will record at least one argument. If that + // doesn't happen, it's a bug: we've been explicitly called in order to record + // arguments, so we fail loudly when there are none to record. + if recorder.args.is_empty() { + panic!( + "The closure passed to `generic_activity_with_arg_recorder` needs to \ + record at least one argument" + ); + } + + builder.from_label_and_args(event_label, &recorder.args) + } else { + builder.from_label(event_label) + }; + TimingGuard::start(profiler, profiler.generic_activity_event_kind, event_id) + }) + } + + /// Record the size of an artifact that the compiler produces + /// + /// `artifact_kind` is the class of artifact (e.g., query_cache, object_file, etc.) + /// `artifact_name` is an identifier to the specific artifact being stored (usually a filename) + #[inline(always)] + pub fn artifact_size<A>(&self, artifact_kind: &str, artifact_name: A, size: u64) + where + A: Borrow<str> + Into<String>, + { + drop(self.exec(EventFilter::ARTIFACT_SIZES, |profiler| { + let builder = EventIdBuilder::new(&profiler.profiler); + let event_label = profiler.get_or_alloc_cached_string(artifact_kind); + let event_arg = profiler.get_or_alloc_cached_string(artifact_name); + let event_id = builder.from_label_and_arg(event_label, event_arg); + let thread_id = get_thread_id(); + + profiler.profiler.record_integer_event( + profiler.artifact_size_event_kind, + event_id, + thread_id, + size, + ); + + TimingGuard::none() + })) + } + + #[inline(always)] + pub fn generic_activity_with_args( + &self, + event_label: &'static str, + event_args: &[String], + ) -> TimingGuard<'_> { + self.exec(EventFilter::GENERIC_ACTIVITIES, |profiler| { + let builder = EventIdBuilder::new(&profiler.profiler); + let event_label = profiler.get_or_alloc_cached_string(event_label); + let event_id = if profiler.event_filter_mask.contains(EventFilter::FUNCTION_ARGS) { + let event_args: Vec<_> = event_args + .iter() + .map(|s| profiler.get_or_alloc_cached_string(&s[..])) + .collect(); + builder.from_label_and_args(event_label, &event_args) + } else { + builder.from_label(event_label) + }; + TimingGuard::start(profiler, profiler.generic_activity_event_kind, event_id) + }) + } + + /// Start profiling a query provider. Profiling continues until the + /// TimingGuard returned from this call is dropped. + #[inline(always)] + pub fn query_provider(&self) -> TimingGuard<'_> { + self.exec(EventFilter::QUERY_PROVIDERS, |profiler| { + TimingGuard::start(profiler, profiler.query_event_kind, EventId::INVALID) + }) + } + + /// Record a query in-memory cache hit. + #[inline(always)] + pub fn query_cache_hit(&self, query_invocation_id: QueryInvocationId) { + self.instant_query_event( + |profiler| profiler.query_cache_hit_event_kind, + query_invocation_id, + EventFilter::QUERY_CACHE_HITS, + ); + } + + /// Start profiling a query being blocked on a concurrent execution. + /// Profiling continues until the TimingGuard returned from this call is + /// dropped. + #[inline(always)] + pub fn query_blocked(&self) -> TimingGuard<'_> { + self.exec(EventFilter::QUERY_BLOCKED, |profiler| { + TimingGuard::start(profiler, profiler.query_blocked_event_kind, EventId::INVALID) + }) + } + + /// Start profiling how long it takes to load a query result from the + /// incremental compilation on-disk cache. Profiling continues until the + /// TimingGuard returned from this call is dropped. + #[inline(always)] + pub fn incr_cache_loading(&self) -> TimingGuard<'_> { + self.exec(EventFilter::INCR_CACHE_LOADS, |profiler| { + TimingGuard::start( + profiler, + profiler.incremental_load_result_event_kind, + EventId::INVALID, + ) + }) + } + + /// Start profiling how long it takes to hash query results for incremental compilation. + /// Profiling continues until the TimingGuard returned from this call is dropped. + #[inline(always)] + pub fn incr_result_hashing(&self) -> TimingGuard<'_> { + self.exec(EventFilter::INCR_RESULT_HASHING, |profiler| { + TimingGuard::start( + profiler, + profiler.incremental_result_hashing_event_kind, + EventId::INVALID, + ) + }) + } + + #[inline(always)] + fn instant_query_event( + &self, + event_kind: fn(&SelfProfiler) -> StringId, + query_invocation_id: QueryInvocationId, + event_filter: EventFilter, + ) { + drop(self.exec(event_filter, |profiler| { + let event_id = StringId::new_virtual(query_invocation_id.0); + let thread_id = get_thread_id(); + + profiler.profiler.record_instant_event( + event_kind(profiler), + EventId::from_virtual(event_id), + thread_id, + ); + + TimingGuard::none() + })); + } + + pub fn with_profiler(&self, f: impl FnOnce(&SelfProfiler)) { + if let Some(profiler) = &self.profiler { + f(&profiler) + } + } + + /// Gets a `StringId` for the given string. This method makes sure that + /// any strings going through it will only be allocated once in the + /// profiling data. + /// Returns `None` if the self-profiling is not enabled. + pub fn get_or_alloc_cached_string(&self, s: &str) -> Option<StringId> { + self.profiler.as_ref().map(|p| p.get_or_alloc_cached_string(s)) + } + + #[inline] + pub fn enabled(&self) -> bool { + self.profiler.is_some() + } + + #[inline] + pub fn llvm_recording_enabled(&self) -> bool { + self.event_filter_mask.contains(EventFilter::LLVM) + } + #[inline] + pub fn get_self_profiler(&self) -> Option<Arc<SelfProfiler>> { + self.profiler.clone() + } +} + +/// A helper for recording costly arguments to self-profiling events. Used with +/// `SelfProfilerRef::generic_activity_with_arg_recorder`. +pub struct EventArgRecorder<'p> { + /// The `SelfProfiler` used to intern the event arguments that users will ask to record. + profiler: &'p SelfProfiler, + + /// The interned event arguments to be recorded in the generic activity event. + /// + /// The most common case, when actually recording event arguments, is to have one argument. Then + /// followed by recording two, in a couple places. + args: SmallVec<[StringId; 2]>, +} + +impl EventArgRecorder<'_> { + /// Records a single argument within the current generic activity being profiled. + /// + /// Note: when self-profiling with costly event arguments, at least one argument + /// needs to be recorded. A panic will be triggered if that doesn't happen. + pub fn record_arg<A>(&mut self, event_arg: A) + where + A: Borrow<str> + Into<String>, + { + let event_arg = self.profiler.get_or_alloc_cached_string(event_arg); + self.args.push(event_arg); + } +} + +pub struct SelfProfiler { + profiler: Profiler, + event_filter_mask: EventFilter, + + string_cache: RwLock<FxHashMap<String, StringId>>, + + query_event_kind: StringId, + generic_activity_event_kind: StringId, + incremental_load_result_event_kind: StringId, + incremental_result_hashing_event_kind: StringId, + query_blocked_event_kind: StringId, + query_cache_hit_event_kind: StringId, + artifact_size_event_kind: StringId, +} + +impl SelfProfiler { + pub fn new( + output_directory: &Path, + crate_name: Option<&str>, + event_filters: Option<&[String]>, + counter_name: &str, + ) -> Result<SelfProfiler, Box<dyn Error + Send + Sync>> { + fs::create_dir_all(output_directory)?; + + let crate_name = crate_name.unwrap_or("unknown-crate"); + // HACK(eddyb) we need to pad the PID, strange as it may seem, as its + // length can behave as a source of entropy for heap addresses, when + // ASLR is disabled and the heap is otherwise determinic. + let pid: u32 = process::id(); + let filename = format!("{}-{:07}.rustc_profile", crate_name, pid); + let path = output_directory.join(&filename); + let profiler = + Profiler::with_counter(&path, measureme::counters::Counter::by_name(counter_name)?)?; + + let query_event_kind = profiler.alloc_string("Query"); + let generic_activity_event_kind = profiler.alloc_string("GenericActivity"); + let incremental_load_result_event_kind = profiler.alloc_string("IncrementalLoadResult"); + let incremental_result_hashing_event_kind = + profiler.alloc_string("IncrementalResultHashing"); + let query_blocked_event_kind = profiler.alloc_string("QueryBlocked"); + let query_cache_hit_event_kind = profiler.alloc_string("QueryCacheHit"); + let artifact_size_event_kind = profiler.alloc_string("ArtifactSize"); + + let mut event_filter_mask = EventFilter::empty(); + + if let Some(event_filters) = event_filters { + let mut unknown_events = vec![]; + for item in event_filters { + if let Some(&(_, mask)) = + EVENT_FILTERS_BY_NAME.iter().find(|&(name, _)| name == item) + { + event_filter_mask |= mask; + } else { + unknown_events.push(item.clone()); + } + } + + // Warn about any unknown event names + if !unknown_events.is_empty() { + unknown_events.sort(); + unknown_events.dedup(); + + warn!( + "Unknown self-profiler events specified: {}. Available options are: {}.", + unknown_events.join(", "), + EVENT_FILTERS_BY_NAME + .iter() + .map(|&(name, _)| name.to_string()) + .collect::<Vec<_>>() + .join(", ") + ); + } + } else { + event_filter_mask = EventFilter::DEFAULT; + } + + Ok(SelfProfiler { + profiler, + event_filter_mask, + string_cache: RwLock::new(FxHashMap::default()), + query_event_kind, + generic_activity_event_kind, + incremental_load_result_event_kind, + incremental_result_hashing_event_kind, + query_blocked_event_kind, + query_cache_hit_event_kind, + artifact_size_event_kind, + }) + } + + /// Allocates a new string in the profiling data. Does not do any caching + /// or deduplication. + pub fn alloc_string<STR: SerializableString + ?Sized>(&self, s: &STR) -> StringId { + self.profiler.alloc_string(s) + } + + /// Gets a `StringId` for the given string. This method makes sure that + /// any strings going through it will only be allocated once in the + /// profiling data. + pub fn get_or_alloc_cached_string<A>(&self, s: A) -> StringId + where + A: Borrow<str> + Into<String>, + { + // Only acquire a read-lock first since we assume that the string is + // already present in the common case. + { + let string_cache = self.string_cache.read(); + + if let Some(&id) = string_cache.get(s.borrow()) { + return id; + } + } + + let mut string_cache = self.string_cache.write(); + // Check if the string has already been added in the small time window + // between dropping the read lock and acquiring the write lock. + match string_cache.entry(s.into()) { + Entry::Occupied(e) => *e.get(), + Entry::Vacant(e) => { + let string_id = self.profiler.alloc_string(&e.key()[..]); + *e.insert(string_id) + } + } + } + + pub fn map_query_invocation_id_to_string(&self, from: QueryInvocationId, to: StringId) { + let from = StringId::new_virtual(from.0); + self.profiler.map_virtual_to_concrete_string(from, to); + } + + pub fn bulk_map_query_invocation_id_to_single_string<I>(&self, from: I, to: StringId) + where + I: Iterator<Item = QueryInvocationId> + ExactSizeIterator, + { + let from = from.map(|qid| StringId::new_virtual(qid.0)); + self.profiler.bulk_map_virtual_to_single_concrete_string(from, to); + } + + pub fn query_key_recording_enabled(&self) -> bool { + self.event_filter_mask.contains(EventFilter::QUERY_KEYS) + } + + pub fn event_id_builder(&self) -> EventIdBuilder<'_> { + EventIdBuilder::new(&self.profiler) + } +} + +#[must_use] +pub struct TimingGuard<'a>(Option<measureme::TimingGuard<'a>>); + +impl<'a> TimingGuard<'a> { + #[inline] + pub fn start( + profiler: &'a SelfProfiler, + event_kind: StringId, + event_id: EventId, + ) -> TimingGuard<'a> { + let thread_id = get_thread_id(); + let raw_profiler = &profiler.profiler; + let timing_guard = + raw_profiler.start_recording_interval_event(event_kind, event_id, thread_id); + TimingGuard(Some(timing_guard)) + } + + #[inline] + pub fn finish_with_query_invocation_id(self, query_invocation_id: QueryInvocationId) { + if let Some(guard) = self.0 { + cold_path(|| { + let event_id = StringId::new_virtual(query_invocation_id.0); + let event_id = EventId::from_virtual(event_id); + guard.finish_with_override_event_id(event_id); + }); + } + } + + #[inline] + pub fn none() -> TimingGuard<'a> { + TimingGuard(None) + } + + #[inline(always)] + pub fn run<R>(self, f: impl FnOnce() -> R) -> R { + let _timer = self; + f() + } +} + +#[must_use] +pub struct VerboseTimingGuard<'a> { + start_and_message: Option<(Instant, Option<usize>, String)>, + _guard: TimingGuard<'a>, +} + +impl<'a> VerboseTimingGuard<'a> { + pub fn start(message: Option<String>, _guard: TimingGuard<'a>) -> Self { + VerboseTimingGuard { + _guard, + start_and_message: message.map(|msg| (Instant::now(), get_resident_set_size(), msg)), + } + } + + #[inline(always)] + pub fn run<R>(self, f: impl FnOnce() -> R) -> R { + let _timer = self; + f() + } +} + +impl Drop for VerboseTimingGuard<'_> { + fn drop(&mut self) { + if let Some((start_time, start_rss, ref message)) = self.start_and_message { + let end_rss = get_resident_set_size(); + let dur = start_time.elapsed(); + + if should_print_passes(dur, start_rss, end_rss) { + print_time_passes_entry(&message, dur, start_rss, end_rss); + } + } + } +} + +fn should_print_passes(dur: Duration, start_rss: Option<usize>, end_rss: Option<usize>) -> bool { + if dur.as_millis() > 5 { + return true; + } + + if let (Some(start_rss), Some(end_rss)) = (start_rss, end_rss) { + let change_rss = end_rss.abs_diff(start_rss); + if change_rss > 0 { + return true; + } + } + + false +} + +pub fn print_time_passes_entry( + what: &str, + dur: Duration, + start_rss: Option<usize>, + end_rss: Option<usize>, +) { + let rss_to_mb = |rss| (rss as f64 / 1_000_000.0).round() as usize; + let rss_change_to_mb = |rss| (rss as f64 / 1_000_000.0).round() as i128; + + let mem_string = match (start_rss, end_rss) { + (Some(start_rss), Some(end_rss)) => { + let change_rss = end_rss as i128 - start_rss as i128; + + format!( + "; rss: {:>4}MB -> {:>4}MB ({:>+5}MB)", + rss_to_mb(start_rss), + rss_to_mb(end_rss), + rss_change_to_mb(change_rss), + ) + } + (Some(start_rss), None) => format!("; rss start: {:>4}MB", rss_to_mb(start_rss)), + (None, Some(end_rss)) => format!("; rss end: {:>4}MB", rss_to_mb(end_rss)), + (None, None) => String::new(), + }; + + eprintln!("time: {:>7}{}\t{}", duration_to_secs_str(dur), mem_string, what); +} + +// Hack up our own formatting for the duration to make it easier for scripts +// to parse (always use the same number of decimal places and the same unit). +pub fn duration_to_secs_str(dur: std::time::Duration) -> String { + format!("{:.3}", dur.as_secs_f64()) +} + +fn get_thread_id() -> u32 { + std::thread::current().id().as_u64().get() as u32 +} + +// Memory reporting +cfg_if! { + if #[cfg(windows)] { + pub fn get_resident_set_size() -> Option<usize> { + use std::mem::{self, MaybeUninit}; + use winapi::shared::minwindef::DWORD; + use winapi::um::processthreadsapi::GetCurrentProcess; + use winapi::um::psapi::{GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS}; + + let mut pmc = MaybeUninit::<PROCESS_MEMORY_COUNTERS>::uninit(); + match unsafe { + GetProcessMemoryInfo(GetCurrentProcess(), pmc.as_mut_ptr(), mem::size_of_val(&pmc) as DWORD) + } { + 0 => None, + _ => { + let pmc = unsafe { pmc.assume_init() }; + Some(pmc.WorkingSetSize as usize) + } + } + } + } else if #[cfg(target_os = "macos")] { + pub fn get_resident_set_size() -> Option<usize> { + use libc::{c_int, c_void, getpid, proc_pidinfo, proc_taskinfo, PROC_PIDTASKINFO}; + use std::mem; + const PROC_TASKINFO_SIZE: c_int = mem::size_of::<proc_taskinfo>() as c_int; + + unsafe { + let mut info: proc_taskinfo = mem::zeroed(); + let info_ptr = &mut info as *mut proc_taskinfo as *mut c_void; + let pid = getpid() as c_int; + let ret = proc_pidinfo(pid, PROC_PIDTASKINFO, 0, info_ptr, PROC_TASKINFO_SIZE); + if ret == PROC_TASKINFO_SIZE { + Some(info.pti_resident_size as usize) + } else { + None + } + } + } + } else if #[cfg(unix)] { + pub fn get_resident_set_size() -> Option<usize> { + let field = 1; + let contents = fs::read("/proc/self/statm").ok()?; + let contents = String::from_utf8(contents).ok()?; + let s = contents.split_whitespace().nth(field)?; + let npages = s.parse::<usize>().ok()?; + Some(npages * 4096) + } + } else { + pub fn get_resident_set_size() -> Option<usize> { + None + } + } +} diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs new file mode 100644 index 000000000..01d292dde --- /dev/null +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -0,0 +1,150 @@ +use crate::fx::{FxHashMap, FxHasher}; +use crate::sync::{Lock, LockGuard}; +use std::borrow::Borrow; +use std::collections::hash_map::RawEntryMut; +use std::hash::{Hash, Hasher}; +use std::mem; + +#[derive(Clone, Default)] +#[cfg_attr(parallel_compiler, repr(align(64)))] +struct CacheAligned<T>(T); + +#[cfg(parallel_compiler)] +// 32 shards is sufficient to reduce contention on an 8-core Ryzen 7 1700, +// but this should be tested on higher core count CPUs. How the `Sharded` type gets used +// may also affect the ideal number of shards. +const SHARD_BITS: usize = 5; + +#[cfg(not(parallel_compiler))] +const SHARD_BITS: usize = 0; + +pub const SHARDS: usize = 1 << SHARD_BITS; + +/// An array of cache-line aligned inner locked structures with convenience methods. +#[derive(Clone)] +pub struct Sharded<T> { + shards: [CacheAligned<Lock<T>>; SHARDS], +} + +impl<T: Default> Default for Sharded<T> { + #[inline] + fn default() -> Self { + Self::new(T::default) + } +} + +impl<T> Sharded<T> { + #[inline] + pub fn new(mut value: impl FnMut() -> T) -> Self { + Sharded { shards: [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))) } + } + + /// The shard is selected by hashing `val` with `FxHasher`. + #[inline] + pub fn get_shard_by_value<K: Hash + ?Sized>(&self, val: &K) -> &Lock<T> { + if SHARDS == 1 { &self.shards[0].0 } else { self.get_shard_by_hash(make_hash(val)) } + } + + #[inline] + pub fn get_shard_by_hash(&self, hash: u64) -> &Lock<T> { + &self.shards[get_shard_index_by_hash(hash)].0 + } + + #[inline] + pub fn get_shard_by_index(&self, i: usize) -> &Lock<T> { + &self.shards[i].0 + } + + pub fn lock_shards(&self) -> Vec<LockGuard<'_, T>> { + (0..SHARDS).map(|i| self.shards[i].0.lock()).collect() + } + + pub fn try_lock_shards(&self) -> Option<Vec<LockGuard<'_, T>>> { + (0..SHARDS).map(|i| self.shards[i].0.try_lock()).collect() + } +} + +pub type ShardedHashMap<K, V> = Sharded<FxHashMap<K, V>>; + +impl<K: Eq, V> ShardedHashMap<K, V> { + pub fn len(&self) -> usize { + self.lock_shards().iter().map(|shard| shard.len()).sum() + } +} + +impl<K: Eq + Hash + Copy> ShardedHashMap<K, ()> { + #[inline] + pub fn intern_ref<Q: ?Sized>(&self, value: &Q, make: impl FnOnce() -> K) -> K + where + K: Borrow<Q>, + Q: Hash + Eq, + { + let hash = make_hash(value); + let mut shard = self.get_shard_by_hash(hash).lock(); + let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, value); + + match entry { + RawEntryMut::Occupied(e) => *e.key(), + RawEntryMut::Vacant(e) => { + let v = make(); + e.insert_hashed_nocheck(hash, v, ()); + v + } + } + } + + #[inline] + pub fn intern<Q>(&self, value: Q, make: impl FnOnce(Q) -> K) -> K + where + K: Borrow<Q>, + Q: Hash + Eq, + { + let hash = make_hash(&value); + let mut shard = self.get_shard_by_hash(hash).lock(); + let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, &value); + + match entry { + RawEntryMut::Occupied(e) => *e.key(), + RawEntryMut::Vacant(e) => { + let v = make(value); + e.insert_hashed_nocheck(hash, v, ()); + v + } + } + } +} + +pub trait IntoPointer { + /// Returns a pointer which outlives `self`. + fn into_pointer(&self) -> *const (); +} + +impl<K: Eq + Hash + Copy + IntoPointer> ShardedHashMap<K, ()> { + pub fn contains_pointer_to<T: Hash + IntoPointer>(&self, value: &T) -> bool { + let hash = make_hash(&value); + let shard = self.get_shard_by_hash(hash).lock(); + let value = value.into_pointer(); + shard.raw_entry().from_hash(hash, |entry| entry.into_pointer() == value).is_some() + } +} + +#[inline] +pub fn make_hash<K: Hash + ?Sized>(val: &K) -> u64 { + let mut state = FxHasher::default(); + val.hash(&mut state); + state.finish() +} + +/// Get a shard with a pre-computed hash value. If `get_shard_by_value` is +/// ever used in combination with `get_shard_by_hash` on a single `Sharded` +/// instance, then `hash` must be computed with `FxHasher`. Otherwise, +/// `hash` can be computed with any hasher, so long as that hasher is used +/// consistently for each `Sharded` instance. +#[inline] +pub fn get_shard_index_by_hash(hash: u64) -> usize { + let hash_len = mem::size_of::<usize>(); + // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits. + // hashbrown also uses the lowest bits, so we can't use those + let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize; + bits % SHARDS +} diff --git a/compiler/rustc_data_structures/src/sip128.rs b/compiler/rustc_data_structures/src/sip128.rs new file mode 100644 index 000000000..90793a97e --- /dev/null +++ b/compiler/rustc_data_structures/src/sip128.rs @@ -0,0 +1,496 @@ +//! This is a copy of `core::hash::sip` adapted to providing 128 bit hashes. + +use std::hash::Hasher; +use std::mem::{self, MaybeUninit}; +use std::ptr; + +#[cfg(test)] +mod tests; + +// The SipHash algorithm operates on 8-byte chunks. +const ELEM_SIZE: usize = mem::size_of::<u64>(); + +// Size of the buffer in number of elements, not including the spill. +// +// The selection of this size was guided by rustc-perf benchmark comparisons of +// different buffer sizes. It should be periodically reevaluated as the compiler +// implementation and input characteristics change. +// +// Using the same-sized buffer for everything we hash is a performance versus +// complexity tradeoff. The ideal buffer size, and whether buffering should even +// be used, depends on what is being hashed. It may be worth it to size the +// buffer appropriately (perhaps by making SipHasher128 generic over the buffer +// size) or disable buffering depending on what is being hashed. But at this +// time, we use the same buffer size for everything. +const BUFFER_CAPACITY: usize = 8; + +// Size of the buffer in bytes, not including the spill. +const BUFFER_SIZE: usize = BUFFER_CAPACITY * ELEM_SIZE; + +// Size of the buffer in number of elements, including the spill. +const BUFFER_WITH_SPILL_CAPACITY: usize = BUFFER_CAPACITY + 1; + +// Size of the buffer in bytes, including the spill. +const BUFFER_WITH_SPILL_SIZE: usize = BUFFER_WITH_SPILL_CAPACITY * ELEM_SIZE; + +// Index of the spill element in the buffer. +const BUFFER_SPILL_INDEX: usize = BUFFER_WITH_SPILL_CAPACITY - 1; + +#[derive(Debug, Clone)] +#[repr(C)] +pub struct SipHasher128 { + // The access pattern during hashing consists of accesses to `nbuf` and + // `buf` until the buffer is full, followed by accesses to `state` and + // `processed`, and then repetition of that pattern until hashing is done. + // This is the basis for the ordering of fields below. However, in practice + // the cache miss-rate for data access is extremely low regardless of order. + nbuf: usize, // how many bytes in buf are valid + buf: [MaybeUninit<u64>; BUFFER_WITH_SPILL_CAPACITY], // unprocessed bytes le + state: State, // hash State + processed: usize, // how many bytes we've processed +} + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +struct State { + // v0, v2 and v1, v3 show up in pairs in the algorithm, + // and simd implementations of SipHash will use vectors + // of v02 and v13. By placing them in this order in the struct, + // the compiler can pick up on just a few simd optimizations by itself. + v0: u64, + v2: u64, + v1: u64, + v3: u64, +} + +macro_rules! compress { + ($state:expr) => {{ compress!($state.v0, $state.v1, $state.v2, $state.v3) }}; + ($v0:expr, $v1:expr, $v2:expr, $v3:expr) => {{ + $v0 = $v0.wrapping_add($v1); + $v1 = $v1.rotate_left(13); + $v1 ^= $v0; + $v0 = $v0.rotate_left(32); + $v2 = $v2.wrapping_add($v3); + $v3 = $v3.rotate_left(16); + $v3 ^= $v2; + $v0 = $v0.wrapping_add($v3); + $v3 = $v3.rotate_left(21); + $v3 ^= $v0; + $v2 = $v2.wrapping_add($v1); + $v1 = $v1.rotate_left(17); + $v1 ^= $v2; + $v2 = $v2.rotate_left(32); + }}; +} + +// Copies up to 8 bytes from source to destination. This performs better than +// `ptr::copy_nonoverlapping` on microbenchmarks and may perform better on real +// workloads since all of the copies have fixed sizes and avoid calling memcpy. +// +// This is specifically designed for copies of up to 8 bytes, because that's the +// maximum of number bytes needed to fill an 8-byte-sized element on which +// SipHash operates. Note that for variable-sized copies which are known to be +// less than 8 bytes, this function will perform more work than necessary unless +// the compiler is able to optimize the extra work away. +#[inline] +unsafe fn copy_nonoverlapping_small(src: *const u8, dst: *mut u8, count: usize) { + debug_assert!(count <= 8); + + if count == 8 { + ptr::copy_nonoverlapping(src, dst, 8); + return; + } + + let mut i = 0; + if i + 3 < count { + ptr::copy_nonoverlapping(src.add(i), dst.add(i), 4); + i += 4; + } + + if i + 1 < count { + ptr::copy_nonoverlapping(src.add(i), dst.add(i), 2); + i += 2 + } + + if i < count { + *dst.add(i) = *src.add(i); + i += 1; + } + + debug_assert_eq!(i, count); +} + +// # Implementation +// +// This implementation uses buffering to reduce the hashing cost for inputs +// consisting of many small integers. Buffering simplifies the integration of +// integer input--the integer write function typically just appends to the +// buffer with a statically sized write, updates metadata, and returns. +// +// Buffering also prevents alternating between writes that do and do not trigger +// the hashing process. Only when the entire buffer is full do we transition +// into hashing. This allows us to keep the hash state in registers for longer, +// instead of loading and storing it before and after processing each element. +// +// When a write fills the buffer, a buffer processing function is invoked to +// hash all of the buffered input. The buffer processing functions are marked +// `#[inline(never)]` so that they aren't inlined into the append functions, +// which ensures the more frequently called append functions remain inlineable +// and don't include register pushing/popping that would only be made necessary +// by inclusion of the complex buffer processing path which uses those +// registers. +// +// The buffer includes a "spill"--an extra element at the end--which simplifies +// the integer write buffer processing path. The value that fills the buffer can +// be written with a statically sized write that may spill over into the spill. +// After the buffer is processed, the part of the value that spilled over can be +// written from the spill to the beginning of the buffer with another statically +// sized write. This write may copy more bytes than actually spilled over, but +// we maintain the metadata such that any extra copied bytes will be ignored by +// subsequent processing. Due to the static sizes, this scheme performs better +// than copying the exact number of bytes needed into the end and beginning of +// the buffer. +// +// The buffer is uninitialized, which improves performance, but may preclude +// efficient implementation of alternative approaches. The improvement is not so +// large that an alternative approach should be disregarded because it cannot be +// efficiently implemented with an uninitialized buffer. On the other hand, an +// uninitialized buffer may become more important should a larger one be used. +// +// # Platform Dependence +// +// The SipHash algorithm operates on byte sequences. It parses the input stream +// as 8-byte little-endian integers. Therefore, given the same byte sequence, it +// produces the same result on big- and little-endian hardware. +// +// However, the Hasher trait has methods which operate on multi-byte integers. +// How they are converted into byte sequences can be endian-dependent (by using +// native byte order) or independent (by consistently using either LE or BE byte +// order). It can also be `isize` and `usize` size dependent (by using the +// native size), or independent (by converting to a common size), supposing the +// values can be represented in 32 bits. +// +// In order to make `SipHasher128` consistent with `SipHasher` in libstd, we +// choose to do the integer to byte sequence conversion in the platform- +// dependent way. Clients can achieve platform-independent hashing by widening +// `isize` and `usize` integers to 64 bits on 32-bit systems and byte-swapping +// integers on big-endian systems before passing them to the writing functions. +// This causes the input byte sequence to look identical on big- and little- +// endian systems (supposing `isize` and `usize` values can be represented in 32 +// bits), which ensures platform-independent results. +impl SipHasher128 { + #[inline] + pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher128 { + let mut hasher = SipHasher128 { + nbuf: 0, + buf: MaybeUninit::uninit_array(), + state: State { + v0: key0 ^ 0x736f6d6570736575, + // The XOR with 0xee is only done on 128-bit algorithm version. + v1: key1 ^ (0x646f72616e646f6d ^ 0xee), + v2: key0 ^ 0x6c7967656e657261, + v3: key1 ^ 0x7465646279746573, + }, + processed: 0, + }; + + unsafe { + // Initialize spill because we read from it in `short_write_process_buffer`. + *hasher.buf.get_unchecked_mut(BUFFER_SPILL_INDEX) = MaybeUninit::zeroed(); + } + + hasher + } + + #[inline] + pub fn short_write<const LEN: usize>(&mut self, bytes: [u8; LEN]) { + let nbuf = self.nbuf; + debug_assert!(LEN <= 8); + debug_assert!(nbuf < BUFFER_SIZE); + debug_assert!(nbuf + LEN < BUFFER_WITH_SPILL_SIZE); + + if nbuf + LEN < BUFFER_SIZE { + unsafe { + // The memcpy call is optimized away because the size is known. + let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf); + ptr::copy_nonoverlapping(bytes.as_ptr(), dst, LEN); + } + + self.nbuf = nbuf + LEN; + + return; + } + + unsafe { self.short_write_process_buffer(bytes) } + } + + // A specialized write function for values with size <= 8 that should only + // be called when the write would cause the buffer to fill. + // + // SAFETY: the write of `x` into `self.buf` starting at byte offset + // `self.nbuf` must cause `self.buf` to become fully initialized (and not + // overflow) if it wasn't already. + #[inline(never)] + unsafe fn short_write_process_buffer<const LEN: usize>(&mut self, bytes: [u8; LEN]) { + let nbuf = self.nbuf; + debug_assert!(LEN <= 8); + debug_assert!(nbuf < BUFFER_SIZE); + debug_assert!(nbuf + LEN >= BUFFER_SIZE); + debug_assert!(nbuf + LEN < BUFFER_WITH_SPILL_SIZE); + + // Copy first part of input into end of buffer, possibly into spill + // element. The memcpy call is optimized away because the size is known. + let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf); + ptr::copy_nonoverlapping(bytes.as_ptr(), dst, LEN); + + // Process buffer. + for i in 0..BUFFER_CAPACITY { + let elem = self.buf.get_unchecked(i).assume_init().to_le(); + self.state.v3 ^= elem; + Sip24Rounds::c_rounds(&mut self.state); + self.state.v0 ^= elem; + } + + // Copy remaining input into start of buffer by copying LEN - 1 + // elements from spill (at most LEN - 1 bytes could have overflowed + // into the spill). The memcpy call is optimized away because the size + // is known. And the whole copy is optimized away for LEN == 1. + let dst = self.buf.as_mut_ptr() as *mut u8; + let src = self.buf.get_unchecked(BUFFER_SPILL_INDEX) as *const _ as *const u8; + ptr::copy_nonoverlapping(src, dst, LEN - 1); + + // This function should only be called when the write fills the buffer. + // Therefore, when LEN == 1, the new `self.nbuf` must be zero. + // LEN is statically known, so the branch is optimized away. + self.nbuf = if LEN == 1 { 0 } else { nbuf + LEN - BUFFER_SIZE }; + self.processed += BUFFER_SIZE; + } + + // A write function for byte slices. + #[inline] + fn slice_write(&mut self, msg: &[u8]) { + let length = msg.len(); + let nbuf = self.nbuf; + debug_assert!(nbuf < BUFFER_SIZE); + + if nbuf + length < BUFFER_SIZE { + unsafe { + let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf); + + if length <= 8 { + copy_nonoverlapping_small(msg.as_ptr(), dst, length); + } else { + // This memcpy is *not* optimized away. + ptr::copy_nonoverlapping(msg.as_ptr(), dst, length); + } + } + + self.nbuf = nbuf + length; + + return; + } + + unsafe { self.slice_write_process_buffer(msg) } + } + + // A write function for byte slices that should only be called when the + // write would cause the buffer to fill. + // + // SAFETY: `self.buf` must be initialized up to the byte offset `self.nbuf`, + // and `msg` must contain enough bytes to initialize the rest of the element + // containing the byte offset `self.nbuf`. + #[inline(never)] + unsafe fn slice_write_process_buffer(&mut self, msg: &[u8]) { + let length = msg.len(); + let nbuf = self.nbuf; + debug_assert!(nbuf < BUFFER_SIZE); + debug_assert!(nbuf + length >= BUFFER_SIZE); + + // Always copy first part of input into current element of buffer. + // This function should only be called when the write fills the buffer, + // so we know that there is enough input to fill the current element. + let valid_in_elem = nbuf % ELEM_SIZE; + let needed_in_elem = ELEM_SIZE - valid_in_elem; + + let src = msg.as_ptr(); + let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf); + copy_nonoverlapping_small(src, dst, needed_in_elem); + + // Process buffer. + + // Using `nbuf / ELEM_SIZE + 1` rather than `(nbuf + needed_in_elem) / + // ELEM_SIZE` to show the compiler that this loop's upper bound is > 0. + // We know that is true, because last step ensured we have a full + // element in the buffer. + let last = nbuf / ELEM_SIZE + 1; + + for i in 0..last { + let elem = self.buf.get_unchecked(i).assume_init().to_le(); + self.state.v3 ^= elem; + Sip24Rounds::c_rounds(&mut self.state); + self.state.v0 ^= elem; + } + + // Process the remaining element-sized chunks of input. + let mut processed = needed_in_elem; + let input_left = length - processed; + let elems_left = input_left / ELEM_SIZE; + let extra_bytes_left = input_left % ELEM_SIZE; + + for _ in 0..elems_left { + let elem = (msg.as_ptr().add(processed) as *const u64).read_unaligned().to_le(); + self.state.v3 ^= elem; + Sip24Rounds::c_rounds(&mut self.state); + self.state.v0 ^= elem; + processed += ELEM_SIZE; + } + + // Copy remaining input into start of buffer. + let src = msg.as_ptr().add(processed); + let dst = self.buf.as_mut_ptr() as *mut u8; + copy_nonoverlapping_small(src, dst, extra_bytes_left); + + self.nbuf = extra_bytes_left; + self.processed += nbuf + processed; + } + + #[inline] + pub fn finish128(mut self) -> (u64, u64) { + debug_assert!(self.nbuf < BUFFER_SIZE); + + // Process full elements in buffer. + let last = self.nbuf / ELEM_SIZE; + + // Since we're consuming self, avoid updating members for a potential + // performance gain. + let mut state = self.state; + + for i in 0..last { + let elem = unsafe { self.buf.get_unchecked(i).assume_init().to_le() }; + state.v3 ^= elem; + Sip24Rounds::c_rounds(&mut state); + state.v0 ^= elem; + } + + // Get remaining partial element. + let elem = if self.nbuf % ELEM_SIZE != 0 { + unsafe { + // Ensure element is initialized by writing zero bytes. At most + // `ELEM_SIZE - 1` are required given the above check. It's safe + // to write this many because we have the spill and we maintain + // `self.nbuf` such that this write will start before the spill. + let dst = (self.buf.as_mut_ptr() as *mut u8).add(self.nbuf); + ptr::write_bytes(dst, 0, ELEM_SIZE - 1); + self.buf.get_unchecked(last).assume_init().to_le() + } + } else { + 0 + }; + + // Finalize the hash. + let length = self.processed + self.nbuf; + let b: u64 = ((length as u64 & 0xff) << 56) | elem; + + state.v3 ^= b; + Sip24Rounds::c_rounds(&mut state); + state.v0 ^= b; + + state.v2 ^= 0xee; + Sip24Rounds::d_rounds(&mut state); + let _0 = state.v0 ^ state.v1 ^ state.v2 ^ state.v3; + + state.v1 ^= 0xdd; + Sip24Rounds::d_rounds(&mut state); + let _1 = state.v0 ^ state.v1 ^ state.v2 ^ state.v3; + + (_0, _1) + } +} + +impl Hasher for SipHasher128 { + #[inline] + fn write_u8(&mut self, i: u8) { + self.short_write(i.to_ne_bytes()); + } + + #[inline] + fn write_u16(&mut self, i: u16) { + self.short_write(i.to_ne_bytes()); + } + + #[inline] + fn write_u32(&mut self, i: u32) { + self.short_write(i.to_ne_bytes()); + } + + #[inline] + fn write_u64(&mut self, i: u64) { + self.short_write(i.to_ne_bytes()); + } + + #[inline] + fn write_usize(&mut self, i: usize) { + self.short_write(i.to_ne_bytes()); + } + + #[inline] + fn write_i8(&mut self, i: i8) { + self.short_write((i as u8).to_ne_bytes()); + } + + #[inline] + fn write_i16(&mut self, i: i16) { + self.short_write((i as u16).to_ne_bytes()); + } + + #[inline] + fn write_i32(&mut self, i: i32) { + self.short_write((i as u32).to_ne_bytes()); + } + + #[inline] + fn write_i64(&mut self, i: i64) { + self.short_write((i as u64).to_ne_bytes()); + } + + #[inline] + fn write_isize(&mut self, i: isize) { + self.short_write((i as usize).to_ne_bytes()); + } + + #[inline] + fn write(&mut self, msg: &[u8]) { + self.slice_write(msg); + } + + #[inline] + fn write_str(&mut self, s: &str) { + // This hasher works byte-wise, and `0xFF` cannot show up in a `str`, + // so just hashing the one extra byte is enough to be prefix-free. + self.write(s.as_bytes()); + self.write_u8(0xFF); + } + + fn finish(&self) -> u64 { + panic!("SipHasher128 cannot provide valid 64 bit hashes") + } +} + +#[derive(Debug, Clone, Default)] +struct Sip24Rounds; + +impl Sip24Rounds { + #[inline] + fn c_rounds(state: &mut State) { + compress!(state); + compress!(state); + } + + #[inline] + fn d_rounds(state: &mut State) { + compress!(state); + compress!(state); + compress!(state); + compress!(state); + } +} diff --git a/compiler/rustc_data_structures/src/sip128/tests.rs b/compiler/rustc_data_structures/src/sip128/tests.rs new file mode 100644 index 000000000..5fe967c41 --- /dev/null +++ b/compiler/rustc_data_structures/src/sip128/tests.rs @@ -0,0 +1,497 @@ +use super::*; + +use std::hash::{Hash, Hasher}; + +// Hash just the bytes of the slice, without length prefix +struct Bytes<'a>(&'a [u8]); + +impl<'a> Hash for Bytes<'a> { + #[allow(unused_must_use)] + fn hash<H: Hasher>(&self, state: &mut H) { + for byte in self.0 { + state.write_u8(*byte); + } + } +} + +fn hash_with<T: Hash>(mut st: SipHasher128, x: &T) -> (u64, u64) { + x.hash(&mut st); + st.finish128() +} + +fn hash<T: Hash>(x: &T) -> (u64, u64) { + hash_with(SipHasher128::new_with_keys(0, 0), x) +} + +const TEST_VECTOR: [[u8; 16]; 64] = [ + [ + 0xa3, 0x81, 0x7f, 0x04, 0xba, 0x25, 0xa8, 0xe6, 0x6d, 0xf6, 0x72, 0x14, 0xc7, 0x55, 0x02, + 0x93, + ], + [ + 0xda, 0x87, 0xc1, 0xd8, 0x6b, 0x99, 0xaf, 0x44, 0x34, 0x76, 0x59, 0x11, 0x9b, 0x22, 0xfc, + 0x45, + ], + [ + 0x81, 0x77, 0x22, 0x8d, 0xa4, 0xa4, 0x5d, 0xc7, 0xfc, 0xa3, 0x8b, 0xde, 0xf6, 0x0a, 0xff, + 0xe4, + ], + [ + 0x9c, 0x70, 0xb6, 0x0c, 0x52, 0x67, 0xa9, 0x4e, 0x5f, 0x33, 0xb6, 0xb0, 0x29, 0x85, 0xed, + 0x51, + ], + [ + 0xf8, 0x81, 0x64, 0xc1, 0x2d, 0x9c, 0x8f, 0xaf, 0x7d, 0x0f, 0x6e, 0x7c, 0x7b, 0xcd, 0x55, + 0x79, + ], + [ + 0x13, 0x68, 0x87, 0x59, 0x80, 0x77, 0x6f, 0x88, 0x54, 0x52, 0x7a, 0x07, 0x69, 0x0e, 0x96, + 0x27, + ], + [ + 0x14, 0xee, 0xca, 0x33, 0x8b, 0x20, 0x86, 0x13, 0x48, 0x5e, 0xa0, 0x30, 0x8f, 0xd7, 0xa1, + 0x5e, + ], + [ + 0xa1, 0xf1, 0xeb, 0xbe, 0xd8, 0xdb, 0xc1, 0x53, 0xc0, 0xb8, 0x4a, 0xa6, 0x1f, 0xf0, 0x82, + 0x39, + ], + [ + 0x3b, 0x62, 0xa9, 0xba, 0x62, 0x58, 0xf5, 0x61, 0x0f, 0x83, 0xe2, 0x64, 0xf3, 0x14, 0x97, + 0xb4, + ], + [ + 0x26, 0x44, 0x99, 0x06, 0x0a, 0xd9, 0xba, 0xab, 0xc4, 0x7f, 0x8b, 0x02, 0xbb, 0x6d, 0x71, + 0xed, + ], + [ + 0x00, 0x11, 0x0d, 0xc3, 0x78, 0x14, 0x69, 0x56, 0xc9, 0x54, 0x47, 0xd3, 0xf3, 0xd0, 0xfb, + 0xba, + ], + [ + 0x01, 0x51, 0xc5, 0x68, 0x38, 0x6b, 0x66, 0x77, 0xa2, 0xb4, 0xdc, 0x6f, 0x81, 0xe5, 0xdc, + 0x18, + ], + [ + 0xd6, 0x26, 0xb2, 0x66, 0x90, 0x5e, 0xf3, 0x58, 0x82, 0x63, 0x4d, 0xf6, 0x85, 0x32, 0xc1, + 0x25, + ], + [ + 0x98, 0x69, 0xe2, 0x47, 0xe9, 0xc0, 0x8b, 0x10, 0xd0, 0x29, 0x93, 0x4f, 0xc4, 0xb9, 0x52, + 0xf7, + ], + [ + 0x31, 0xfc, 0xef, 0xac, 0x66, 0xd7, 0xde, 0x9c, 0x7e, 0xc7, 0x48, 0x5f, 0xe4, 0x49, 0x49, + 0x02, + ], + [ + 0x54, 0x93, 0xe9, 0x99, 0x33, 0xb0, 0xa8, 0x11, 0x7e, 0x08, 0xec, 0x0f, 0x97, 0xcf, 0xc3, + 0xd9, + ], + [ + 0x6e, 0xe2, 0xa4, 0xca, 0x67, 0xb0, 0x54, 0xbb, 0xfd, 0x33, 0x15, 0xbf, 0x85, 0x23, 0x05, + 0x77, + ], + [ + 0x47, 0x3d, 0x06, 0xe8, 0x73, 0x8d, 0xb8, 0x98, 0x54, 0xc0, 0x66, 0xc4, 0x7a, 0xe4, 0x77, + 0x40, + ], + [ + 0xa4, 0x26, 0xe5, 0xe4, 0x23, 0xbf, 0x48, 0x85, 0x29, 0x4d, 0xa4, 0x81, 0xfe, 0xae, 0xf7, + 0x23, + ], + [ + 0x78, 0x01, 0x77, 0x31, 0xcf, 0x65, 0xfa, 0xb0, 0x74, 0xd5, 0x20, 0x89, 0x52, 0x51, 0x2e, + 0xb1, + ], + [ + 0x9e, 0x25, 0xfc, 0x83, 0x3f, 0x22, 0x90, 0x73, 0x3e, 0x93, 0x44, 0xa5, 0xe8, 0x38, 0x39, + 0xeb, + ], + [ + 0x56, 0x8e, 0x49, 0x5a, 0xbe, 0x52, 0x5a, 0x21, 0x8a, 0x22, 0x14, 0xcd, 0x3e, 0x07, 0x1d, + 0x12, + ], + [ + 0x4a, 0x29, 0xb5, 0x45, 0x52, 0xd1, 0x6b, 0x9a, 0x46, 0x9c, 0x10, 0x52, 0x8e, 0xff, 0x0a, + 0xae, + ], + [ + 0xc9, 0xd1, 0x84, 0xdd, 0xd5, 0xa9, 0xf5, 0xe0, 0xcf, 0x8c, 0xe2, 0x9a, 0x9a, 0xbf, 0x69, + 0x1c, + ], + [ + 0x2d, 0xb4, 0x79, 0xae, 0x78, 0xbd, 0x50, 0xd8, 0x88, 0x2a, 0x8a, 0x17, 0x8a, 0x61, 0x32, + 0xad, + ], + [ + 0x8e, 0xce, 0x5f, 0x04, 0x2d, 0x5e, 0x44, 0x7b, 0x50, 0x51, 0xb9, 0xea, 0xcb, 0x8d, 0x8f, + 0x6f, + ], + [ + 0x9c, 0x0b, 0x53, 0xb4, 0xb3, 0xc3, 0x07, 0xe8, 0x7e, 0xae, 0xe0, 0x86, 0x78, 0x14, 0x1f, + 0x66, + ], + [ + 0xab, 0xf2, 0x48, 0xaf, 0x69, 0xa6, 0xea, 0xe4, 0xbf, 0xd3, 0xeb, 0x2f, 0x12, 0x9e, 0xeb, + 0x94, + ], + [ + 0x06, 0x64, 0xda, 0x16, 0x68, 0x57, 0x4b, 0x88, 0xb9, 0x35, 0xf3, 0x02, 0x73, 0x58, 0xae, + 0xf4, + ], + [ + 0xaa, 0x4b, 0x9d, 0xc4, 0xbf, 0x33, 0x7d, 0xe9, 0x0c, 0xd4, 0xfd, 0x3c, 0x46, 0x7c, 0x6a, + 0xb7, + ], + [ + 0xea, 0x5c, 0x7f, 0x47, 0x1f, 0xaf, 0x6b, 0xde, 0x2b, 0x1a, 0xd7, 0xd4, 0x68, 0x6d, 0x22, + 0x87, + ], + [ + 0x29, 0x39, 0xb0, 0x18, 0x32, 0x23, 0xfa, 0xfc, 0x17, 0x23, 0xde, 0x4f, 0x52, 0xc4, 0x3d, + 0x35, + ], + [ + 0x7c, 0x39, 0x56, 0xca, 0x5e, 0xea, 0xfc, 0x3e, 0x36, 0x3e, 0x9d, 0x55, 0x65, 0x46, 0xeb, + 0x68, + ], + [ + 0x77, 0xc6, 0x07, 0x71, 0x46, 0xf0, 0x1c, 0x32, 0xb6, 0xb6, 0x9d, 0x5f, 0x4e, 0xa9, 0xff, + 0xcf, + ], + [ + 0x37, 0xa6, 0x98, 0x6c, 0xb8, 0x84, 0x7e, 0xdf, 0x09, 0x25, 0xf0, 0xf1, 0x30, 0x9b, 0x54, + 0xde, + ], + [ + 0xa7, 0x05, 0xf0, 0xe6, 0x9d, 0xa9, 0xa8, 0xf9, 0x07, 0x24, 0x1a, 0x2e, 0x92, 0x3c, 0x8c, + 0xc8, + ], + [ + 0x3d, 0xc4, 0x7d, 0x1f, 0x29, 0xc4, 0x48, 0x46, 0x1e, 0x9e, 0x76, 0xed, 0x90, 0x4f, 0x67, + 0x11, + ], + [ + 0x0d, 0x62, 0xbf, 0x01, 0xe6, 0xfc, 0x0e, 0x1a, 0x0d, 0x3c, 0x47, 0x51, 0xc5, 0xd3, 0x69, + 0x2b, + ], + [ + 0x8c, 0x03, 0x46, 0x8b, 0xca, 0x7c, 0x66, 0x9e, 0xe4, 0xfd, 0x5e, 0x08, 0x4b, 0xbe, 0xe7, + 0xb5, + ], + [ + 0x52, 0x8a, 0x5b, 0xb9, 0x3b, 0xaf, 0x2c, 0x9c, 0x44, 0x73, 0xcc, 0xe5, 0xd0, 0xd2, 0x2b, + 0xd9, + ], + [ + 0xdf, 0x6a, 0x30, 0x1e, 0x95, 0xc9, 0x5d, 0xad, 0x97, 0xae, 0x0c, 0xc8, 0xc6, 0x91, 0x3b, + 0xd8, + ], + [ + 0x80, 0x11, 0x89, 0x90, 0x2c, 0x85, 0x7f, 0x39, 0xe7, 0x35, 0x91, 0x28, 0x5e, 0x70, 0xb6, + 0xdb, + ], + [ + 0xe6, 0x17, 0x34, 0x6a, 0xc9, 0xc2, 0x31, 0xbb, 0x36, 0x50, 0xae, 0x34, 0xcc, 0xca, 0x0c, + 0x5b, + ], + [ + 0x27, 0xd9, 0x34, 0x37, 0xef, 0xb7, 0x21, 0xaa, 0x40, 0x18, 0x21, 0xdc, 0xec, 0x5a, 0xdf, + 0x89, + ], + [ + 0x89, 0x23, 0x7d, 0x9d, 0xed, 0x9c, 0x5e, 0x78, 0xd8, 0xb1, 0xc9, 0xb1, 0x66, 0xcc, 0x73, + 0x42, + ], + [ + 0x4a, 0x6d, 0x80, 0x91, 0xbf, 0x5e, 0x7d, 0x65, 0x11, 0x89, 0xfa, 0x94, 0xa2, 0x50, 0xb1, + 0x4c, + ], + [ + 0x0e, 0x33, 0xf9, 0x60, 0x55, 0xe7, 0xae, 0x89, 0x3f, 0xfc, 0x0e, 0x3d, 0xcf, 0x49, 0x29, + 0x02, + ], + [ + 0xe6, 0x1c, 0x43, 0x2b, 0x72, 0x0b, 0x19, 0xd1, 0x8e, 0xc8, 0xd8, 0x4b, 0xdc, 0x63, 0x15, + 0x1b, + ], + [ + 0xf7, 0xe5, 0xae, 0xf5, 0x49, 0xf7, 0x82, 0xcf, 0x37, 0x90, 0x55, 0xa6, 0x08, 0x26, 0x9b, + 0x16, + ], + [ + 0x43, 0x8d, 0x03, 0x0f, 0xd0, 0xb7, 0xa5, 0x4f, 0xa8, 0x37, 0xf2, 0xad, 0x20, 0x1a, 0x64, + 0x03, + ], + [ + 0xa5, 0x90, 0xd3, 0xee, 0x4f, 0xbf, 0x04, 0xe3, 0x24, 0x7e, 0x0d, 0x27, 0xf2, 0x86, 0x42, + 0x3f, + ], + [ + 0x5f, 0xe2, 0xc1, 0xa1, 0x72, 0xfe, 0x93, 0xc4, 0xb1, 0x5c, 0xd3, 0x7c, 0xae, 0xf9, 0xf5, + 0x38, + ], + [ + 0x2c, 0x97, 0x32, 0x5c, 0xbd, 0x06, 0xb3, 0x6e, 0xb2, 0x13, 0x3d, 0xd0, 0x8b, 0x3a, 0x01, + 0x7c, + ], + [ + 0x92, 0xc8, 0x14, 0x22, 0x7a, 0x6b, 0xca, 0x94, 0x9f, 0xf0, 0x65, 0x9f, 0x00, 0x2a, 0xd3, + 0x9e, + ], + [ + 0xdc, 0xe8, 0x50, 0x11, 0x0b, 0xd8, 0x32, 0x8c, 0xfb, 0xd5, 0x08, 0x41, 0xd6, 0x91, 0x1d, + 0x87, + ], + [ + 0x67, 0xf1, 0x49, 0x84, 0xc7, 0xda, 0x79, 0x12, 0x48, 0xe3, 0x2b, 0xb5, 0x92, 0x25, 0x83, + 0xda, + ], + [ + 0x19, 0x38, 0xf2, 0xcf, 0x72, 0xd5, 0x4e, 0xe9, 0x7e, 0x94, 0x16, 0x6f, 0xa9, 0x1d, 0x2a, + 0x36, + ], + [ + 0x74, 0x48, 0x1e, 0x96, 0x46, 0xed, 0x49, 0xfe, 0x0f, 0x62, 0x24, 0x30, 0x16, 0x04, 0x69, + 0x8e, + ], + [ + 0x57, 0xfc, 0xa5, 0xde, 0x98, 0xa9, 0xd6, 0xd8, 0x00, 0x64, 0x38, 0xd0, 0x58, 0x3d, 0x8a, + 0x1d, + ], + [ + 0x9f, 0xec, 0xde, 0x1c, 0xef, 0xdc, 0x1c, 0xbe, 0xd4, 0x76, 0x36, 0x74, 0xd9, 0x57, 0x53, + 0x59, + ], + [ + 0xe3, 0x04, 0x0c, 0x00, 0xeb, 0x28, 0xf1, 0x53, 0x66, 0xca, 0x73, 0xcb, 0xd8, 0x72, 0xe7, + 0x40, + ], + [ + 0x76, 0x97, 0x00, 0x9a, 0x6a, 0x83, 0x1d, 0xfe, 0xcc, 0xa9, 0x1c, 0x59, 0x93, 0x67, 0x0f, + 0x7a, + ], + [ + 0x58, 0x53, 0x54, 0x23, 0x21, 0xf5, 0x67, 0xa0, 0x05, 0xd5, 0x47, 0xa4, 0xf0, 0x47, 0x59, + 0xbd, + ], + [ + 0x51, 0x50, 0xd1, 0x77, 0x2f, 0x50, 0x83, 0x4a, 0x50, 0x3e, 0x06, 0x9a, 0x97, 0x3f, 0xbd, + 0x7c, + ], +]; + +// Test vector from reference implementation +#[test] +fn test_siphash_2_4_test_vector() { + let k0 = 0x_07_06_05_04_03_02_01_00; + let k1 = 0x_0f_0e_0d_0c_0b_0a_09_08; + + let mut input: Vec<u8> = Vec::new(); + + for i in 0..64 { + let out = hash_with(SipHasher128::new_with_keys(k0, k1), &Bytes(&input[..])); + let expected = ( + ((TEST_VECTOR[i][0] as u64) << 0) + | ((TEST_VECTOR[i][1] as u64) << 8) + | ((TEST_VECTOR[i][2] as u64) << 16) + | ((TEST_VECTOR[i][3] as u64) << 24) + | ((TEST_VECTOR[i][4] as u64) << 32) + | ((TEST_VECTOR[i][5] as u64) << 40) + | ((TEST_VECTOR[i][6] as u64) << 48) + | ((TEST_VECTOR[i][7] as u64) << 56), + ((TEST_VECTOR[i][8] as u64) << 0) + | ((TEST_VECTOR[i][9] as u64) << 8) + | ((TEST_VECTOR[i][10] as u64) << 16) + | ((TEST_VECTOR[i][11] as u64) << 24) + | ((TEST_VECTOR[i][12] as u64) << 32) + | ((TEST_VECTOR[i][13] as u64) << 40) + | ((TEST_VECTOR[i][14] as u64) << 48) + | ((TEST_VECTOR[i][15] as u64) << 56), + ); + + assert_eq!(out, expected); + input.push(i as u8); + } +} + +#[test] +#[cfg(target_arch = "arm")] +fn test_hash_usize() { + let val = 0xdeadbeef_deadbeef_u64; + assert!(hash(&(val as u64)) != hash(&(val as usize))); + assert_eq!(hash(&(val as u32)), hash(&(val as usize))); +} +#[test] +#[cfg(target_arch = "x86_64")] +fn test_hash_usize() { + let val = 0xdeadbeef_deadbeef_u64; + assert_eq!(hash(&(val as u64)), hash(&(val as usize))); + assert!(hash(&(val as u32)) != hash(&(val as usize))); +} +#[test] +#[cfg(target_arch = "x86")] +fn test_hash_usize() { + let val = 0xdeadbeef_deadbeef_u64; + assert!(hash(&(val as u64)) != hash(&(val as usize))); + assert_eq!(hash(&(val as u32)), hash(&(val as usize))); +} + +#[test] +fn test_hash_idempotent() { + let val64 = 0xdeadbeef_deadbeef_u64; + assert_eq!(hash(&val64), hash(&val64)); + let val32 = 0xdeadbeef_u32; + assert_eq!(hash(&val32), hash(&val32)); +} + +#[test] +fn test_hash_no_bytes_dropped_64() { + let val = 0xdeadbeef_deadbeef_u64; + + assert!(hash(&val) != hash(&zero_byte(val, 0))); + assert!(hash(&val) != hash(&zero_byte(val, 1))); + assert!(hash(&val) != hash(&zero_byte(val, 2))); + assert!(hash(&val) != hash(&zero_byte(val, 3))); + assert!(hash(&val) != hash(&zero_byte(val, 4))); + assert!(hash(&val) != hash(&zero_byte(val, 5))); + assert!(hash(&val) != hash(&zero_byte(val, 6))); + assert!(hash(&val) != hash(&zero_byte(val, 7))); + + fn zero_byte(val: u64, byte: usize) -> u64 { + assert!(byte < 8); + val & !(0xff << (byte * 8)) + } +} + +#[test] +fn test_hash_no_bytes_dropped_32() { + let val = 0xdeadbeef_u32; + + assert!(hash(&val) != hash(&zero_byte(val, 0))); + assert!(hash(&val) != hash(&zero_byte(val, 1))); + assert!(hash(&val) != hash(&zero_byte(val, 2))); + assert!(hash(&val) != hash(&zero_byte(val, 3))); + + fn zero_byte(val: u32, byte: usize) -> u32 { + assert!(byte < 4); + val & !(0xff << (byte * 8)) + } +} + +#[test] +fn test_hash_no_concat_alias() { + let s = ("aa", "bb"); + let t = ("aabb", ""); + let u = ("a", "abb"); + + assert!(s != t && t != u); + assert!(hash(&s) != hash(&t) && hash(&s) != hash(&u)); + + let u = [1, 0, 0, 0]; + let v = (&u[..1], &u[1..3], &u[3..]); + let w = (&u[..], &u[4..4], &u[4..4]); + + assert!(v != w); + assert!(hash(&v) != hash(&w)); +} + +#[test] +fn test_short_write_works() { + let test_u8 = 0xFF_u8; + let test_u16 = 0x1122_u16; + let test_u32 = 0x22334455_u32; + let test_u64 = 0x33445566_778899AA_u64; + let test_u128 = 0x11223344_55667788_99AABBCC_DDEEFF77_u128; + let test_usize = 0xD0C0B0A0_usize; + + let test_i8 = -1_i8; + let test_i16 = -2_i16; + let test_i32 = -3_i32; + let test_i64 = -4_i64; + let test_i128 = -5_i128; + let test_isize = -6_isize; + + let mut h1 = SipHasher128::new_with_keys(0, 0); + h1.write(b"bytes"); + h1.write(b"string"); + h1.write_u8(test_u8); + h1.write_u16(test_u16); + h1.write_u32(test_u32); + h1.write_u64(test_u64); + h1.write_u128(test_u128); + h1.write_usize(test_usize); + h1.write_i8(test_i8); + h1.write_i16(test_i16); + h1.write_i32(test_i32); + h1.write_i64(test_i64); + h1.write_i128(test_i128); + h1.write_isize(test_isize); + + let mut h2 = SipHasher128::new_with_keys(0, 0); + h2.write(b"bytes"); + h2.write(b"string"); + h2.write(&test_u8.to_ne_bytes()); + h2.write(&test_u16.to_ne_bytes()); + h2.write(&test_u32.to_ne_bytes()); + h2.write(&test_u64.to_ne_bytes()); + h2.write(&test_u128.to_ne_bytes()); + h2.write(&test_usize.to_ne_bytes()); + h2.write(&test_i8.to_ne_bytes()); + h2.write(&test_i16.to_ne_bytes()); + h2.write(&test_i32.to_ne_bytes()); + h2.write(&test_i64.to_ne_bytes()); + h2.write(&test_i128.to_ne_bytes()); + h2.write(&test_isize.to_ne_bytes()); + + let h1_hash = h1.finish128(); + let h2_hash = h2.finish128(); + + assert_eq!(h1_hash, h2_hash); +} + +macro_rules! test_fill_buffer { + ($type:ty, $write_method:ident) => {{ + // Test filling and overfilling the buffer from all possible offsets + // for a given integer type and its corresponding write method. + const SIZE: usize = std::mem::size_of::<$type>(); + let input = [42; BUFFER_SIZE]; + let x = 0x01234567_89ABCDEF_76543210_FEDCBA98_u128 as $type; + let x_bytes = &x.to_ne_bytes(); + + for i in 1..=SIZE { + let s = &input[..BUFFER_SIZE - i]; + + let mut h1 = SipHasher128::new_with_keys(7, 13); + h1.write(s); + h1.$write_method(x); + + let mut h2 = SipHasher128::new_with_keys(7, 13); + h2.write(s); + h2.write(x_bytes); + + let h1_hash = h1.finish128(); + let h2_hash = h2.finish128(); + + assert_eq!(h1_hash, h2_hash); + } + }}; +} + +#[test] +fn test_fill_buffer() { + test_fill_buffer!(u8, write_u8); + test_fill_buffer!(u16, write_u16); + test_fill_buffer!(u32, write_u32); + test_fill_buffer!(u64, write_u64); + test_fill_buffer!(u128, write_u128); + test_fill_buffer!(usize, write_usize); + + test_fill_buffer!(i8, write_i8); + test_fill_buffer!(i16, write_i16); + test_fill_buffer!(i32, write_i32); + test_fill_buffer!(i64, write_i64); + test_fill_buffer!(i128, write_i128); + test_fill_buffer!(isize, write_isize); +} diff --git a/compiler/rustc_data_structures/src/small_c_str.rs b/compiler/rustc_data_structures/src/small_c_str.rs new file mode 100644 index 000000000..3a8ab8ff9 --- /dev/null +++ b/compiler/rustc_data_structures/src/small_c_str.rs @@ -0,0 +1,81 @@ +use std::ffi; +use std::ops::Deref; + +use smallvec::SmallVec; + +#[cfg(test)] +mod tests; + +const SIZE: usize = 36; + +/// Like SmallVec but for C strings. +#[derive(Clone)] +pub struct SmallCStr { + data: SmallVec<[u8; SIZE]>, +} + +impl SmallCStr { + #[inline] + pub fn new(s: &str) -> SmallCStr { + let len = s.len(); + let len1 = len + 1; + let data = if len < SIZE { + let mut buf = [0; SIZE]; + buf[..len].copy_from_slice(s.as_bytes()); + SmallVec::from_buf_and_len(buf, len1) + } else { + let mut data = Vec::with_capacity(len1); + data.extend_from_slice(s.as_bytes()); + data.push(0); + SmallVec::from_vec(data) + }; + if let Err(e) = ffi::CStr::from_bytes_with_nul(&data) { + panic!("The string \"{}\" cannot be converted into a CStr: {}", s, e); + } + SmallCStr { data } + } + + #[inline] + pub fn new_with_nul(s: &str) -> SmallCStr { + let b = s.as_bytes(); + if let Err(e) = ffi::CStr::from_bytes_with_nul(b) { + panic!("The string \"{}\" cannot be converted into a CStr: {}", s, e); + } + SmallCStr { data: SmallVec::from_slice(s.as_bytes()) } + } + + #[inline] + pub fn as_c_str(&self) -> &ffi::CStr { + unsafe { ffi::CStr::from_bytes_with_nul_unchecked(&self.data) } + } + + #[inline] + pub fn len_with_nul(&self) -> usize { + self.data.len() + } + + pub fn spilled(&self) -> bool { + self.data.spilled() + } +} + +impl Deref for SmallCStr { + type Target = ffi::CStr; + + #[inline] + fn deref(&self) -> &ffi::CStr { + self.as_c_str() + } +} + +impl<'a> FromIterator<&'a str> for SmallCStr { + fn from_iter<T: IntoIterator<Item = &'a str>>(iter: T) -> Self { + let mut data = + iter.into_iter().flat_map(|s| s.as_bytes()).copied().collect::<SmallVec<_>>(); + data.push(0); + if let Err(e) = ffi::CStr::from_bytes_with_nul(&data) { + panic!("The iterator {:?} cannot be converted into a CStr: {}", data, e); + } + Self { data } + } +} diff --git a/compiler/rustc_data_structures/src/small_c_str/tests.rs b/compiler/rustc_data_structures/src/small_c_str/tests.rs new file mode 100644 index 000000000..47277604b --- /dev/null +++ b/compiler/rustc_data_structures/src/small_c_str/tests.rs @@ -0,0 +1,45 @@ +use super::*; + +#[test] +fn short() { + const TEXT: &str = "abcd"; + let reference = ffi::CString::new(TEXT.to_string()).unwrap(); + + let scs = SmallCStr::new(TEXT); + + assert_eq!(scs.len_with_nul(), TEXT.len() + 1); + assert_eq!(scs.as_c_str(), reference.as_c_str()); + assert!(!scs.spilled()); +} + +#[test] +fn empty() { + const TEXT: &str = ""; + let reference = ffi::CString::new(TEXT.to_string()).unwrap(); + + let scs = SmallCStr::new(TEXT); + + assert_eq!(scs.len_with_nul(), TEXT.len() + 1); + assert_eq!(scs.as_c_str(), reference.as_c_str()); + assert!(!scs.spilled()); +} + +#[test] +fn long() { + const TEXT: &str = "01234567890123456789012345678901234567890123456789\ + 01234567890123456789012345678901234567890123456789\ + 01234567890123456789012345678901234567890123456789"; + let reference = ffi::CString::new(TEXT.to_string()).unwrap(); + + let scs = SmallCStr::new(TEXT); + + assert_eq!(scs.len_with_nul(), TEXT.len() + 1); + assert_eq!(scs.as_c_str(), reference.as_c_str()); + assert!(scs.spilled()); +} + +#[test] +#[should_panic] +fn internal_nul() { + let _ = SmallCStr::new("abcd\0def"); +} diff --git a/compiler/rustc_data_structures/src/small_str.rs b/compiler/rustc_data_structures/src/small_str.rs new file mode 100644 index 000000000..800acb1b0 --- /dev/null +++ b/compiler/rustc_data_structures/src/small_str.rs @@ -0,0 +1,68 @@ +use smallvec::SmallVec; + +#[cfg(test)] +mod tests; + +/// Like SmallVec but for strings. +#[derive(Default)] +pub struct SmallStr<const N: usize>(SmallVec<[u8; N]>); + +impl<const N: usize> SmallStr<N> { + #[inline] + pub fn new() -> Self { + SmallStr(SmallVec::default()) + } + + #[inline] + pub fn push_str(&mut self, s: &str) { + self.0.extend_from_slice(s.as_bytes()); + } + + #[inline] + pub fn empty(&self) -> bool { + self.0.is_empty() + } + + #[inline] + pub fn spilled(&self) -> bool { + self.0.spilled() + } + + #[inline] + pub fn as_str(&self) -> &str { + unsafe { std::str::from_utf8_unchecked(self.0.as_slice()) } + } +} + +impl<const N: usize> std::ops::Deref for SmallStr<N> { + type Target = str; + + #[inline] + fn deref(&self) -> &str { + self.as_str() + } +} + +impl<const N: usize, A: AsRef<str>> FromIterator<A> for SmallStr<N> { + #[inline] + fn from_iter<T>(iter: T) -> Self + where + T: IntoIterator<Item = A>, + { + let mut s = SmallStr::default(); + s.extend(iter); + s + } +} + +impl<const N: usize, A: AsRef<str>> Extend<A> for SmallStr<N> { + #[inline] + fn extend<T>(&mut self, iter: T) + where + T: IntoIterator<Item = A>, + { + for a in iter.into_iter() { + self.push_str(a.as_ref()); + } + } +} diff --git a/compiler/rustc_data_structures/src/small_str/tests.rs b/compiler/rustc_data_structures/src/small_str/tests.rs new file mode 100644 index 000000000..7635a9b72 --- /dev/null +++ b/compiler/rustc_data_structures/src/small_str/tests.rs @@ -0,0 +1,20 @@ +use super::*; + +#[test] +fn empty() { + let s = SmallStr::<1>::new(); + assert!(s.empty()); + assert_eq!("", s.as_str()); + assert!(!s.spilled()); +} + +#[test] +fn from_iter() { + let s = ["aa", "bb", "cc"].iter().collect::<SmallStr<6>>(); + assert_eq!("aabbcc", s.as_str()); + assert!(!s.spilled()); + + let s = ["aa", "bb", "cc", "dd"].iter().collect::<SmallStr<6>>(); + assert_eq!("aabbccdd", s.as_str()); + assert!(s.spilled()); +} diff --git a/compiler/rustc_data_structures/src/snapshot_map/mod.rs b/compiler/rustc_data_structures/src/snapshot_map/mod.rs new file mode 100644 index 000000000..8a50179cd --- /dev/null +++ b/compiler/rustc_data_structures/src/snapshot_map/mod.rs @@ -0,0 +1,143 @@ +use crate::fx::FxHashMap; +use crate::undo_log::{Rollback, Snapshots, UndoLogs, VecLog}; +use std::borrow::{Borrow, BorrowMut}; +use std::hash::Hash; +use std::marker::PhantomData; +use std::ops; + +pub use crate::undo_log::Snapshot; + +#[cfg(test)] +mod tests; + +pub type SnapshotMapStorage<K, V> = SnapshotMap<K, V, FxHashMap<K, V>, ()>; +pub type SnapshotMapRef<'a, K, V, L> = SnapshotMap<K, V, &'a mut FxHashMap<K, V>, &'a mut L>; + +#[derive(Clone)] +pub struct SnapshotMap<K, V, M = FxHashMap<K, V>, L = VecLog<UndoLog<K, V>>> { + map: M, + undo_log: L, + _marker: PhantomData<(K, V)>, +} + +// HACK(eddyb) manual impl avoids `Default` bounds on `K` and `V`. +impl<K, V, M, L> Default for SnapshotMap<K, V, M, L> +where + M: Default, + L: Default, +{ + fn default() -> Self { + SnapshotMap { map: Default::default(), undo_log: Default::default(), _marker: PhantomData } + } +} + +#[derive(Clone)] +pub enum UndoLog<K, V> { + Inserted(K), + Overwrite(K, V), + Purged, +} + +impl<K, V, M, L> SnapshotMap<K, V, M, L> { + #[inline] + pub fn with_log<L2>(&mut self, undo_log: L2) -> SnapshotMap<K, V, &mut M, L2> { + SnapshotMap { map: &mut self.map, undo_log, _marker: PhantomData } + } +} + +impl<K, V, M, L> SnapshotMap<K, V, M, L> +where + K: Hash + Clone + Eq, + M: BorrowMut<FxHashMap<K, V>> + Borrow<FxHashMap<K, V>>, + L: UndoLogs<UndoLog<K, V>>, +{ + pub fn clear(&mut self) { + self.map.borrow_mut().clear(); + self.undo_log.clear(); + } + + pub fn insert(&mut self, key: K, value: V) -> bool { + match self.map.borrow_mut().insert(key.clone(), value) { + None => { + self.undo_log.push(UndoLog::Inserted(key)); + true + } + Some(old_value) => { + self.undo_log.push(UndoLog::Overwrite(key, old_value)); + false + } + } + } + + pub fn remove(&mut self, key: K) -> bool { + match self.map.borrow_mut().remove(&key) { + Some(old_value) => { + self.undo_log.push(UndoLog::Overwrite(key, old_value)); + true + } + None => false, + } + } + + pub fn get(&self, key: &K) -> Option<&V> { + self.map.borrow().get(key) + } +} + +impl<K, V> SnapshotMap<K, V> +where + K: Hash + Clone + Eq, +{ + pub fn snapshot(&mut self) -> Snapshot { + self.undo_log.start_snapshot() + } + + pub fn commit(&mut self, snapshot: Snapshot) { + self.undo_log.commit(snapshot) + } + + pub fn rollback_to(&mut self, snapshot: Snapshot) { + let map = &mut self.map; + self.undo_log.rollback_to(|| map, snapshot) + } +} + +impl<'k, K, V, M, L> ops::Index<&'k K> for SnapshotMap<K, V, M, L> +where + K: Hash + Clone + Eq, + M: Borrow<FxHashMap<K, V>>, +{ + type Output = V; + fn index(&self, key: &'k K) -> &V { + &self.map.borrow()[key] + } +} + +impl<K, V, M, L> Rollback<UndoLog<K, V>> for SnapshotMap<K, V, M, L> +where + K: Eq + Hash, + M: Rollback<UndoLog<K, V>>, +{ + fn reverse(&mut self, undo: UndoLog<K, V>) { + self.map.reverse(undo) + } +} + +impl<K, V> Rollback<UndoLog<K, V>> for FxHashMap<K, V> +where + K: Eq + Hash, +{ + fn reverse(&mut self, undo: UndoLog<K, V>) { + match undo { + UndoLog::Inserted(key) => { + self.remove(&key); + } + + UndoLog::Overwrite(key, old_value) => { + self.insert(key, old_value); + } + + UndoLog::Purged => {} + } + } +} diff --git a/compiler/rustc_data_structures/src/snapshot_map/tests.rs b/compiler/rustc_data_structures/src/snapshot_map/tests.rs new file mode 100644 index 000000000..72ca53c2b --- /dev/null +++ b/compiler/rustc_data_structures/src/snapshot_map/tests.rs @@ -0,0 +1,43 @@ +use super::SnapshotMap; + +#[test] +fn basic() { + let mut map = SnapshotMap::default(); + map.insert(22, "twenty-two"); + let snapshot = map.snapshot(); + map.insert(22, "thirty-three"); + assert_eq!(map[&22], "thirty-three"); + map.insert(44, "forty-four"); + assert_eq!(map[&44], "forty-four"); + assert_eq!(map.get(&33), None); + map.rollback_to(snapshot); + assert_eq!(map[&22], "twenty-two"); + assert_eq!(map.get(&33), None); + assert_eq!(map.get(&44), None); +} + +#[test] +#[should_panic] +fn out_of_order() { + let mut map = SnapshotMap::default(); + map.insert(22, "twenty-two"); + let snapshot1 = map.snapshot(); + map.insert(33, "thirty-three"); + let snapshot2 = map.snapshot(); + map.insert(44, "forty-four"); + map.rollback_to(snapshot1); // bogus, but accepted + map.rollback_to(snapshot2); // asserts +} + +#[test] +fn nested_commit_then_rollback() { + let mut map = SnapshotMap::default(); + map.insert(22, "twenty-two"); + let snapshot1 = map.snapshot(); + let snapshot2 = map.snapshot(); + map.insert(22, "thirty-three"); + map.commit(snapshot2); + assert_eq!(map[&22], "thirty-three"); + map.rollback_to(snapshot1); + assert_eq!(map[&22], "twenty-two"); +} diff --git a/compiler/rustc_data_structures/src/sorted_map.rs b/compiler/rustc_data_structures/src/sorted_map.rs new file mode 100644 index 000000000..9efea1228 --- /dev/null +++ b/compiler/rustc_data_structures/src/sorted_map.rs @@ -0,0 +1,302 @@ +use crate::stable_hasher::{HashStable, StableHasher}; +use std::borrow::Borrow; +use std::cmp::Ordering; +use std::iter::FromIterator; +use std::mem; +use std::ops::{Bound, Index, IndexMut, RangeBounds}; + +mod index_map; + +pub use index_map::SortedIndexMultiMap; + +/// `SortedMap` is a data structure with similar characteristics as BTreeMap but +/// slightly different trade-offs: lookup, insertion, and removal are *O*(log(*n*)) +/// and elements can be iterated in order cheaply. +/// +/// `SortedMap` can be faster than a `BTreeMap` for small sizes (<50) since it +/// stores data in a more compact way. It also supports accessing contiguous +/// ranges of elements as a slice, and slices of already sorted elements can be +/// inserted efficiently. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Encodable, Decodable)] +pub struct SortedMap<K, V> { + data: Vec<(K, V)>, +} + +impl<K, V> Default for SortedMap<K, V> { + #[inline] + fn default() -> SortedMap<K, V> { + SortedMap { data: Vec::new() } + } +} + +impl<K, V> SortedMap<K, V> { + #[inline] + pub const fn new() -> SortedMap<K, V> { + SortedMap { data: Vec::new() } + } +} + +impl<K: Ord, V> SortedMap<K, V> { + /// Construct a `SortedMap` from a presorted set of elements. This is faster + /// than creating an empty map and then inserting the elements individually. + /// + /// It is up to the caller to make sure that the elements are sorted by key + /// and that there are no duplicates. + #[inline] + pub fn from_presorted_elements(elements: Vec<(K, V)>) -> SortedMap<K, V> { + debug_assert!(elements.array_windows().all(|[fst, snd]| fst.0 < snd.0)); + + SortedMap { data: elements } + } + + #[inline] + pub fn insert(&mut self, key: K, mut value: V) -> Option<V> { + match self.lookup_index_for(&key) { + Ok(index) => { + let slot = unsafe { self.data.get_unchecked_mut(index) }; + mem::swap(&mut slot.1, &mut value); + Some(value) + } + Err(index) => { + self.data.insert(index, (key, value)); + None + } + } + } + + #[inline] + pub fn remove(&mut self, key: &K) -> Option<V> { + match self.lookup_index_for(key) { + Ok(index) => Some(self.data.remove(index).1), + Err(_) => None, + } + } + + #[inline] + pub fn get<Q>(&self, key: &Q) -> Option<&V> + where + K: Borrow<Q>, + Q: Ord + ?Sized, + { + match self.lookup_index_for(key) { + Ok(index) => unsafe { Some(&self.data.get_unchecked(index).1) }, + Err(_) => None, + } + } + + #[inline] + pub fn get_mut<Q>(&mut self, key: &Q) -> Option<&mut V> + where + K: Borrow<Q>, + Q: Ord + ?Sized, + { + match self.lookup_index_for(key) { + Ok(index) => unsafe { Some(&mut self.data.get_unchecked_mut(index).1) }, + Err(_) => None, + } + } + + #[inline] + pub fn clear(&mut self) { + self.data.clear(); + } + + /// Iterate over elements, sorted by key + #[inline] + pub fn iter(&self) -> std::slice::Iter<'_, (K, V)> { + self.data.iter() + } + + /// Iterate over the keys, sorted + #[inline] + pub fn keys(&self) -> impl Iterator<Item = &K> + ExactSizeIterator + DoubleEndedIterator { + self.data.iter().map(|&(ref k, _)| k) + } + + /// Iterate over values, sorted by key + #[inline] + pub fn values(&self) -> impl Iterator<Item = &V> + ExactSizeIterator + DoubleEndedIterator { + self.data.iter().map(|&(_, ref v)| v) + } + + #[inline] + pub fn len(&self) -> usize { + self.data.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + pub fn range<R>(&self, range: R) -> &[(K, V)] + where + R: RangeBounds<K>, + { + let (start, end) = self.range_slice_indices(range); + &self.data[start..end] + } + + #[inline] + pub fn remove_range<R>(&mut self, range: R) + where + R: RangeBounds<K>, + { + let (start, end) = self.range_slice_indices(range); + self.data.splice(start..end, std::iter::empty()); + } + + /// Mutate all keys with the given function `f`. This mutation must not + /// change the sort-order of keys. + #[inline] + pub fn offset_keys<F>(&mut self, f: F) + where + F: Fn(&mut K), + { + self.data.iter_mut().map(|&mut (ref mut k, _)| k).for_each(f); + } + + /// Inserts a presorted range of elements into the map. If the range can be + /// inserted as a whole in between to existing elements of the map, this + /// will be faster than inserting the elements individually. + /// + /// It is up to the caller to make sure that the elements are sorted by key + /// and that there are no duplicates. + #[inline] + pub fn insert_presorted(&mut self, mut elements: Vec<(K, V)>) { + if elements.is_empty() { + return; + } + + debug_assert!(elements.array_windows().all(|[fst, snd]| fst.0 < snd.0)); + + let start_index = self.lookup_index_for(&elements[0].0); + + let drain = match start_index { + Ok(index) => { + let mut drain = elements.drain(..); + self.data[index] = drain.next().unwrap(); + drain + } + Err(index) => { + if index == self.data.len() || elements.last().unwrap().0 < self.data[index].0 { + // We can copy the whole range without having to mix with + // existing elements. + self.data.splice(index..index, elements.drain(..)); + return; + } + + let mut drain = elements.drain(..); + self.data.insert(index, drain.next().unwrap()); + drain + } + }; + + // Insert the rest + for (k, v) in drain { + self.insert(k, v); + } + } + + /// Looks up the key in `self.data` via `slice::binary_search()`. + #[inline(always)] + fn lookup_index_for<Q>(&self, key: &Q) -> Result<usize, usize> + where + K: Borrow<Q>, + Q: Ord + ?Sized, + { + self.data.binary_search_by(|&(ref x, _)| x.borrow().cmp(key)) + } + + #[inline] + fn range_slice_indices<R>(&self, range: R) -> (usize, usize) + where + R: RangeBounds<K>, + { + let start = match range.start_bound() { + Bound::Included(ref k) => match self.lookup_index_for(k) { + Ok(index) | Err(index) => index, + }, + Bound::Excluded(ref k) => match self.lookup_index_for(k) { + Ok(index) => index + 1, + Err(index) => index, + }, + Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + Bound::Included(ref k) => match self.lookup_index_for(k) { + Ok(index) => index + 1, + Err(index) => index, + }, + Bound::Excluded(ref k) => match self.lookup_index_for(k) { + Ok(index) | Err(index) => index, + }, + Bound::Unbounded => self.data.len(), + }; + + (start, end) + } + + #[inline] + pub fn contains_key<Q>(&self, key: &Q) -> bool + where + K: Borrow<Q>, + Q: Ord + ?Sized, + { + self.get(key).is_some() + } +} + +impl<K: Ord, V> IntoIterator for SortedMap<K, V> { + type Item = (K, V); + type IntoIter = std::vec::IntoIter<(K, V)>; + + fn into_iter(self) -> Self::IntoIter { + self.data.into_iter() + } +} + +impl<'a, K, Q, V> Index<&'a Q> for SortedMap<K, V> +where + K: Ord + Borrow<Q>, + Q: Ord + ?Sized, +{ + type Output = V; + + fn index(&self, key: &Q) -> &Self::Output { + self.get(key).expect("no entry found for key") + } +} + +impl<'a, K, Q, V> IndexMut<&'a Q> for SortedMap<K, V> +where + K: Ord + Borrow<Q>, + Q: Ord + ?Sized, +{ + fn index_mut(&mut self, key: &Q) -> &mut Self::Output { + self.get_mut(key).expect("no entry found for key") + } +} + +impl<K: Ord, V> FromIterator<(K, V)> for SortedMap<K, V> { + fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> Self { + let mut data: Vec<(K, V)> = iter.into_iter().collect(); + + data.sort_unstable_by(|&(ref k1, _), &(ref k2, _)| k1.cmp(k2)); + data.dedup_by(|&mut (ref k1, _), &mut (ref k2, _)| k1.cmp(k2) == Ordering::Equal); + + SortedMap { data } + } +} + +impl<K: HashStable<CTX>, V: HashStable<CTX>, CTX> HashStable<CTX> for SortedMap<K, V> { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + self.data.hash_stable(ctx, hasher); + } +} + +#[cfg(test)] +mod tests; diff --git a/compiler/rustc_data_structures/src/sorted_map/index_map.rs b/compiler/rustc_data_structures/src/sorted_map/index_map.rs new file mode 100644 index 000000000..0ec32dc43 --- /dev/null +++ b/compiler/rustc_data_structures/src/sorted_map/index_map.rs @@ -0,0 +1,154 @@ +//! A variant of `SortedMap` that preserves insertion order. + +use std::hash::{Hash, Hasher}; +use std::iter::FromIterator; + +use crate::stable_hasher::{HashStable, StableHasher}; +use rustc_index::vec::{Idx, IndexVec}; + +/// An indexed multi-map that preserves insertion order while permitting both *O*(log *n*) lookup of +/// an item by key and *O*(1) lookup by index. +/// +/// This data structure is a hybrid of an [`IndexVec`] and a [`SortedMap`]. Like `IndexVec`, +/// `SortedIndexMultiMap` assigns a typed index to each item while preserving insertion order. +/// Like `SortedMap`, `SortedIndexMultiMap` has efficient lookup of items by key. However, this +/// is accomplished by sorting an array of item indices instead of the items themselves. +/// +/// Unlike `SortedMap`, this data structure can hold multiple equivalent items at once, so the +/// `get_by_key` method and its variants return an iterator instead of an `Option`. Equivalent +/// items will be yielded in insertion order. +/// +/// Unlike a general-purpose map like `BTreeSet` or `HashSet`, `SortedMap` and +/// `SortedIndexMultiMap` require *O*(*n*) time to insert a single item. This is because we may need +/// to insert into the middle of the sorted array. Users should avoid mutating this data structure +/// in-place. +/// +/// [`SortedMap`]: super::SortedMap +#[derive(Clone, Debug)] +pub struct SortedIndexMultiMap<I: Idx, K, V> { + /// The elements of the map in insertion order. + items: IndexVec<I, (K, V)>, + + /// Indices of the items in the set, sorted by the item's key. + idx_sorted_by_item_key: Vec<I>, +} + +impl<I: Idx, K: Ord, V> SortedIndexMultiMap<I, K, V> { + #[inline] + pub fn new() -> Self { + SortedIndexMultiMap { items: IndexVec::new(), idx_sorted_by_item_key: Vec::new() } + } + + #[inline] + pub fn len(&self) -> usize { + self.items.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + /// Returns an iterator over the items in the map in insertion order. + #[inline] + pub fn into_iter(self) -> impl DoubleEndedIterator<Item = (K, V)> { + self.items.into_iter() + } + + /// Returns an iterator over the items in the map in insertion order along with their indices. + #[inline] + pub fn into_iter_enumerated(self) -> impl DoubleEndedIterator<Item = (I, (K, V))> { + self.items.into_iter_enumerated() + } + + /// Returns an iterator over the items in the map in insertion order. + #[inline] + pub fn iter(&self) -> impl '_ + DoubleEndedIterator<Item = (&K, &V)> { + self.items.iter().map(|(ref k, ref v)| (k, v)) + } + + /// Returns an iterator over the items in the map in insertion order along with their indices. + #[inline] + pub fn iter_enumerated(&self) -> impl '_ + DoubleEndedIterator<Item = (I, (&K, &V))> { + self.items.iter_enumerated().map(|(i, (ref k, ref v))| (i, (k, v))) + } + + /// Returns the item in the map with the given index. + #[inline] + pub fn get(&self, idx: I) -> Option<&(K, V)> { + self.items.get(idx) + } + + /// Returns an iterator over the items in the map that are equal to `key`. + /// + /// If there are multiple items that are equivalent to `key`, they will be yielded in + /// insertion order. + #[inline] + pub fn get_by_key(&self, key: K) -> impl Iterator<Item = &V> + '_ { + self.get_by_key_enumerated(key).map(|(_, v)| v) + } + + /// Returns an iterator over the items in the map that are equal to `key` along with their + /// indices. + /// + /// If there are multiple items that are equivalent to `key`, they will be yielded in + /// insertion order. + #[inline] + pub fn get_by_key_enumerated(&self, key: K) -> impl Iterator<Item = (I, &V)> + '_ { + let lower_bound = self.idx_sorted_by_item_key.partition_point(|&i| self.items[i].0 < key); + self.idx_sorted_by_item_key[lower_bound..].iter().map_while(move |&i| { + let (k, v) = &self.items[i]; + (k == &key).then_some((i, v)) + }) + } +} + +impl<I: Idx, K: Eq, V: Eq> Eq for SortedIndexMultiMap<I, K, V> {} +impl<I: Idx, K: PartialEq, V: PartialEq> PartialEq for SortedIndexMultiMap<I, K, V> { + fn eq(&self, other: &Self) -> bool { + // No need to compare the sorted index. If the items are the same, the index will be too. + self.items == other.items + } +} + +impl<I: Idx, K, V> Hash for SortedIndexMultiMap<I, K, V> +where + K: Hash, + V: Hash, +{ + fn hash<H: Hasher>(&self, hasher: &mut H) { + self.items.hash(hasher) + } +} +impl<I: Idx, K, V, C> HashStable<C> for SortedIndexMultiMap<I, K, V> +where + K: HashStable<C>, + V: HashStable<C>, +{ + fn hash_stable(&self, ctx: &mut C, hasher: &mut StableHasher) { + self.items.hash_stable(ctx, hasher) + } +} + +impl<I: Idx, K: Ord, V> FromIterator<(K, V)> for SortedIndexMultiMap<I, K, V> { + fn from_iter<J>(iter: J) -> Self + where + J: IntoIterator<Item = (K, V)>, + { + let items = IndexVec::from_iter(iter); + let mut idx_sorted_by_item_key: Vec<_> = items.indices().collect(); + + // `sort_by_key` is stable, so insertion order is preserved for duplicate items. + idx_sorted_by_item_key.sort_by_key(|&idx| &items[idx].0); + + SortedIndexMultiMap { items, idx_sorted_by_item_key } + } +} + +impl<I: Idx, K, V> std::ops::Index<I> for SortedIndexMultiMap<I, K, V> { + type Output = V; + + fn index(&self, idx: I) -> &Self::Output { + &self.items[idx].1 + } +} diff --git a/compiler/rustc_data_structures/src/sorted_map/tests.rs b/compiler/rustc_data_structures/src/sorted_map/tests.rs new file mode 100644 index 000000000..1e977d709 --- /dev/null +++ b/compiler/rustc_data_structures/src/sorted_map/tests.rs @@ -0,0 +1,222 @@ +use super::{SortedIndexMultiMap, SortedMap}; + +#[test] +fn test_sorted_index_multi_map() { + let entries: Vec<_> = vec![(2, 0), (1, 0), (2, 1), (3, 0), (2, 2)]; + let set: SortedIndexMultiMap<usize, _, _> = entries.iter().copied().collect(); + + // Insertion order is preserved. + assert!(entries.iter().map(|(ref k, ref v)| (k, v)).eq(set.iter())); + + // Indexing + for (i, expect) in entries.iter().enumerate() { + assert_eq!(set[i], expect.1); + } + + // `get_by_key` works. + assert_eq!(set.get_by_key(3).copied().collect::<Vec<_>>(), vec![0]); + assert!(set.get_by_key(4).next().is_none()); + + // `get_by_key` returns items in insertion order. + let twos: Vec<_> = set.get_by_key_enumerated(2).collect(); + let idxs: Vec<usize> = twos.iter().map(|(i, _)| *i).collect(); + let values: Vec<usize> = twos.iter().map(|(_, &v)| v).collect(); + + assert_eq!(idxs, vec![0, 2, 4]); + assert_eq!(values, vec![0, 1, 2]); +} + +#[test] +fn test_insert_and_iter() { + let mut map = SortedMap::new(); + let mut expected = Vec::new(); + + for x in 0..100 { + assert_eq!(map.iter().cloned().collect::<Vec<_>>(), expected); + + let x = 1000 - x * 2; + map.insert(x, x); + expected.insert(0, (x, x)); + } +} + +#[test] +fn test_get_and_index() { + let mut map = SortedMap::new(); + let mut expected = Vec::new(); + + for x in 0..100 { + let x = 1000 - x; + if x & 1 == 0 { + map.insert(x, x); + } + expected.push(x); + } + + for mut x in expected { + if x & 1 == 0 { + assert_eq!(map.get(&x), Some(&x)); + assert_eq!(map.get_mut(&x), Some(&mut x)); + assert_eq!(map[&x], x); + assert_eq!(&mut map[&x], &mut x); + } else { + assert_eq!(map.get(&x), None); + assert_eq!(map.get_mut(&x), None); + } + } +} + +#[test] +fn test_range() { + let mut map = SortedMap::new(); + map.insert(1, 1); + map.insert(3, 3); + map.insert(6, 6); + map.insert(9, 9); + + let keys = |s: &[(_, _)]| s.into_iter().map(|e| e.0).collect::<Vec<u32>>(); + + for start in 0..11 { + for end in 0..11 { + if end < start { + continue; + } + + let mut expected = vec![1, 3, 6, 9]; + expected.retain(|&x| x >= start && x < end); + + assert_eq!(keys(map.range(start..end)), expected, "range = {}..{}", start, end); + } + } +} + +#[test] +fn test_offset_keys() { + let mut map = SortedMap::new(); + map.insert(1, 1); + map.insert(3, 3); + map.insert(6, 6); + + map.offset_keys(|k| *k += 1); + + let mut expected = SortedMap::new(); + expected.insert(2, 1); + expected.insert(4, 3); + expected.insert(7, 6); + + assert_eq!(map, expected); +} + +fn keys(s: SortedMap<u32, u32>) -> Vec<u32> { + s.into_iter().map(|(k, _)| k).collect::<Vec<u32>>() +} + +fn elements(s: SortedMap<u32, u32>) -> Vec<(u32, u32)> { + s.into_iter().collect::<Vec<(u32, u32)>>() +} + +#[test] +fn test_remove_range() { + let mut map = SortedMap::new(); + map.insert(1, 1); + map.insert(3, 3); + map.insert(6, 6); + map.insert(9, 9); + + for start in 0..11 { + for end in 0..11 { + if end < start { + continue; + } + + let mut expected = vec![1, 3, 6, 9]; + expected.retain(|&x| x < start || x >= end); + + let mut map = map.clone(); + map.remove_range(start..end); + + assert_eq!(keys(map), expected, "range = {}..{}", start, end); + } + } +} + +#[test] +fn test_remove() { + let mut map = SortedMap::new(); + let mut expected = Vec::new(); + + for x in 0..10 { + map.insert(x, x); + expected.push((x, x)); + } + + for x in 0..10 { + let mut map = map.clone(); + let mut expected = expected.clone(); + + assert_eq!(map.remove(&x), Some(x)); + expected.remove(x as usize); + + assert_eq!(map.iter().cloned().collect::<Vec<_>>(), expected); + } +} + +#[test] +fn test_insert_presorted_non_overlapping() { + let mut map = SortedMap::new(); + map.insert(2, 0); + map.insert(8, 0); + + map.insert_presorted(vec![(3, 0), (7, 0)]); + + let expected = vec![2, 3, 7, 8]; + assert_eq!(keys(map), expected); +} + +#[test] +fn test_insert_presorted_first_elem_equal() { + let mut map = SortedMap::new(); + map.insert(2, 2); + map.insert(8, 8); + + map.insert_presorted(vec![(2, 0), (7, 7)]); + + let expected = vec![(2, 0), (7, 7), (8, 8)]; + assert_eq!(elements(map), expected); +} + +#[test] +fn test_insert_presorted_last_elem_equal() { + let mut map = SortedMap::new(); + map.insert(2, 2); + map.insert(8, 8); + + map.insert_presorted(vec![(3, 3), (8, 0)]); + + let expected = vec![(2, 2), (3, 3), (8, 0)]; + assert_eq!(elements(map), expected); +} + +#[test] +fn test_insert_presorted_shuffle() { + let mut map = SortedMap::new(); + map.insert(2, 2); + map.insert(7, 7); + + map.insert_presorted(vec![(1, 1), (3, 3), (8, 8)]); + + let expected = vec![(1, 1), (2, 2), (3, 3), (7, 7), (8, 8)]; + assert_eq!(elements(map), expected); +} + +#[test] +fn test_insert_presorted_at_end() { + let mut map = SortedMap::new(); + map.insert(1, 1); + map.insert(2, 2); + + map.insert_presorted(vec![(3, 3), (8, 8)]); + + let expected = vec![(1, 1), (2, 2), (3, 3), (8, 8)]; + assert_eq!(elements(map), expected); +} diff --git a/compiler/rustc_data_structures/src/sso/either_iter.rs b/compiler/rustc_data_structures/src/sso/either_iter.rs new file mode 100644 index 000000000..131eeef45 --- /dev/null +++ b/compiler/rustc_data_structures/src/sso/either_iter.rs @@ -0,0 +1,75 @@ +use std::fmt; +use std::iter::ExactSizeIterator; +use std::iter::FusedIterator; +use std::iter::Iterator; + +/// Iterator which may contain instance of +/// one of two specific implementations. +/// +/// Note: For most methods providing custom +/// implementation may marginally +/// improve performance by avoiding +/// doing Left/Right match on every step +/// and doing it only once instead. +#[derive(Clone)] +pub enum EitherIter<L, R> { + Left(L), + Right(R), +} + +impl<L, R> Iterator for EitherIter<L, R> +where + L: Iterator, + R: Iterator<Item = L::Item>, +{ + type Item = L::Item; + + fn next(&mut self) -> Option<Self::Item> { + match self { + EitherIter::Left(l) => l.next(), + EitherIter::Right(r) => r.next(), + } + } + + fn size_hint(&self) -> (usize, Option<usize>) { + match self { + EitherIter::Left(l) => l.size_hint(), + EitherIter::Right(r) => r.size_hint(), + } + } +} + +impl<L, R> ExactSizeIterator for EitherIter<L, R> +where + L: ExactSizeIterator, + R: ExactSizeIterator, + EitherIter<L, R>: Iterator, +{ + fn len(&self) -> usize { + match self { + EitherIter::Left(l) => l.len(), + EitherIter::Right(r) => r.len(), + } + } +} + +impl<L, R> FusedIterator for EitherIter<L, R> +where + L: FusedIterator, + R: FusedIterator, + EitherIter<L, R>: Iterator, +{ +} + +impl<L, R> fmt::Debug for EitherIter<L, R> +where + L: fmt::Debug, + R: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + EitherIter::Left(l) => l.fmt(f), + EitherIter::Right(r) => r.fmt(f), + } + } +} diff --git a/compiler/rustc_data_structures/src/sso/map.rs b/compiler/rustc_data_structures/src/sso/map.rs new file mode 100644 index 000000000..ec6a62016 --- /dev/null +++ b/compiler/rustc_data_structures/src/sso/map.rs @@ -0,0 +1,557 @@ +use super::either_iter::EitherIter; +use crate::fx::FxHashMap; +use arrayvec::ArrayVec; +use std::fmt; +use std::hash::Hash; +use std::iter::FromIterator; +use std::ops::Index; + +// For pointer-sized arguments arrays +// are faster than set/map for up to 64 +// arguments. +// +// On the other hand such a big array +// hurts cache performance, makes passing +// sso structures around very expensive. +// +// Biggest performance benefit is gained +// for reasonably small arrays that stay +// small in vast majority of cases. +// +// '8' is chosen as a sane default, to be +// reevaluated later. +const SSO_ARRAY_SIZE: usize = 8; + +/// Small-storage-optimized implementation of a map. +/// +/// Stores elements in a small array up to a certain length +/// and switches to `HashMap` when that length is exceeded. +// +// FIXME: Implements subset of HashMap API. +// +// Missing HashMap API: +// all hasher-related +// try_reserve +// shrink_to (unstable) +// drain_filter (unstable) +// into_keys/into_values (unstable) +// all raw_entry-related +// PartialEq/Eq (requires sorting the array) +// Entry::or_insert_with_key +// Vacant/Occupied entries and related +// +// FIXME: In HashMap most methods accepting key reference +// accept reference to generic `Q` where `K: Borrow<Q>`. +// +// However, using this approach in `HashMap::get` apparently +// breaks inlining and noticeably reduces performance. +// +// Performance *should* be the same given that borrow is +// a NOP in most cases, but in practice that's not the case. +// +// Further investigation is required. +// +// Affected methods: +// SsoHashMap::get +// SsoHashMap::get_mut +// SsoHashMap::get_entry +// SsoHashMap::get_key_value +// SsoHashMap::contains_key +// SsoHashMap::remove +// SsoHashMap::remove_entry +// Index::index +// SsoHashSet::take +// SsoHashSet::get +// SsoHashSet::remove +// SsoHashSet::contains + +#[derive(Clone)] +pub enum SsoHashMap<K, V> { + Array(ArrayVec<(K, V), SSO_ARRAY_SIZE>), + Map(FxHashMap<K, V>), +} + +impl<K, V> SsoHashMap<K, V> { + /// Creates an empty `SsoHashMap`. + #[inline] + pub fn new() -> Self { + SsoHashMap::Array(ArrayVec::new()) + } + + /// Creates an empty `SsoHashMap` with the specified capacity. + pub fn with_capacity(cap: usize) -> Self { + if cap <= SSO_ARRAY_SIZE { + Self::new() + } else { + SsoHashMap::Map(FxHashMap::with_capacity_and_hasher(cap, Default::default())) + } + } + + /// Clears the map, removing all key-value pairs. Keeps the allocated memory + /// for reuse. + pub fn clear(&mut self) { + match self { + SsoHashMap::Array(array) => array.clear(), + SsoHashMap::Map(map) => map.clear(), + } + } + + /// Returns the number of elements the map can hold without reallocating. + pub fn capacity(&self) -> usize { + match self { + SsoHashMap::Array(_) => SSO_ARRAY_SIZE, + SsoHashMap::Map(map) => map.capacity(), + } + } + + /// Returns the number of elements in the map. + pub fn len(&self) -> usize { + match self { + SsoHashMap::Array(array) => array.len(), + SsoHashMap::Map(map) => map.len(), + } + } + + /// Returns `true` if the map contains no elements. + pub fn is_empty(&self) -> bool { + match self { + SsoHashMap::Array(array) => array.is_empty(), + SsoHashMap::Map(map) => map.is_empty(), + } + } + + /// An iterator visiting all key-value pairs in arbitrary order. + /// The iterator element type is `(&'a K, &'a V)`. + #[inline] + pub fn iter(&self) -> <&Self as IntoIterator>::IntoIter { + self.into_iter() + } + + /// An iterator visiting all key-value pairs in arbitrary order, + /// with mutable references to the values. + /// The iterator element type is `(&'a K, &'a mut V)`. + #[inline] + pub fn iter_mut(&mut self) -> impl Iterator<Item = (&'_ K, &'_ mut V)> { + self.into_iter() + } + + /// An iterator visiting all keys in arbitrary order. + /// The iterator element type is `&'a K`. + pub fn keys(&self) -> impl Iterator<Item = &'_ K> { + match self { + SsoHashMap::Array(array) => EitherIter::Left(array.iter().map(|(k, _v)| k)), + SsoHashMap::Map(map) => EitherIter::Right(map.keys()), + } + } + + /// An iterator visiting all values in arbitrary order. + /// The iterator element type is `&'a V`. + pub fn values(&self) -> impl Iterator<Item = &'_ V> { + match self { + SsoHashMap::Array(array) => EitherIter::Left(array.iter().map(|(_k, v)| v)), + SsoHashMap::Map(map) => EitherIter::Right(map.values()), + } + } + + /// An iterator visiting all values mutably in arbitrary order. + /// The iterator element type is `&'a mut V`. + pub fn values_mut(&mut self) -> impl Iterator<Item = &'_ mut V> { + match self { + SsoHashMap::Array(array) => EitherIter::Left(array.iter_mut().map(|(_k, v)| v)), + SsoHashMap::Map(map) => EitherIter::Right(map.values_mut()), + } + } + + /// Clears the map, returning all key-value pairs as an iterator. Keeps the + /// allocated memory for reuse. + pub fn drain(&mut self) -> impl Iterator<Item = (K, V)> + '_ { + match self { + SsoHashMap::Array(array) => EitherIter::Left(array.drain(..)), + SsoHashMap::Map(map) => EitherIter::Right(map.drain()), + } + } +} + +impl<K: Eq + Hash, V> SsoHashMap<K, V> { + /// Changes underlying storage from array to hashmap + /// if array is full. + fn migrate_if_full(&mut self) { + if let SsoHashMap::Array(array) = self { + if array.is_full() { + *self = SsoHashMap::Map(array.drain(..).collect()); + } + } + } + + /// Reserves capacity for at least `additional` more elements to be inserted + /// in the `SsoHashMap`. The collection may reserve more space to avoid + /// frequent reallocations. + pub fn reserve(&mut self, additional: usize) { + match self { + SsoHashMap::Array(array) => { + if SSO_ARRAY_SIZE < (array.len() + additional) { + let mut map: FxHashMap<K, V> = array.drain(..).collect(); + map.reserve(additional); + *self = SsoHashMap::Map(map); + } + } + SsoHashMap::Map(map) => map.reserve(additional), + } + } + + /// Shrinks the capacity of the map as much as possible. It will drop + /// down as much as possible while maintaining the internal rules + /// and possibly leaving some space in accordance with the resize policy. + pub fn shrink_to_fit(&mut self) { + if let SsoHashMap::Map(map) = self { + if map.len() <= SSO_ARRAY_SIZE { + *self = SsoHashMap::Array(map.drain().collect()); + } else { + map.shrink_to_fit(); + } + } + } + + /// Retains only the elements specified by the predicate. + pub fn retain<F>(&mut self, mut f: F) + where + F: FnMut(&K, &mut V) -> bool, + { + match self { + SsoHashMap::Array(array) => array.retain(|(k, v)| f(k, v)), + SsoHashMap::Map(map) => map.retain(f), + } + } + + /// Inserts a key-value pair into the map. + /// + /// If the map did not have this key present, [`None`] is returned. + /// + /// If the map did have this key present, the value is updated, and the old + /// value is returned. The key is not updated, though; this matters for + /// types that can be `==` without being identical. See the [module-level + /// documentation] for more. + pub fn insert(&mut self, key: K, value: V) -> Option<V> { + match self { + SsoHashMap::Array(array) => { + for (k, v) in array.iter_mut() { + if *k == key { + let old_value = std::mem::replace(v, value); + return Some(old_value); + } + } + if let Err(error) = array.try_push((key, value)) { + let mut map: FxHashMap<K, V> = array.drain(..).collect(); + let (key, value) = error.element(); + map.insert(key, value); + *self = SsoHashMap::Map(map); + } + None + } + SsoHashMap::Map(map) => map.insert(key, value), + } + } + + /// Removes a key from the map, returning the value at the key if the key + /// was previously in the map. + pub fn remove(&mut self, key: &K) -> Option<V> { + match self { + SsoHashMap::Array(array) => { + if let Some(index) = array.iter().position(|(k, _v)| k == key) { + Some(array.swap_remove(index).1) + } else { + None + } + } + SsoHashMap::Map(map) => map.remove(key), + } + } + + /// Removes a key from the map, returning the stored key and value if the + /// key was previously in the map. + pub fn remove_entry(&mut self, key: &K) -> Option<(K, V)> { + match self { + SsoHashMap::Array(array) => { + if let Some(index) = array.iter().position(|(k, _v)| k == key) { + Some(array.swap_remove(index)) + } else { + None + } + } + SsoHashMap::Map(map) => map.remove_entry(key), + } + } + + /// Returns a reference to the value corresponding to the key. + pub fn get(&self, key: &K) -> Option<&V> { + match self { + SsoHashMap::Array(array) => { + for (k, v) in array { + if k == key { + return Some(v); + } + } + None + } + SsoHashMap::Map(map) => map.get(key), + } + } + + /// Returns a mutable reference to the value corresponding to the key. + pub fn get_mut(&mut self, key: &K) -> Option<&mut V> { + match self { + SsoHashMap::Array(array) => { + for (k, v) in array { + if k == key { + return Some(v); + } + } + None + } + SsoHashMap::Map(map) => map.get_mut(key), + } + } + + /// Returns the key-value pair corresponding to the supplied key. + pub fn get_key_value(&self, key: &K) -> Option<(&K, &V)> { + match self { + SsoHashMap::Array(array) => { + for (k, v) in array { + if k == key { + return Some((k, v)); + } + } + None + } + SsoHashMap::Map(map) => map.get_key_value(key), + } + } + + /// Returns `true` if the map contains a value for the specified key. + pub fn contains_key(&self, key: &K) -> bool { + match self { + SsoHashMap::Array(array) => array.iter().any(|(k, _v)| k == key), + SsoHashMap::Map(map) => map.contains_key(key), + } + } + + /// Gets the given key's corresponding entry in the map for in-place manipulation. + #[inline] + pub fn entry(&mut self, key: K) -> Entry<'_, K, V> { + Entry { ssomap: self, key } + } +} + +impl<K, V> Default for SsoHashMap<K, V> { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl<K: Eq + Hash, V> FromIterator<(K, V)> for SsoHashMap<K, V> { + fn from_iter<I: IntoIterator<Item = (K, V)>>(iter: I) -> SsoHashMap<K, V> { + let mut map: SsoHashMap<K, V> = Default::default(); + map.extend(iter); + map + } +} + +impl<K: Eq + Hash, V> Extend<(K, V)> for SsoHashMap<K, V> { + fn extend<I>(&mut self, iter: I) + where + I: IntoIterator<Item = (K, V)>, + { + for (key, value) in iter.into_iter() { + self.insert(key, value); + } + } + + #[inline] + fn extend_one(&mut self, (k, v): (K, V)) { + self.insert(k, v); + } + + fn extend_reserve(&mut self, additional: usize) { + match self { + SsoHashMap::Array(array) => { + if SSO_ARRAY_SIZE < (array.len() + additional) { + let mut map: FxHashMap<K, V> = array.drain(..).collect(); + map.extend_reserve(additional); + *self = SsoHashMap::Map(map); + } + } + SsoHashMap::Map(map) => map.extend_reserve(additional), + } + } +} + +impl<'a, K, V> Extend<(&'a K, &'a V)> for SsoHashMap<K, V> +where + K: Eq + Hash + Copy, + V: Copy, +{ + fn extend<T: IntoIterator<Item = (&'a K, &'a V)>>(&mut self, iter: T) { + self.extend(iter.into_iter().map(|(k, v)| (*k, *v))) + } + + #[inline] + fn extend_one(&mut self, (&k, &v): (&'a K, &'a V)) { + self.insert(k, v); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + Extend::<(K, V)>::extend_reserve(self, additional) + } +} + +impl<K, V> IntoIterator for SsoHashMap<K, V> { + type IntoIter = EitherIter< + <ArrayVec<(K, V), 8> as IntoIterator>::IntoIter, + <FxHashMap<K, V> as IntoIterator>::IntoIter, + >; + type Item = <Self::IntoIter as Iterator>::Item; + + fn into_iter(self) -> Self::IntoIter { + match self { + SsoHashMap::Array(array) => EitherIter::Left(array.into_iter()), + SsoHashMap::Map(map) => EitherIter::Right(map.into_iter()), + } + } +} + +/// adapts Item of array reference iterator to Item of hashmap reference iterator. +#[inline(always)] +fn adapt_array_ref_it<K, V>(pair: &(K, V)) -> (&K, &V) { + let (a, b) = pair; + (a, b) +} + +/// adapts Item of array mut reference iterator to Item of hashmap mut reference iterator. +#[inline(always)] +fn adapt_array_mut_it<K, V>(pair: &mut (K, V)) -> (&K, &mut V) { + let (a, b) = pair; + (a, b) +} + +impl<'a, K, V> IntoIterator for &'a SsoHashMap<K, V> { + type IntoIter = EitherIter< + std::iter::Map< + <&'a ArrayVec<(K, V), 8> as IntoIterator>::IntoIter, + fn(&'a (K, V)) -> (&'a K, &'a V), + >, + <&'a FxHashMap<K, V> as IntoIterator>::IntoIter, + >; + type Item = <Self::IntoIter as Iterator>::Item; + + fn into_iter(self) -> Self::IntoIter { + match self { + SsoHashMap::Array(array) => EitherIter::Left(array.into_iter().map(adapt_array_ref_it)), + SsoHashMap::Map(map) => EitherIter::Right(map.iter()), + } + } +} + +impl<'a, K, V> IntoIterator for &'a mut SsoHashMap<K, V> { + type IntoIter = EitherIter< + std::iter::Map< + <&'a mut ArrayVec<(K, V), 8> as IntoIterator>::IntoIter, + fn(&'a mut (K, V)) -> (&'a K, &'a mut V), + >, + <&'a mut FxHashMap<K, V> as IntoIterator>::IntoIter, + >; + type Item = <Self::IntoIter as Iterator>::Item; + + fn into_iter(self) -> Self::IntoIter { + match self { + SsoHashMap::Array(array) => EitherIter::Left(array.into_iter().map(adapt_array_mut_it)), + SsoHashMap::Map(map) => EitherIter::Right(map.iter_mut()), + } + } +} + +impl<K, V> fmt::Debug for SsoHashMap<K, V> +where + K: fmt::Debug, + V: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_map().entries(self.iter()).finish() + } +} + +impl<'a, K, V> Index<&'a K> for SsoHashMap<K, V> +where + K: Eq + Hash, +{ + type Output = V; + + #[inline] + fn index(&self, key: &K) -> &V { + self.get(key).expect("no entry found for key") + } +} + +/// A view into a single entry in a map. +pub struct Entry<'a, K, V> { + ssomap: &'a mut SsoHashMap<K, V>, + key: K, +} + +impl<'a, K: Eq + Hash, V> Entry<'a, K, V> { + /// Provides in-place mutable access to an occupied entry before any + /// potential inserts into the map. + pub fn and_modify<F>(self, f: F) -> Self + where + F: FnOnce(&mut V), + { + if let Some(value) = self.ssomap.get_mut(&self.key) { + f(value); + } + self + } + + /// Ensures a value is in the entry by inserting the default if empty, and returns + /// a mutable reference to the value in the entry. + #[inline] + pub fn or_insert(self, value: V) -> &'a mut V { + self.or_insert_with(|| value) + } + + /// Ensures a value is in the entry by inserting the result of the default function if empty, + /// and returns a mutable reference to the value in the entry. + pub fn or_insert_with<F: FnOnce() -> V>(self, default: F) -> &'a mut V { + self.ssomap.migrate_if_full(); + match self.ssomap { + SsoHashMap::Array(array) => { + let key_ref = &self.key; + let found_index = array.iter().position(|(k, _v)| k == key_ref); + let index = if let Some(index) = found_index { + index + } else { + let index = array.len(); + array.try_push((self.key, default())).unwrap(); + index + }; + &mut array[index].1 + } + SsoHashMap::Map(map) => map.entry(self.key).or_insert_with(default), + } + } + + /// Returns a reference to this entry's key. + #[inline] + pub fn key(&self) -> &K { + &self.key + } +} + +impl<'a, K: Eq + Hash, V: Default> Entry<'a, K, V> { + /// Ensures a value is in the entry by inserting the default value if empty, + /// and returns a mutable reference to the value in the entry. + #[inline] + pub fn or_default(self) -> &'a mut V { + self.or_insert_with(Default::default) + } +} diff --git a/compiler/rustc_data_structures/src/sso/mod.rs b/compiler/rustc_data_structures/src/sso/mod.rs new file mode 100644 index 000000000..dd21bc8e6 --- /dev/null +++ b/compiler/rustc_data_structures/src/sso/mod.rs @@ -0,0 +1,6 @@ +mod either_iter; +mod map; +mod set; + +pub use map::SsoHashMap; +pub use set::SsoHashSet; diff --git a/compiler/rustc_data_structures/src/sso/set.rs b/compiler/rustc_data_structures/src/sso/set.rs new file mode 100644 index 000000000..4fda3adb7 --- /dev/null +++ b/compiler/rustc_data_structures/src/sso/set.rs @@ -0,0 +1,238 @@ +use std::fmt; +use std::hash::Hash; +use std::iter::FromIterator; + +use super::map::SsoHashMap; + +/// Small-storage-optimized implementation of a set. +/// +/// Stores elements in a small array up to a certain length +/// and switches to `HashSet` when that length is exceeded. +// +// FIXME: Implements subset of HashSet API. +// +// Missing HashSet API: +// all hasher-related +// try_reserve +// shrink_to (unstable) +// drain_filter (unstable) +// replace +// get_or_insert/get_or_insert_owned/get_or_insert_with (unstable) +// difference/symmetric_difference/intersection/union +// is_disjoint/is_subset/is_superset +// PartialEq/Eq (requires SsoHashMap implementation) +// BitOr/BitAnd/BitXor/Sub +#[derive(Clone)] +pub struct SsoHashSet<T> { + map: SsoHashMap<T, ()>, +} + +/// Adapter function used ot return +/// result if SsoHashMap functions into +/// result SsoHashSet should return. +#[inline(always)] +fn entry_to_key<K, V>((k, _v): (K, V)) -> K { + k +} + +impl<T> SsoHashSet<T> { + /// Creates an empty `SsoHashSet`. + #[inline] + pub fn new() -> Self { + Self { map: SsoHashMap::new() } + } + + /// Creates an empty `SsoHashSet` with the specified capacity. + #[inline] + pub fn with_capacity(cap: usize) -> Self { + Self { map: SsoHashMap::with_capacity(cap) } + } + + /// Clears the set, removing all values. + #[inline] + pub fn clear(&mut self) { + self.map.clear() + } + + /// Returns the number of elements the set can hold without reallocating. + #[inline] + pub fn capacity(&self) -> usize { + self.map.capacity() + } + + /// Returns the number of elements in the set. + #[inline] + pub fn len(&self) -> usize { + self.map.len() + } + + /// Returns `true` if the set contains no elements. + #[inline] + pub fn is_empty(&self) -> bool { + self.map.is_empty() + } + + /// An iterator visiting all elements in arbitrary order. + /// The iterator element type is `&'a T`. + #[inline] + pub fn iter(&self) -> impl Iterator<Item = &T> { + self.into_iter() + } + + /// Clears the set, returning all elements in an iterator. + #[inline] + pub fn drain(&mut self) -> impl Iterator<Item = T> + '_ { + self.map.drain().map(entry_to_key) + } +} + +impl<T: Eq + Hash> SsoHashSet<T> { + /// Reserves capacity for at least `additional` more elements to be inserted + /// in the `SsoHashSet`. The collection may reserve more space to avoid + /// frequent reallocations. + #[inline] + pub fn reserve(&mut self, additional: usize) { + self.map.reserve(additional) + } + + /// Shrinks the capacity of the set as much as possible. It will drop + /// down as much as possible while maintaining the internal rules + /// and possibly leaving some space in accordance with the resize policy. + #[inline] + pub fn shrink_to_fit(&mut self) { + self.map.shrink_to_fit() + } + + /// Retains only the elements specified by the predicate. + #[inline] + pub fn retain<F>(&mut self, mut f: F) + where + F: FnMut(&T) -> bool, + { + self.map.retain(|k, _v| f(k)) + } + + /// Removes and returns the value in the set, if any, that is equal to the given one. + #[inline] + pub fn take(&mut self, value: &T) -> Option<T> { + self.map.remove_entry(value).map(entry_to_key) + } + + /// Returns a reference to the value in the set, if any, that is equal to the given value. + #[inline] + pub fn get(&self, value: &T) -> Option<&T> { + self.map.get_key_value(value).map(entry_to_key) + } + + /// Adds a value to the set. + /// + /// Returns whether the value was newly inserted. That is: + /// + /// - If the set did not previously contain this value, `true` is returned. + /// - If the set already contained this value, `false` is returned. + #[inline] + pub fn insert(&mut self, elem: T) -> bool { + self.map.insert(elem, ()).is_none() + } + + /// Removes a value from the set. Returns whether the value was + /// present in the set. + #[inline] + pub fn remove(&mut self, value: &T) -> bool { + self.map.remove(value).is_some() + } + + /// Returns `true` if the set contains a value. + #[inline] + pub fn contains(&self, value: &T) -> bool { + self.map.contains_key(value) + } +} + +impl<T: Eq + Hash> FromIterator<T> for SsoHashSet<T> { + fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> SsoHashSet<T> { + let mut set: SsoHashSet<T> = Default::default(); + set.extend(iter); + set + } +} + +impl<T> Default for SsoHashSet<T> { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl<T: Eq + Hash> Extend<T> for SsoHashSet<T> { + fn extend<I>(&mut self, iter: I) + where + I: IntoIterator<Item = T>, + { + for val in iter.into_iter() { + self.insert(val); + } + } + + #[inline] + fn extend_one(&mut self, item: T) { + self.insert(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.map.extend_reserve(additional) + } +} + +impl<'a, T> Extend<&'a T> for SsoHashSet<T> +where + T: 'a + Eq + Hash + Copy, +{ + #[inline] + fn extend<I: IntoIterator<Item = &'a T>>(&mut self, iter: I) { + self.extend(iter.into_iter().cloned()); + } + + #[inline] + fn extend_one(&mut self, &item: &'a T) { + self.insert(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + Extend::<T>::extend_reserve(self, additional) + } +} + +impl<T> IntoIterator for SsoHashSet<T> { + type IntoIter = std::iter::Map<<SsoHashMap<T, ()> as IntoIterator>::IntoIter, fn((T, ())) -> T>; + type Item = <Self::IntoIter as Iterator>::Item; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.map.into_iter().map(entry_to_key) + } +} + +impl<'a, T> IntoIterator for &'a SsoHashSet<T> { + type IntoIter = std::iter::Map< + <&'a SsoHashMap<T, ()> as IntoIterator>::IntoIter, + fn((&'a T, &'a ())) -> &'a T, + >; + type Item = <Self::IntoIter as Iterator>::Item; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.map.iter().map(entry_to_key) + } +} + +impl<T> fmt::Debug for SsoHashSet<T> +where + T: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_set().entries(self.iter()).finish() + } +} diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs new file mode 100644 index 000000000..ce8591734 --- /dev/null +++ b/compiler/rustc_data_structures/src/stable_hasher.rs @@ -0,0 +1,650 @@ +use crate::sip128::SipHasher128; +use rustc_index::bit_set; +use rustc_index::vec; +use smallvec::SmallVec; +use std::hash::{BuildHasher, Hash, Hasher}; +use std::marker::PhantomData; +use std::mem; + +#[cfg(test)] +mod tests; + +/// When hashing something that ends up affecting properties like symbol names, +/// we want these symbol names to be calculated independently of other factors +/// like what architecture you're compiling *from*. +/// +/// To that end we always convert integers to little-endian format before +/// hashing and the architecture dependent `isize` and `usize` types are +/// extended to 64 bits if needed. +pub struct StableHasher { + state: SipHasher128, +} + +impl ::std::fmt::Debug for StableHasher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.state) + } +} + +pub trait StableHasherResult: Sized { + fn finish(hasher: StableHasher) -> Self; +} + +impl StableHasher { + #[inline] + pub fn new() -> Self { + StableHasher { state: SipHasher128::new_with_keys(0, 0) } + } + + #[inline] + pub fn finish<W: StableHasherResult>(self) -> W { + W::finish(self) + } +} + +impl StableHasherResult for u128 { + #[inline] + fn finish(hasher: StableHasher) -> Self { + let (_0, _1) = hasher.finalize(); + u128::from(_0) | (u128::from(_1) << 64) + } +} + +impl StableHasherResult for u64 { + #[inline] + fn finish(hasher: StableHasher) -> Self { + hasher.finalize().0 + } +} + +impl StableHasher { + #[inline] + pub fn finalize(self) -> (u64, u64) { + self.state.finish128() + } +} + +impl Hasher for StableHasher { + fn finish(&self) -> u64 { + panic!("use StableHasher::finalize instead"); + } + + #[inline] + fn write(&mut self, bytes: &[u8]) { + self.state.write(bytes); + } + + #[inline] + fn write_str(&mut self, s: &str) { + self.state.write_str(s); + } + + #[inline] + fn write_length_prefix(&mut self, len: usize) { + // Our impl for `usize` will extend it if needed. + self.write_usize(len); + } + + #[inline] + fn write_u8(&mut self, i: u8) { + self.state.write_u8(i); + } + + #[inline] + fn write_u16(&mut self, i: u16) { + self.state.short_write(i.to_le_bytes()); + } + + #[inline] + fn write_u32(&mut self, i: u32) { + self.state.short_write(i.to_le_bytes()); + } + + #[inline] + fn write_u64(&mut self, i: u64) { + self.state.short_write(i.to_le_bytes()); + } + + #[inline] + fn write_u128(&mut self, i: u128) { + self.state.write(&i.to_le_bytes()); + } + + #[inline] + fn write_usize(&mut self, i: usize) { + // Always treat usize as u64 so we get the same results on 32 and 64 bit + // platforms. This is important for symbol hashes when cross compiling, + // for example. + self.state.short_write((i as u64).to_le_bytes()); + } + + #[inline] + fn write_i8(&mut self, i: i8) { + self.state.write_i8(i); + } + + #[inline] + fn write_i16(&mut self, i: i16) { + self.state.short_write((i as u16).to_le_bytes()); + } + + #[inline] + fn write_i32(&mut self, i: i32) { + self.state.short_write((i as u32).to_le_bytes()); + } + + #[inline] + fn write_i64(&mut self, i: i64) { + self.state.short_write((i as u64).to_le_bytes()); + } + + #[inline] + fn write_i128(&mut self, i: i128) { + self.state.write(&(i as u128).to_le_bytes()); + } + + #[inline] + fn write_isize(&mut self, i: isize) { + // Always treat isize as a 64-bit number so we get the same results on 32 and 64 bit + // platforms. This is important for symbol hashes when cross compiling, + // for example. Sign extending here is preferable as it means that the + // same negative number hashes the same on both 32 and 64 bit platforms. + let value = i as u64; + + // Cold path + #[cold] + #[inline(never)] + fn hash_value(state: &mut SipHasher128, value: u64) { + state.write_u8(0xFF); + state.short_write(value.to_le_bytes()); + } + + // `isize` values often seem to have a small (positive) numeric value in practice. + // To exploit this, if the value is small, we will hash a smaller amount of bytes. + // However, we cannot just skip the leading zero bytes, as that would produce the same hash + // e.g. if you hash two values that have the same bit pattern when they are swapped. + // See https://github.com/rust-lang/rust/pull/93014 for context. + // + // Therefore, we employ the following strategy: + // 1) When we encounter a value that fits within a single byte (the most common case), we + // hash just that byte. This is the most common case that is being optimized. However, we do + // not do this for the value 0xFF, as that is a reserved prefix (a bit like in UTF-8). + // 2) When we encounter a larger value, we hash a "marker" 0xFF and then the corresponding + // 8 bytes. Since this prefix cannot occur when we hash a single byte, when we hash two + // `isize`s that fit within a different amount of bytes, they should always produce a different + // byte stream for the hasher. + if value < 0xFF { + self.state.write_u8(value as u8); + } else { + hash_value(&mut self.state, value); + } + } +} + +/// Something that implements `HashStable<CTX>` can be hashed in a way that is +/// stable across multiple compilation sessions. +/// +/// Note that `HashStable` imposes rather more strict requirements than usual +/// hash functions: +/// +/// - Stable hashes are sometimes used as identifiers. Therefore they must +/// conform to the corresponding `PartialEq` implementations: +/// +/// - `x == y` implies `hash_stable(x) == hash_stable(y)`, and +/// - `x != y` implies `hash_stable(x) != hash_stable(y)`. +/// +/// That second condition is usually not required for hash functions +/// (e.g. `Hash`). In practice this means that `hash_stable` must feed any +/// information into the hasher that a `PartialEq` comparison takes into +/// account. See [#49300](https://github.com/rust-lang/rust/issues/49300) +/// for an example where violating this invariant has caused trouble in the +/// past. +/// +/// - `hash_stable()` must be independent of the current +/// compilation session. E.g. they must not hash memory addresses or other +/// things that are "randomly" assigned per compilation session. +/// +/// - `hash_stable()` must be independent of the host architecture. The +/// `StableHasher` takes care of endianness and `isize`/`usize` platform +/// differences. +pub trait HashStable<CTX> { + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher); +} + +/// Implement this for types that can be turned into stable keys like, for +/// example, for DefId that can be converted to a DefPathHash. This is used for +/// bringing maps into a predictable order before hashing them. +pub trait ToStableHashKey<HCX> { + type KeyType: Ord + Sized + HashStable<HCX>; + fn to_stable_hash_key(&self, hcx: &HCX) -> Self::KeyType; +} + +/// Implement HashStable by just calling `Hash::hash()`. +/// +/// **WARNING** This is only valid for types that *really* don't need any context for fingerprinting. +/// But it is easy to misuse this macro (see [#96013](https://github.com/rust-lang/rust/issues/96013) +/// for examples). Therefore this macro is not exported and should only be used in the limited cases +/// here in this module. +/// +/// Use `#[derive(HashStable_Generic)]` instead. +macro_rules! impl_stable_hash_via_hash { + ($t:ty) => { + impl<CTX> $crate::stable_hasher::HashStable<CTX> for $t { + #[inline] + fn hash_stable(&self, _: &mut CTX, hasher: &mut $crate::stable_hasher::StableHasher) { + ::std::hash::Hash::hash(self, hasher); + } + } + }; +} + +impl_stable_hash_via_hash!(i8); +impl_stable_hash_via_hash!(i16); +impl_stable_hash_via_hash!(i32); +impl_stable_hash_via_hash!(i64); +impl_stable_hash_via_hash!(isize); + +impl_stable_hash_via_hash!(u8); +impl_stable_hash_via_hash!(u16); +impl_stable_hash_via_hash!(u32); +impl_stable_hash_via_hash!(u64); +impl_stable_hash_via_hash!(usize); + +impl_stable_hash_via_hash!(u128); +impl_stable_hash_via_hash!(i128); + +impl_stable_hash_via_hash!(char); +impl_stable_hash_via_hash!(()); + +impl<CTX> HashStable<CTX> for ! { + fn hash_stable(&self, _ctx: &mut CTX, _hasher: &mut StableHasher) { + unreachable!() + } +} + +impl<CTX, T> HashStable<CTX> for PhantomData<T> { + fn hash_stable(&self, _ctx: &mut CTX, _hasher: &mut StableHasher) {} +} + +impl<CTX> HashStable<CTX> for ::std::num::NonZeroU32 { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + self.get().hash_stable(ctx, hasher) + } +} + +impl<CTX> HashStable<CTX> for ::std::num::NonZeroUsize { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + self.get().hash_stable(ctx, hasher) + } +} + +impl<CTX> HashStable<CTX> for f32 { + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + let val: u32 = unsafe { ::std::mem::transmute(*self) }; + val.hash_stable(ctx, hasher); + } +} + +impl<CTX> HashStable<CTX> for f64 { + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + let val: u64 = unsafe { ::std::mem::transmute(*self) }; + val.hash_stable(ctx, hasher); + } +} + +impl<CTX> HashStable<CTX> for ::std::cmp::Ordering { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + (*self as i8).hash_stable(ctx, hasher); + } +} + +impl<T1: HashStable<CTX>, CTX> HashStable<CTX> for (T1,) { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + let (ref _0,) = *self; + _0.hash_stable(ctx, hasher); + } +} + +impl<T1: HashStable<CTX>, T2: HashStable<CTX>, CTX> HashStable<CTX> for (T1, T2) { + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + let (ref _0, ref _1) = *self; + _0.hash_stable(ctx, hasher); + _1.hash_stable(ctx, hasher); + } +} + +impl<T1, T2, T3, CTX> HashStable<CTX> for (T1, T2, T3) +where + T1: HashStable<CTX>, + T2: HashStable<CTX>, + T3: HashStable<CTX>, +{ + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + let (ref _0, ref _1, ref _2) = *self; + _0.hash_stable(ctx, hasher); + _1.hash_stable(ctx, hasher); + _2.hash_stable(ctx, hasher); + } +} + +impl<T1, T2, T3, T4, CTX> HashStable<CTX> for (T1, T2, T3, T4) +where + T1: HashStable<CTX>, + T2: HashStable<CTX>, + T3: HashStable<CTX>, + T4: HashStable<CTX>, +{ + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + let (ref _0, ref _1, ref _2, ref _3) = *self; + _0.hash_stable(ctx, hasher); + _1.hash_stable(ctx, hasher); + _2.hash_stable(ctx, hasher); + _3.hash_stable(ctx, hasher); + } +} + +impl<T: HashStable<CTX>, CTX> HashStable<CTX> for [T] { + default fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + self.len().hash_stable(ctx, hasher); + for item in self { + item.hash_stable(ctx, hasher); + } + } +} + +impl<CTX> HashStable<CTX> for [u8] { + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + self.len().hash_stable(ctx, hasher); + hasher.write(self); + } +} + +impl<T: HashStable<CTX>, CTX> HashStable<CTX> for Vec<T> { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + (&self[..]).hash_stable(ctx, hasher); + } +} + +impl<K, V, R, CTX> HashStable<CTX> for indexmap::IndexMap<K, V, R> +where + K: HashStable<CTX> + Eq + Hash, + V: HashStable<CTX>, + R: BuildHasher, +{ + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + self.len().hash_stable(ctx, hasher); + for kv in self { + kv.hash_stable(ctx, hasher); + } + } +} + +impl<K, R, CTX> HashStable<CTX> for indexmap::IndexSet<K, R> +where + K: HashStable<CTX> + Eq + Hash, + R: BuildHasher, +{ + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + self.len().hash_stable(ctx, hasher); + for key in self { + key.hash_stable(ctx, hasher); + } + } +} + +impl<A, CTX> HashStable<CTX> for SmallVec<[A; 1]> +where + A: HashStable<CTX>, +{ + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + (&self[..]).hash_stable(ctx, hasher); + } +} + +impl<T: ?Sized + HashStable<CTX>, CTX> HashStable<CTX> for Box<T> { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + (**self).hash_stable(ctx, hasher); + } +} + +impl<T: ?Sized + HashStable<CTX>, CTX> HashStable<CTX> for ::std::rc::Rc<T> { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + (**self).hash_stable(ctx, hasher); + } +} + +impl<T: ?Sized + HashStable<CTX>, CTX> HashStable<CTX> for ::std::sync::Arc<T> { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + (**self).hash_stable(ctx, hasher); + } +} + +impl<CTX> HashStable<CTX> for str { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + self.as_bytes().hash_stable(ctx, hasher); + } +} + +impl<CTX> HashStable<CTX> for String { + #[inline] + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + (&self[..]).hash_stable(hcx, hasher); + } +} + +impl<HCX> ToStableHashKey<HCX> for String { + type KeyType = String; + #[inline] + fn to_stable_hash_key(&self, _: &HCX) -> Self::KeyType { + self.clone() + } +} + +impl<CTX> HashStable<CTX> for bool { + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + (if *self { 1u8 } else { 0u8 }).hash_stable(ctx, hasher); + } +} + +impl<T, CTX> HashStable<CTX> for Option<T> +where + T: HashStable<CTX>, +{ + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + if let Some(ref value) = *self { + 1u8.hash_stable(ctx, hasher); + value.hash_stable(ctx, hasher); + } else { + 0u8.hash_stable(ctx, hasher); + } + } +} + +impl<T1, T2, CTX> HashStable<CTX> for Result<T1, T2> +where + T1: HashStable<CTX>, + T2: HashStable<CTX>, +{ + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + mem::discriminant(self).hash_stable(ctx, hasher); + match *self { + Ok(ref x) => x.hash_stable(ctx, hasher), + Err(ref x) => x.hash_stable(ctx, hasher), + } + } +} + +impl<'a, T, CTX> HashStable<CTX> for &'a T +where + T: HashStable<CTX> + ?Sized, +{ + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + (**self).hash_stable(ctx, hasher); + } +} + +impl<T, CTX> HashStable<CTX> for ::std::mem::Discriminant<T> { + #[inline] + fn hash_stable(&self, _: &mut CTX, hasher: &mut StableHasher) { + ::std::hash::Hash::hash(self, hasher); + } +} + +impl<T, CTX> HashStable<CTX> for ::std::ops::RangeInclusive<T> +where + T: HashStable<CTX>, +{ + #[inline] + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + self.start().hash_stable(ctx, hasher); + self.end().hash_stable(ctx, hasher); + } +} + +impl<I: vec::Idx, T, CTX> HashStable<CTX> for vec::IndexVec<I, T> +where + T: HashStable<CTX>, +{ + fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) { + self.len().hash_stable(ctx, hasher); + for v in &self.raw { + v.hash_stable(ctx, hasher); + } + } +} + +impl<I: vec::Idx, CTX> HashStable<CTX> for bit_set::BitSet<I> { + fn hash_stable(&self, _ctx: &mut CTX, hasher: &mut StableHasher) { + ::std::hash::Hash::hash(self, hasher); + } +} + +impl<R: vec::Idx, C: vec::Idx, CTX> HashStable<CTX> for bit_set::BitMatrix<R, C> { + fn hash_stable(&self, _ctx: &mut CTX, hasher: &mut StableHasher) { + ::std::hash::Hash::hash(self, hasher); + } +} + +impl<T, CTX> HashStable<CTX> for bit_set::FiniteBitSet<T> +where + T: HashStable<CTX> + bit_set::FiniteBitSetTy, +{ + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + self.0.hash_stable(hcx, hasher); + } +} + +impl_stable_hash_via_hash!(::std::path::Path); +impl_stable_hash_via_hash!(::std::path::PathBuf); + +impl<K, V, R, HCX> HashStable<HCX> for ::std::collections::HashMap<K, V, R> +where + K: ToStableHashKey<HCX> + Eq, + V: HashStable<HCX>, + R: BuildHasher, +{ + #[inline] + fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { + stable_hash_reduce(hcx, hasher, self.iter(), self.len(), |hasher, hcx, (key, value)| { + let key = key.to_stable_hash_key(hcx); + key.hash_stable(hcx, hasher); + value.hash_stable(hcx, hasher); + }); + } +} + +impl<K, R, HCX> HashStable<HCX> for ::std::collections::HashSet<K, R> +where + K: ToStableHashKey<HCX> + Eq, + R: BuildHasher, +{ + fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { + stable_hash_reduce(hcx, hasher, self.iter(), self.len(), |hasher, hcx, key| { + let key = key.to_stable_hash_key(hcx); + key.hash_stable(hcx, hasher); + }); + } +} + +impl<K, V, HCX> HashStable<HCX> for ::std::collections::BTreeMap<K, V> +where + K: ToStableHashKey<HCX>, + V: HashStable<HCX>, +{ + fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { + stable_hash_reduce(hcx, hasher, self.iter(), self.len(), |hasher, hcx, (key, value)| { + let key = key.to_stable_hash_key(hcx); + key.hash_stable(hcx, hasher); + value.hash_stable(hcx, hasher); + }); + } +} + +impl<K, HCX> HashStable<HCX> for ::std::collections::BTreeSet<K> +where + K: ToStableHashKey<HCX>, +{ + fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { + stable_hash_reduce(hcx, hasher, self.iter(), self.len(), |hasher, hcx, key| { + let key = key.to_stable_hash_key(hcx); + key.hash_stable(hcx, hasher); + }); + } +} + +fn stable_hash_reduce<HCX, I, C, F>( + hcx: &mut HCX, + hasher: &mut StableHasher, + mut collection: C, + length: usize, + hash_function: F, +) where + C: Iterator<Item = I>, + F: Fn(&mut StableHasher, &mut HCX, I), +{ + length.hash_stable(hcx, hasher); + + match length { + 1 => { + hash_function(hasher, hcx, collection.next().unwrap()); + } + _ => { + let hash = collection + .map(|value| { + let mut hasher = StableHasher::new(); + hash_function(&mut hasher, hcx, value); + hasher.finish::<u128>() + }) + .reduce(|accum, value| accum.wrapping_add(value)); + hash.hash_stable(hcx, hasher); + } + } +} + +/// Controls what data we do or do not hash. +/// Whenever a `HashStable` implementation caches its +/// result, it needs to include `HashingControls` as part +/// of the key, to ensure that it does not produce an incorrect +/// result (for example, using a `Fingerprint` produced while +/// hashing `Span`s when a `Fingerprint` without `Span`s is +/// being requested) +#[derive(Clone, Hash, Eq, PartialEq, Debug)] +pub struct HashingControls { + pub hash_spans: bool, +} diff --git a/compiler/rustc_data_structures/src/stable_hasher/tests.rs b/compiler/rustc_data_structures/src/stable_hasher/tests.rs new file mode 100644 index 000000000..b0d66c32a --- /dev/null +++ b/compiler/rustc_data_structures/src/stable_hasher/tests.rs @@ -0,0 +1,163 @@ +use super::*; + +// The tests below compare the computed hashes to particular expected values +// in order to test that we produce the same results on different platforms, +// regardless of endianness and `usize` and `isize` size differences (this +// of course assumes we run these tests on platforms that differ in those +// ways). The expected values depend on the hashing algorithm used, so they +// need to be updated whenever StableHasher changes its hashing algorithm. + +#[test] +fn test_hash_integers() { + // Test that integers are handled consistently across platforms. + let test_u8 = 0xAB_u8; + let test_u16 = 0xFFEE_u16; + let test_u32 = 0x445577AA_u32; + let test_u64 = 0x01234567_13243546_u64; + let test_u128 = 0x22114433_66557788_99AACCBB_EEDDFF77_u128; + let test_usize = 0xD0C0B0A0_usize; + + let test_i8 = -100_i8; + let test_i16 = -200_i16; + let test_i32 = -300_i32; + let test_i64 = -400_i64; + let test_i128 = -500_i128; + let test_isize = -600_isize; + + let mut h = StableHasher::new(); + test_u8.hash(&mut h); + test_u16.hash(&mut h); + test_u32.hash(&mut h); + test_u64.hash(&mut h); + test_u128.hash(&mut h); + test_usize.hash(&mut h); + test_i8.hash(&mut h); + test_i16.hash(&mut h); + test_i32.hash(&mut h); + test_i64.hash(&mut h); + test_i128.hash(&mut h); + test_isize.hash(&mut h); + + // This depends on the hashing algorithm. See note at top of file. + let expected = (1784307454142909076, 11471672289340283879); + + assert_eq!(h.finalize(), expected); +} + +#[test] +fn test_hash_usize() { + // Test that usize specifically is handled consistently across platforms. + let test_usize = 0xABCDEF01_usize; + + let mut h = StableHasher::new(); + test_usize.hash(&mut h); + + // This depends on the hashing algorithm. See note at top of file. + let expected = (5798740672699530587, 11186240177685111648); + + assert_eq!(h.finalize(), expected); +} + +#[test] +fn test_hash_isize() { + // Test that isize specifically is handled consistently across platforms. + let test_isize = -7_isize; + + let mut h = StableHasher::new(); + test_isize.hash(&mut h); + + // This depends on the hashing algorithm. See note at top of file. + let expected = (2789913510339652884, 674280939192711005); + + assert_eq!(h.finalize(), expected); +} + +fn hash<T: HashStable<()>>(t: &T) -> u128 { + let mut h = StableHasher::new(); + let ctx = &mut (); + t.hash_stable(ctx, &mut h); + h.finish() +} + +// Check that bit set hash includes the domain size. +#[test] +fn test_hash_bit_set() { + use rustc_index::bit_set::BitSet; + let a: BitSet<usize> = BitSet::new_empty(1); + let b: BitSet<usize> = BitSet::new_empty(2); + assert_ne!(a, b); + assert_ne!(hash(&a), hash(&b)); +} + +// Check that bit matrix hash includes the matrix dimensions. +#[test] +fn test_hash_bit_matrix() { + use rustc_index::bit_set::BitMatrix; + let a: BitMatrix<usize, usize> = BitMatrix::new(1, 1); + let b: BitMatrix<usize, usize> = BitMatrix::new(1, 2); + assert_ne!(a, b); + assert_ne!(hash(&a), hash(&b)); +} + +// Check that exchanging the value of two adjacent fields changes the hash. +#[test] +fn test_attribute_permutation() { + macro_rules! test_type { + ($ty: ty) => {{ + struct Foo { + a: $ty, + b: $ty, + } + + impl<CTX> HashStable<CTX> for Foo { + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + self.a.hash_stable(hcx, hasher); + self.b.hash_stable(hcx, hasher); + } + } + + #[allow(overflowing_literals)] + let mut item = Foo { a: 0xFF, b: 0xFF_FF }; + let hash_a = hash(&item); + std::mem::swap(&mut item.a, &mut item.b); + let hash_b = hash(&item); + assert_ne!( + hash_a, + hash_b, + "The hash stayed the same after values were swapped for type `{}`!", + stringify!($ty) + ); + }}; + } + + test_type!(u16); + test_type!(u32); + test_type!(u64); + test_type!(u128); + + test_type!(i16); + test_type!(i32); + test_type!(i64); + test_type!(i128); +} + +// Check that the `isize` hashing optimization does not produce the same hash when permuting two +// values. +#[test] +fn test_isize_compression() { + fn check_hash(a: u64, b: u64) { + let hash_a = hash(&(a as isize, b as isize)); + let hash_b = hash(&(b as isize, a as isize)); + assert_ne!( + hash_a, hash_b, + "The hash stayed the same when permuting values `{a}` and `{b}!", + ); + } + + check_hash(0xAA, 0xAAAA); + check_hash(0xFF, 0xFFFF); + check_hash(0xAAAA, 0xAAAAAA); + check_hash(0xAAAAAA, 0xAAAAAAAA); + check_hash(0xFF, 0xFFFFFFFFFFFFFFFF); + check_hash(u64::MAX /* -1 */, 1); +} diff --git a/compiler/rustc_data_structures/src/stack.rs b/compiler/rustc_data_structures/src/stack.rs new file mode 100644 index 000000000..3bdd67512 --- /dev/null +++ b/compiler/rustc_data_structures/src/stack.rs @@ -0,0 +1,18 @@ +// This is the amount of bytes that need to be left on the stack before increasing the size. +// It must be at least as large as the stack required by any code that does not call +// `ensure_sufficient_stack`. +const RED_ZONE: usize = 100 * 1024; // 100k + +// Only the first stack that is pushed, grows exponentially (2^n * STACK_PER_RECURSION) from then +// on. This flag has performance relevant characteristics. Don't set it too high. +const STACK_PER_RECURSION: usize = 1 * 1024 * 1024; // 1MB + +/// Grows the stack on demand to prevent stack overflow. Call this in strategic locations +/// to "break up" recursive calls. E.g. almost any call to `visit_expr` or equivalent can benefit +/// from this. +/// +/// Should not be sprinkled around carelessly, as it causes a little bit of overhead. +#[inline] +pub fn ensure_sufficient_stack<R>(f: impl FnOnce() -> R) -> R { + stacker::maybe_grow(RED_ZONE, STACK_PER_RECURSION, f) +} diff --git a/compiler/rustc_data_structures/src/steal.rs b/compiler/rustc_data_structures/src/steal.rs new file mode 100644 index 000000000..a3ece6550 --- /dev/null +++ b/compiler/rustc_data_structures/src/steal.rs @@ -0,0 +1,55 @@ +use crate::stable_hasher::{HashStable, StableHasher}; +use crate::sync::{MappedReadGuard, ReadGuard, RwLock}; + +/// The `Steal` struct is intended to used as the value for a query. +/// Specifically, we sometimes have queries (*cough* MIR *cough*) +/// where we create a large, complex value that we want to iteratively +/// update (e.g., optimize). We could clone the value for each +/// optimization, but that'd be expensive. And yet we don't just want +/// to mutate it in place, because that would spoil the idea that +/// queries are these pure functions that produce an immutable value +/// (since if you did the query twice, you could observe the mutations). +/// So instead we have the query produce a `&'tcx Steal<mir::Body<'tcx>>` +/// (to be very specific). Now we can read from this +/// as much as we want (using `borrow()`), but you can also +/// `steal()`. Once you steal, any further attempt to read will panic. +/// Therefore, we know that -- assuming no ICE -- nobody is observing +/// the fact that the MIR was updated. +/// +/// Obviously, whenever you have a query that yields a `Steal` value, +/// you must treat it with caution, and make sure that you know that +/// -- once the value is stolen -- it will never be read from again. +// +// FIXME(#41710): what is the best way to model linear queries? +#[derive(Debug)] +pub struct Steal<T> { + value: RwLock<Option<T>>, +} + +impl<T> Steal<T> { + pub fn new(value: T) -> Self { + Steal { value: RwLock::new(Some(value)) } + } + + #[track_caller] + pub fn borrow(&self) -> MappedReadGuard<'_, T> { + let borrow = self.value.borrow(); + if borrow.is_none() { + panic!("attempted to read from stolen value: {}", std::any::type_name::<T>()); + } + ReadGuard::map(borrow, |opt| opt.as_ref().unwrap()) + } + + #[track_caller] + pub fn steal(&self) -> T { + let value_ref = &mut *self.value.try_write().expect("stealing value which is locked"); + let value = value_ref.take(); + value.expect("attempt to steal from stolen value") + } +} + +impl<CTX, T: HashStable<CTX>> HashStable<CTX> for Steal<T> { + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + self.borrow().hash_stable(hcx, hasher); + } +} diff --git a/compiler/rustc_data_structures/src/svh.rs b/compiler/rustc_data_structures/src/svh.rs new file mode 100644 index 000000000..61654b9e8 --- /dev/null +++ b/compiler/rustc_data_structures/src/svh.rs @@ -0,0 +1,69 @@ +//! Calculation and management of a Strict Version Hash for crates +//! +//! The SVH is used for incremental compilation to track when HIR +//! nodes have changed between compilations, and also to detect +//! mismatches where we have two versions of the same crate that were +//! compiled from distinct sources. + +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; +use std::fmt; +use std::hash::{Hash, Hasher}; + +use crate::stable_hasher; + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct Svh { + hash: u64, +} + +impl Svh { + /// Creates a new `Svh` given the hash. If you actually want to + /// compute the SVH from some HIR, you want the `calculate_svh` + /// function found in `rustc_incremental`. + pub fn new(hash: u64) -> Svh { + Svh { hash } + } + + pub fn as_u64(&self) -> u64 { + self.hash + } + + pub fn to_string(&self) -> String { + format!("{:016x}", self.hash) + } +} + +impl Hash for Svh { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.hash.to_le().hash(state); + } +} + +impl fmt::Display for Svh { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(&self.to_string()) + } +} + +impl<S: Encoder> Encodable<S> for Svh { + fn encode(&self, s: &mut S) { + s.emit_u64(self.as_u64().to_le()); + } +} + +impl<D: Decoder> Decodable<D> for Svh { + fn decode(d: &mut D) -> Svh { + Svh::new(u64::from_le(d.read_u64())) + } +} + +impl<T> stable_hasher::HashStable<T> for Svh { + #[inline] + fn hash_stable(&self, ctx: &mut T, hasher: &mut stable_hasher::StableHasher) { + let Svh { hash } = *self; + hash.hash_stable(ctx, hasher); + } +} diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs new file mode 100644 index 000000000..52952a793 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync.rs @@ -0,0 +1,630 @@ +//! This module defines types which are thread safe if cfg!(parallel_compiler) is true. +//! +//! `Lrc` is an alias of `Arc` if cfg!(parallel_compiler) is true, `Rc` otherwise. +//! +//! `Lock` is a mutex. +//! It internally uses `parking_lot::Mutex` if cfg!(parallel_compiler) is true, +//! `RefCell` otherwise. +//! +//! `RwLock` is a read-write lock. +//! It internally uses `parking_lot::RwLock` if cfg!(parallel_compiler) is true, +//! `RefCell` otherwise. +//! +//! `MTLock` is a mutex which disappears if cfg!(parallel_compiler) is false. +//! +//! `MTRef` is an immutable reference if cfg!(parallel_compiler), and a mutable reference otherwise. +//! +//! `rustc_erase_owner!` erases an OwningRef owner into Erased or Erased + Send + Sync +//! depending on the value of cfg!(parallel_compiler). + +use crate::owning_ref::{Erased, OwningRef}; +use std::collections::HashMap; +use std::hash::{BuildHasher, Hash}; +use std::ops::{Deref, DerefMut}; +use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; + +pub use std::sync::atomic::Ordering; +pub use std::sync::atomic::Ordering::SeqCst; + +cfg_if! { + if #[cfg(not(parallel_compiler))] { + pub auto trait Send {} + pub auto trait Sync {} + + impl<T: ?Sized> Send for T {} + impl<T: ?Sized> Sync for T {} + + #[macro_export] + macro_rules! rustc_erase_owner { + ($v:expr) => { + $v.erase_owner() + } + } + + use std::ops::Add; + + /// This is a single threaded variant of `AtomicU64`, `AtomicUsize`, etc. + /// It has explicit ordering arguments and is only intended for use with + /// the native atomic types. + /// You should use this type through the `AtomicU64`, `AtomicUsize`, etc, type aliases + /// as it's not intended to be used separately. + #[derive(Debug)] + pub struct Atomic<T: Copy>(Cell<T>); + + impl<T: Copy> Atomic<T> { + #[inline] + pub fn new(v: T) -> Self { + Atomic(Cell::new(v)) + } + } + + impl<T: Copy> Atomic<T> { + #[inline] + pub fn into_inner(self) -> T { + self.0.into_inner() + } + + #[inline] + pub fn load(&self, _: Ordering) -> T { + self.0.get() + } + + #[inline] + pub fn store(&self, val: T, _: Ordering) { + self.0.set(val) + } + + #[inline] + pub fn swap(&self, val: T, _: Ordering) -> T { + self.0.replace(val) + } + } + + impl<T: Copy + PartialEq> Atomic<T> { + #[inline] + pub fn compare_exchange(&self, + current: T, + new: T, + _: Ordering, + _: Ordering) + -> Result<T, T> { + let read = self.0.get(); + if read == current { + self.0.set(new); + Ok(read) + } else { + Err(read) + } + } + } + + impl<T: Add<Output=T> + Copy> Atomic<T> { + #[inline] + pub fn fetch_add(&self, val: T, _: Ordering) -> T { + let old = self.0.get(); + self.0.set(old + val); + old + } + } + + pub type AtomicUsize = Atomic<usize>; + pub type AtomicBool = Atomic<bool>; + pub type AtomicU32 = Atomic<u32>; + pub type AtomicU64 = Atomic<u64>; + + pub fn join<A, B, RA, RB>(oper_a: A, oper_b: B) -> (RA, RB) + where A: FnOnce() -> RA, + B: FnOnce() -> RB + { + (oper_a(), oper_b()) + } + + #[macro_export] + macro_rules! parallel { + ($($blocks:tt),*) => { + // We catch panics here ensuring that all the blocks execute. + // This makes behavior consistent with the parallel compiler. + let mut panic = None; + $( + if let Err(p) = ::std::panic::catch_unwind( + ::std::panic::AssertUnwindSafe(|| $blocks) + ) { + if panic.is_none() { + panic = Some(p); + } + } + )* + if let Some(panic) = panic { + ::std::panic::resume_unwind(panic); + } + } + } + + pub use std::iter::Iterator as ParallelIterator; + + pub fn par_iter<T: IntoIterator>(t: T) -> T::IntoIter { + t.into_iter() + } + + pub fn par_for_each_in<T: IntoIterator>(t: T, mut for_each: impl FnMut(T::Item) + Sync + Send) { + // We catch panics here ensuring that all the loop iterations execute. + // This makes behavior consistent with the parallel compiler. + let mut panic = None; + t.into_iter().for_each(|i| { + if let Err(p) = catch_unwind(AssertUnwindSafe(|| for_each(i))) { + if panic.is_none() { + panic = Some(p); + } + } + }); + if let Some(panic) = panic { + resume_unwind(panic); + } + } + + pub type MetadataRef = OwningRef<Box<dyn Erased>, [u8]>; + + pub use std::rc::Rc as Lrc; + pub use std::rc::Weak as Weak; + pub use std::cell::Ref as ReadGuard; + pub use std::cell::Ref as MappedReadGuard; + pub use std::cell::RefMut as WriteGuard; + pub use std::cell::RefMut as MappedWriteGuard; + pub use std::cell::RefMut as LockGuard; + pub use std::cell::RefMut as MappedLockGuard; + + pub use std::cell::OnceCell; + + use std::cell::RefCell as InnerRwLock; + use std::cell::RefCell as InnerLock; + + use std::cell::Cell; + + #[derive(Debug)] + pub struct WorkerLocal<T>(OneThread<T>); + + impl<T> WorkerLocal<T> { + /// Creates a new worker local where the `initial` closure computes the + /// value this worker local should take for each thread in the thread pool. + #[inline] + pub fn new<F: FnMut(usize) -> T>(mut f: F) -> WorkerLocal<T> { + WorkerLocal(OneThread::new(f(0))) + } + + /// Returns the worker-local value for each thread + #[inline] + pub fn into_inner(self) -> Vec<T> { + vec![OneThread::into_inner(self.0)] + } + } + + impl<T> Deref for WorkerLocal<T> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + &*self.0 + } + } + + pub type MTRef<'a, T> = &'a mut T; + + #[derive(Debug, Default)] + pub struct MTLock<T>(T); + + impl<T> MTLock<T> { + #[inline(always)] + pub fn new(inner: T) -> Self { + MTLock(inner) + } + + #[inline(always)] + pub fn into_inner(self) -> T { + self.0 + } + + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + &mut self.0 + } + + #[inline(always)] + pub fn lock(&self) -> &T { + &self.0 + } + + #[inline(always)] + pub fn lock_mut(&mut self) -> &mut T { + &mut self.0 + } + } + + // FIXME: Probably a bad idea (in the threaded case) + impl<T: Clone> Clone for MTLock<T> { + #[inline] + fn clone(&self) -> Self { + MTLock(self.0.clone()) + } + } + } else { + pub use std::marker::Send as Send; + pub use std::marker::Sync as Sync; + + pub use parking_lot::RwLockReadGuard as ReadGuard; + pub use parking_lot::MappedRwLockReadGuard as MappedReadGuard; + pub use parking_lot::RwLockWriteGuard as WriteGuard; + pub use parking_lot::MappedRwLockWriteGuard as MappedWriteGuard; + + pub use parking_lot::MutexGuard as LockGuard; + pub use parking_lot::MappedMutexGuard as MappedLockGuard; + + pub use std::sync::OnceLock as OnceCell; + + pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32, AtomicU64}; + + pub use std::sync::Arc as Lrc; + pub use std::sync::Weak as Weak; + + pub type MTRef<'a, T> = &'a T; + + #[derive(Debug, Default)] + pub struct MTLock<T>(Lock<T>); + + impl<T> MTLock<T> { + #[inline(always)] + pub fn new(inner: T) -> Self { + MTLock(Lock::new(inner)) + } + + #[inline(always)] + pub fn into_inner(self) -> T { + self.0.into_inner() + } + + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + self.0.get_mut() + } + + #[inline(always)] + pub fn lock(&self) -> LockGuard<'_, T> { + self.0.lock() + } + + #[inline(always)] + pub fn lock_mut(&self) -> LockGuard<'_, T> { + self.lock() + } + } + + use parking_lot::Mutex as InnerLock; + use parking_lot::RwLock as InnerRwLock; + + use std::thread; + pub use rayon::{join, scope}; + + /// Runs a list of blocks in parallel. The first block is executed immediately on + /// the current thread. Use that for the longest running block. + #[macro_export] + macro_rules! parallel { + (impl $fblock:tt [$($c:tt,)*] [$block:tt $(, $rest:tt)*]) => { + parallel!(impl $fblock [$block, $($c,)*] [$($rest),*]) + }; + (impl $fblock:tt [$($blocks:tt,)*] []) => { + ::rustc_data_structures::sync::scope(|s| { + $( + s.spawn(|_| $blocks); + )* + $fblock; + }) + }; + ($fblock:tt, $($blocks:tt),*) => { + // Reverse the order of the later blocks since Rayon executes them in reverse order + // when using a single thread. This ensures the execution order matches that + // of a single threaded rustc + parallel!(impl $fblock [] [$($blocks),*]); + }; + } + + pub use rayon_core::WorkerLocal; + + pub use rayon::iter::ParallelIterator; + use rayon::iter::IntoParallelIterator; + + pub fn par_iter<T: IntoParallelIterator>(t: T) -> T::Iter { + t.into_par_iter() + } + + pub fn par_for_each_in<T: IntoParallelIterator>( + t: T, + for_each: impl Fn(T::Item) + Sync + Send, + ) { + let ps: Vec<_> = t.into_par_iter().map(|i| catch_unwind(AssertUnwindSafe(|| for_each(i)))).collect(); + ps.into_iter().for_each(|p| if let Err(panic) = p { + resume_unwind(panic) + }); + } + + pub type MetadataRef = OwningRef<Box<dyn Erased + Send + Sync>, [u8]>; + + /// This makes locks panic if they are already held. + /// It is only useful when you are running in a single thread + const ERROR_CHECKING: bool = false; + + #[macro_export] + macro_rules! rustc_erase_owner { + ($v:expr) => {{ + let v = $v; + ::rustc_data_structures::sync::assert_send_val(&v); + v.erase_send_sync_owner() + }} + } + } +} + +pub fn assert_sync<T: ?Sized + Sync>() {} +pub fn assert_send<T: ?Sized + Send>() {} +pub fn assert_send_val<T: ?Sized + Send>(_t: &T) {} +pub fn assert_send_sync_val<T: ?Sized + Sync + Send>(_t: &T) {} + +pub trait HashMapExt<K, V> { + /// Same as HashMap::insert, but it may panic if there's already an + /// entry for `key` with a value not equal to `value` + fn insert_same(&mut self, key: K, value: V); +} + +impl<K: Eq + Hash, V: Eq, S: BuildHasher> HashMapExt<K, V> for HashMap<K, V, S> { + fn insert_same(&mut self, key: K, value: V) { + self.entry(key).and_modify(|old| assert!(*old == value)).or_insert(value); + } +} + +#[derive(Debug)] +pub struct Lock<T>(InnerLock<T>); + +impl<T> Lock<T> { + #[inline(always)] + pub fn new(inner: T) -> Self { + Lock(InnerLock::new(inner)) + } + + #[inline(always)] + pub fn into_inner(self) -> T { + self.0.into_inner() + } + + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + self.0.get_mut() + } + + #[cfg(parallel_compiler)] + #[inline(always)] + pub fn try_lock(&self) -> Option<LockGuard<'_, T>> { + self.0.try_lock() + } + + #[cfg(not(parallel_compiler))] + #[inline(always)] + pub fn try_lock(&self) -> Option<LockGuard<'_, T>> { + self.0.try_borrow_mut().ok() + } + + #[cfg(parallel_compiler)] + #[inline(always)] + pub fn lock(&self) -> LockGuard<'_, T> { + if ERROR_CHECKING { + self.0.try_lock().expect("lock was already held") + } else { + self.0.lock() + } + } + + #[cfg(not(parallel_compiler))] + #[inline(always)] + pub fn lock(&self) -> LockGuard<'_, T> { + self.0.borrow_mut() + } + + #[inline(always)] + pub fn with_lock<F: FnOnce(&mut T) -> R, R>(&self, f: F) -> R { + f(&mut *self.lock()) + } + + #[inline(always)] + pub fn borrow(&self) -> LockGuard<'_, T> { + self.lock() + } + + #[inline(always)] + pub fn borrow_mut(&self) -> LockGuard<'_, T> { + self.lock() + } +} + +impl<T: Default> Default for Lock<T> { + #[inline] + fn default() -> Self { + Lock::new(T::default()) + } +} + +// FIXME: Probably a bad idea +impl<T: Clone> Clone for Lock<T> { + #[inline] + fn clone(&self) -> Self { + Lock::new(self.borrow().clone()) + } +} + +#[derive(Debug, Default)] +pub struct RwLock<T>(InnerRwLock<T>); + +impl<T> RwLock<T> { + #[inline(always)] + pub fn new(inner: T) -> Self { + RwLock(InnerRwLock::new(inner)) + } + + #[inline(always)] + pub fn into_inner(self) -> T { + self.0.into_inner() + } + + #[inline(always)] + pub fn get_mut(&mut self) -> &mut T { + self.0.get_mut() + } + + #[cfg(not(parallel_compiler))] + #[inline(always)] + pub fn read(&self) -> ReadGuard<'_, T> { + self.0.borrow() + } + + #[cfg(parallel_compiler)] + #[inline(always)] + pub fn read(&self) -> ReadGuard<'_, T> { + if ERROR_CHECKING { + self.0.try_read().expect("lock was already held") + } else { + self.0.read() + } + } + + #[inline(always)] + pub fn with_read_lock<F: FnOnce(&T) -> R, R>(&self, f: F) -> R { + f(&*self.read()) + } + + #[cfg(not(parallel_compiler))] + #[inline(always)] + pub fn try_write(&self) -> Result<WriteGuard<'_, T>, ()> { + self.0.try_borrow_mut().map_err(|_| ()) + } + + #[cfg(parallel_compiler)] + #[inline(always)] + pub fn try_write(&self) -> Result<WriteGuard<'_, T>, ()> { + self.0.try_write().ok_or(()) + } + + #[cfg(not(parallel_compiler))] + #[inline(always)] + pub fn write(&self) -> WriteGuard<'_, T> { + self.0.borrow_mut() + } + + #[cfg(parallel_compiler)] + #[inline(always)] + pub fn write(&self) -> WriteGuard<'_, T> { + if ERROR_CHECKING { + self.0.try_write().expect("lock was already held") + } else { + self.0.write() + } + } + + #[inline(always)] + pub fn with_write_lock<F: FnOnce(&mut T) -> R, R>(&self, f: F) -> R { + f(&mut *self.write()) + } + + #[inline(always)] + pub fn borrow(&self) -> ReadGuard<'_, T> { + self.read() + } + + #[inline(always)] + pub fn borrow_mut(&self) -> WriteGuard<'_, T> { + self.write() + } + + #[cfg(not(parallel_compiler))] + #[inline(always)] + pub fn clone_guard<'a>(rg: &ReadGuard<'a, T>) -> ReadGuard<'a, T> { + ReadGuard::clone(rg) + } + + #[cfg(parallel_compiler)] + #[inline(always)] + pub fn clone_guard<'a>(rg: &ReadGuard<'a, T>) -> ReadGuard<'a, T> { + ReadGuard::rwlock(&rg).read() + } + + #[cfg(not(parallel_compiler))] + #[inline(always)] + pub fn leak(&self) -> &T { + ReadGuard::leak(self.read()) + } + + #[cfg(parallel_compiler)] + #[inline(always)] + pub fn leak(&self) -> &T { + let guard = self.read(); + let ret = unsafe { &*(&*guard as *const T) }; + std::mem::forget(guard); + ret + } +} + +// FIXME: Probably a bad idea +impl<T: Clone> Clone for RwLock<T> { + #[inline] + fn clone(&self) -> Self { + RwLock::new(self.borrow().clone()) + } +} + +/// A type which only allows its inner value to be used in one thread. +/// It will panic if it is used on multiple threads. +#[derive(Debug)] +pub struct OneThread<T> { + #[cfg(parallel_compiler)] + thread: thread::ThreadId, + inner: T, +} + +#[cfg(parallel_compiler)] +unsafe impl<T> std::marker::Sync for OneThread<T> {} +#[cfg(parallel_compiler)] +unsafe impl<T> std::marker::Send for OneThread<T> {} + +impl<T> OneThread<T> { + #[inline(always)] + fn check(&self) { + #[cfg(parallel_compiler)] + assert_eq!(thread::current().id(), self.thread); + } + + #[inline(always)] + pub fn new(inner: T) -> Self { + OneThread { + #[cfg(parallel_compiler)] + thread: thread::current().id(), + inner, + } + } + + #[inline(always)] + pub fn into_inner(value: Self) -> T { + value.check(); + value.inner + } +} + +impl<T> Deref for OneThread<T> { + type Target = T; + + fn deref(&self) -> &T { + self.check(); + &self.inner + } +} + +impl<T> DerefMut for OneThread<T> { + fn deref_mut(&mut self) -> &mut T { + self.check(); + &mut self.inner + } +} diff --git a/compiler/rustc_data_structures/src/tagged_ptr.rs b/compiler/rustc_data_structures/src/tagged_ptr.rs new file mode 100644 index 000000000..651bc556c --- /dev/null +++ b/compiler/rustc_data_structures/src/tagged_ptr.rs @@ -0,0 +1,168 @@ +//! This module implements tagged pointers. +//! +//! In order to utilize the pointer packing, you must have two types: a pointer, +//! and a tag. +//! +//! The pointer must implement the `Pointer` trait, with the primary requirement +//! being conversion to and from a usize. Note that the pointer must be +//! dereferenceable, so raw pointers generally cannot implement the `Pointer` +//! trait. This implies that the pointer must also be nonzero. +//! +//! Many common pointer types already implement the `Pointer` trait. +//! +//! The tag must implement the `Tag` trait. We assert that the tag and `Pointer` +//! are compatible at compile time. + +use std::mem::ManuallyDrop; +use std::ops::Deref; +use std::rc::Rc; +use std::sync::Arc; + +mod copy; +mod drop; + +pub use copy::CopyTaggedPtr; +pub use drop::TaggedPtr; + +/// This describes the pointer type encapsulated by TaggedPtr. +/// +/// # Safety +/// +/// The usize returned from `into_usize` must be a valid, dereferenceable, +/// pointer to `<Self as Deref>::Target`. Note that pointers to `Pointee` must +/// be thin, even though `Pointee` may not be sized. +/// +/// Note that the returned pointer from `into_usize` should be castable to `&mut +/// <Self as Deref>::Target` if `Pointer: DerefMut`. +/// +/// The BITS constant must be correct. At least `BITS` bits, least-significant, +/// must be zero on all returned pointers from `into_usize`. +/// +/// For example, if the alignment of `Pointee` is 2, then `BITS` should be 1. +pub unsafe trait Pointer: Deref { + /// Most likely the value you want to use here is the following, unless + /// your Pointee type is unsized (e.g., `ty::List<T>` in rustc) in which + /// case you'll need to manually figure out what the right type to pass to + /// align_of is. + /// + /// ```ignore UNSOLVED (what to do about the Self) + /// # use std::ops::Deref; + /// std::mem::align_of::<<Self as Deref>::Target>().trailing_zeros() as usize; + /// ``` + const BITS: usize; + fn into_usize(self) -> usize; + + /// # Safety + /// + /// The passed `ptr` must be returned from `into_usize`. + /// + /// This acts as `ptr::read` semantically, it should not be called more than + /// once on non-`Copy` `Pointer`s. + unsafe fn from_usize(ptr: usize) -> Self; + + /// This provides a reference to the `Pointer` itself, rather than the + /// `Deref::Target`. It is used for cases where we want to call methods that + /// may be implement differently for the Pointer than the Pointee (e.g., + /// `Rc::clone` vs cloning the inner value). + /// + /// # Safety + /// + /// The passed `ptr` must be returned from `into_usize`. + unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R; +} + +/// This describes tags that the `TaggedPtr` struct can hold. +/// +/// # Safety +/// +/// The BITS constant must be correct. +/// +/// No more than `BITS` least significant bits may be set in the returned usize. +pub unsafe trait Tag: Copy { + const BITS: usize; + + fn into_usize(self) -> usize; + + /// # Safety + /// + /// The passed `tag` must be returned from `into_usize`. + unsafe fn from_usize(tag: usize) -> Self; +} + +unsafe impl<T> Pointer for Box<T> { + const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize; + #[inline] + fn into_usize(self) -> usize { + Box::into_raw(self) as usize + } + #[inline] + unsafe fn from_usize(ptr: usize) -> Self { + Box::from_raw(ptr as *mut T) + } + unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R { + let raw = ManuallyDrop::new(Self::from_usize(ptr)); + f(&raw) + } +} + +unsafe impl<T> Pointer for Rc<T> { + const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize; + #[inline] + fn into_usize(self) -> usize { + Rc::into_raw(self) as usize + } + #[inline] + unsafe fn from_usize(ptr: usize) -> Self { + Rc::from_raw(ptr as *const T) + } + unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R { + let raw = ManuallyDrop::new(Self::from_usize(ptr)); + f(&raw) + } +} + +unsafe impl<T> Pointer for Arc<T> { + const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize; + #[inline] + fn into_usize(self) -> usize { + Arc::into_raw(self) as usize + } + #[inline] + unsafe fn from_usize(ptr: usize) -> Self { + Arc::from_raw(ptr as *const T) + } + unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R { + let raw = ManuallyDrop::new(Self::from_usize(ptr)); + f(&raw) + } +} + +unsafe impl<'a, T: 'a> Pointer for &'a T { + const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize; + #[inline] + fn into_usize(self) -> usize { + self as *const T as usize + } + #[inline] + unsafe fn from_usize(ptr: usize) -> Self { + &*(ptr as *const T) + } + unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R { + f(&*(&ptr as *const usize as *const Self)) + } +} + +unsafe impl<'a, T: 'a> Pointer for &'a mut T { + const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize; + #[inline] + fn into_usize(self) -> usize { + self as *mut T as usize + } + #[inline] + unsafe fn from_usize(ptr: usize) -> Self { + &mut *(ptr as *mut T) + } + unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R { + f(&*(&ptr as *const usize as *const Self)) + } +} diff --git a/compiler/rustc_data_structures/src/tagged_ptr/copy.rs b/compiler/rustc_data_structures/src/tagged_ptr/copy.rs new file mode 100644 index 000000000..e1d3e0bd3 --- /dev/null +++ b/compiler/rustc_data_structures/src/tagged_ptr/copy.rs @@ -0,0 +1,185 @@ +use super::{Pointer, Tag}; +use crate::stable_hasher::{HashStable, StableHasher}; +use std::fmt; +use std::marker::PhantomData; +use std::num::NonZeroUsize; + +/// A `Copy` TaggedPtr. +/// +/// You should use this instead of the `TaggedPtr` type in all cases where +/// `P: Copy`. +/// +/// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without +/// unpacking. Otherwise we don't implement PartialEq/Eq/Hash; if you want that, +/// wrap the TaggedPtr. +pub struct CopyTaggedPtr<P, T, const COMPARE_PACKED: bool> +where + P: Pointer, + T: Tag, +{ + packed: NonZeroUsize, + data: PhantomData<(P, T)>, +} + +impl<P, T, const COMPARE_PACKED: bool> Copy for CopyTaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer, + T: Tag, + P: Copy, +{ +} + +impl<P, T, const COMPARE_PACKED: bool> Clone for CopyTaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer, + T: Tag, + P: Copy, +{ + fn clone(&self) -> Self { + *self + } +} + +// We pack the tag into the *upper* bits of the pointer to ease retrieval of the +// value; a left shift is a multiplication and those are embeddable in +// instruction encoding. +impl<P, T, const COMPARE_PACKED: bool> CopyTaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer, + T: Tag, +{ + const TAG_BIT_SHIFT: usize = usize::BITS as usize - T::BITS; + const ASSERTION: () = { + assert!(T::BITS <= P::BITS); + // Used for the transmute_copy's below + assert!(std::mem::size_of::<&P::Target>() == std::mem::size_of::<usize>()); + }; + + pub fn new(pointer: P, tag: T) -> Self { + // Trigger assert! + let () = Self::ASSERTION; + let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT; + + Self { + // SAFETY: We know that the pointer is non-null, as it must be + // dereferenceable per `Pointer` safety contract. + packed: unsafe { + NonZeroUsize::new_unchecked((P::into_usize(pointer) >> T::BITS) | packed_tag) + }, + data: PhantomData, + } + } + + pub(super) fn pointer_raw(&self) -> usize { + self.packed.get() << T::BITS + } + pub fn pointer(self) -> P + where + P: Copy, + { + // SAFETY: pointer_raw returns the original pointer + // + // Note that this isn't going to double-drop or anything because we have + // P: Copy + unsafe { P::from_usize(self.pointer_raw()) } + } + pub fn pointer_ref(&self) -> &P::Target { + // SAFETY: pointer_raw returns the original pointer + unsafe { std::mem::transmute_copy(&self.pointer_raw()) } + } + pub fn pointer_mut(&mut self) -> &mut P::Target + where + P: std::ops::DerefMut, + { + // SAFETY: pointer_raw returns the original pointer + unsafe { std::mem::transmute_copy(&self.pointer_raw()) } + } + #[inline] + pub fn tag(&self) -> T { + unsafe { T::from_usize(self.packed.get() >> Self::TAG_BIT_SHIFT) } + } + #[inline] + pub fn set_tag(&mut self, tag: T) { + let mut packed = self.packed.get(); + let new_tag = T::into_usize(tag) << Self::TAG_BIT_SHIFT; + let tag_mask = (1 << T::BITS) - 1; + packed &= !(tag_mask << Self::TAG_BIT_SHIFT); + packed |= new_tag; + self.packed = unsafe { NonZeroUsize::new_unchecked(packed) }; + } +} + +impl<P, T, const COMPARE_PACKED: bool> std::ops::Deref for CopyTaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer, + T: Tag, +{ + type Target = P::Target; + fn deref(&self) -> &Self::Target { + self.pointer_ref() + } +} + +impl<P, T, const COMPARE_PACKED: bool> std::ops::DerefMut for CopyTaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer + std::ops::DerefMut, + T: Tag, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + self.pointer_mut() + } +} + +impl<P, T, const COMPARE_PACKED: bool> fmt::Debug for CopyTaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer, + P::Target: fmt::Debug, + T: Tag + fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CopyTaggedPtr") + .field("pointer", &self.pointer_ref()) + .field("tag", &self.tag()) + .finish() + } +} + +impl<P, T> PartialEq for CopyTaggedPtr<P, T, true> +where + P: Pointer, + T: Tag, +{ + fn eq(&self, other: &Self) -> bool { + self.packed == other.packed + } +} + +impl<P, T> Eq for CopyTaggedPtr<P, T, true> +where + P: Pointer, + T: Tag, +{ +} + +impl<P, T> std::hash::Hash for CopyTaggedPtr<P, T, true> +where + P: Pointer, + T: Tag, +{ + fn hash<H: std::hash::Hasher>(&self, state: &mut H) { + self.packed.hash(state); + } +} + +impl<P, T, HCX, const COMPARE_PACKED: bool> HashStable<HCX> for CopyTaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer + HashStable<HCX>, + T: Tag + HashStable<HCX>, +{ + fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { + unsafe { + Pointer::with_ref(self.pointer_raw(), |p: &P| p.hash_stable(hcx, hasher)); + } + self.tag().hash_stable(hcx, hasher); + } +} diff --git a/compiler/rustc_data_structures/src/tagged_ptr/drop.rs b/compiler/rustc_data_structures/src/tagged_ptr/drop.rs new file mode 100644 index 000000000..d44ccd368 --- /dev/null +++ b/compiler/rustc_data_structures/src/tagged_ptr/drop.rs @@ -0,0 +1,133 @@ +use super::{Pointer, Tag}; +use crate::stable_hasher::{HashStable, StableHasher}; +use std::fmt; + +use super::CopyTaggedPtr; + +/// A TaggedPtr implementing `Drop`. +/// +/// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without +/// unpacking. Otherwise we don't implement PartialEq/Eq/Hash; if you want that, +/// wrap the TaggedPtr. +pub struct TaggedPtr<P, T, const COMPARE_PACKED: bool> +where + P: Pointer, + T: Tag, +{ + raw: CopyTaggedPtr<P, T, COMPARE_PACKED>, +} + +impl<P, T, const COMPARE_PACKED: bool> Clone for TaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer + Clone, + T: Tag, +{ + fn clone(&self) -> Self { + unsafe { Self::new(P::with_ref(self.raw.pointer_raw(), |p| p.clone()), self.raw.tag()) } + } +} + +// We pack the tag into the *upper* bits of the pointer to ease retrieval of the +// value; a right shift is a multiplication and those are embeddable in +// instruction encoding. +impl<P, T, const COMPARE_PACKED: bool> TaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer, + T: Tag, +{ + pub fn new(pointer: P, tag: T) -> Self { + TaggedPtr { raw: CopyTaggedPtr::new(pointer, tag) } + } + + pub fn pointer_ref(&self) -> &P::Target { + self.raw.pointer_ref() + } + pub fn tag(&self) -> T { + self.raw.tag() + } +} + +impl<P, T, const COMPARE_PACKED: bool> std::ops::Deref for TaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer, + T: Tag, +{ + type Target = P::Target; + fn deref(&self) -> &Self::Target { + self.raw.pointer_ref() + } +} + +impl<P, T, const COMPARE_PACKED: bool> std::ops::DerefMut for TaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer + std::ops::DerefMut, + T: Tag, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + self.raw.pointer_mut() + } +} + +impl<P, T, const COMPARE_PACKED: bool> Drop for TaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer, + T: Tag, +{ + fn drop(&mut self) { + // No need to drop the tag, as it's Copy + unsafe { + std::mem::drop(P::from_usize(self.raw.pointer_raw())); + } + } +} + +impl<P, T, const COMPARE_PACKED: bool> fmt::Debug for TaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer, + P::Target: fmt::Debug, + T: Tag + fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TaggedPtr") + .field("pointer", &self.pointer_ref()) + .field("tag", &self.tag()) + .finish() + } +} + +impl<P, T> PartialEq for TaggedPtr<P, T, true> +where + P: Pointer, + T: Tag, +{ + fn eq(&self, other: &Self) -> bool { + self.raw.eq(&other.raw) + } +} + +impl<P, T> Eq for TaggedPtr<P, T, true> +where + P: Pointer, + T: Tag, +{ +} + +impl<P, T> std::hash::Hash for TaggedPtr<P, T, true> +where + P: Pointer, + T: Tag, +{ + fn hash<H: std::hash::Hasher>(&self, state: &mut H) { + self.raw.hash(state); + } +} + +impl<P, T, HCX, const COMPARE_PACKED: bool> HashStable<HCX> for TaggedPtr<P, T, COMPARE_PACKED> +where + P: Pointer + HashStable<HCX>, + T: Tag + HashStable<HCX>, +{ + fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { + self.raw.hash_stable(hcx, hasher); + } +} diff --git a/compiler/rustc_data_structures/src/temp_dir.rs b/compiler/rustc_data_structures/src/temp_dir.rs new file mode 100644 index 000000000..a780d2386 --- /dev/null +++ b/compiler/rustc_data_structures/src/temp_dir.rs @@ -0,0 +1,34 @@ +use std::mem::ManuallyDrop; +use std::path::Path; +use tempfile::TempDir; + +/// This is used to avoid TempDir being dropped on error paths unintentionally. +#[derive(Debug)] +pub struct MaybeTempDir { + dir: ManuallyDrop<TempDir>, + // Whether the TempDir should be deleted on drop. + keep: bool, +} + +impl Drop for MaybeTempDir { + fn drop(&mut self) { + // SAFETY: We are in the destructor, and no further access will + // occur. + let dir = unsafe { ManuallyDrop::take(&mut self.dir) }; + if self.keep { + dir.into_path(); + } + } +} + +impl AsRef<Path> for MaybeTempDir { + fn as_ref(&self) -> &Path { + self.dir.path() + } +} + +impl MaybeTempDir { + pub fn new(dir: TempDir, keep_on_drop: bool) -> MaybeTempDir { + MaybeTempDir { dir: ManuallyDrop::new(dir), keep: keep_on_drop } + } +} diff --git a/compiler/rustc_data_structures/src/thin_vec.rs b/compiler/rustc_data_structures/src/thin_vec.rs new file mode 100644 index 000000000..716259142 --- /dev/null +++ b/compiler/rustc_data_structures/src/thin_vec.rs @@ -0,0 +1,135 @@ +use crate::stable_hasher::{HashStable, StableHasher}; + +use std::iter::FromIterator; + +/// A vector type optimized for cases where this size is usually 0 (cf. `SmallVec`). +/// The `Option<Box<..>>` wrapping allows us to represent a zero sized vector with `None`, +/// which uses only a single (null) pointer. +#[derive(Clone, Encodable, Decodable, Debug, Hash, Eq, PartialEq)] +pub struct ThinVec<T>(Option<Box<Vec<T>>>); + +impl<T> ThinVec<T> { + pub fn new() -> Self { + ThinVec(None) + } + + pub fn iter(&self) -> std::slice::Iter<'_, T> { + self.into_iter() + } + + pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> { + self.into_iter() + } + + pub fn push(&mut self, item: T) { + match *self { + ThinVec(Some(ref mut vec)) => vec.push(item), + ThinVec(None) => *self = vec![item].into(), + } + } +} + +impl<T> From<Vec<T>> for ThinVec<T> { + fn from(vec: Vec<T>) -> Self { + if vec.is_empty() { ThinVec(None) } else { ThinVec(Some(Box::new(vec))) } + } +} + +impl<T> Into<Vec<T>> for ThinVec<T> { + fn into(self) -> Vec<T> { + match self { + ThinVec(None) => Vec::new(), + ThinVec(Some(vec)) => *vec, + } + } +} + +impl<T> ::std::ops::Deref for ThinVec<T> { + type Target = [T]; + fn deref(&self) -> &[T] { + match *self { + ThinVec(None) => &[], + ThinVec(Some(ref vec)) => vec, + } + } +} + +impl<T> ::std::ops::DerefMut for ThinVec<T> { + fn deref_mut(&mut self) -> &mut [T] { + match *self { + ThinVec(None) => &mut [], + ThinVec(Some(ref mut vec)) => vec, + } + } +} + +impl<T> FromIterator<T> for ThinVec<T> { + fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self { + // `Vec::from_iter()` should not allocate if the iterator is empty. + let vec: Vec<_> = iter.into_iter().collect(); + if vec.is_empty() { ThinVec(None) } else { ThinVec(Some(Box::new(vec))) } + } +} + +impl<T> IntoIterator for ThinVec<T> { + type Item = T; + type IntoIter = std::vec::IntoIter<T>; + + fn into_iter(self) -> Self::IntoIter { + // This is still performant because `Vec::new()` does not allocate. + self.0.map_or_else(Vec::new, |ptr| *ptr).into_iter() + } +} + +impl<'a, T> IntoIterator for &'a ThinVec<T> { + type Item = &'a T; + type IntoIter = std::slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.as_ref().iter() + } +} + +impl<'a, T> IntoIterator for &'a mut ThinVec<T> { + type Item = &'a mut T; + type IntoIter = std::slice::IterMut<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.as_mut().iter_mut() + } +} + +impl<T> Extend<T> for ThinVec<T> { + fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) { + match *self { + ThinVec(Some(ref mut vec)) => vec.extend(iter), + ThinVec(None) => *self = iter.into_iter().collect::<Vec<_>>().into(), + } + } + + fn extend_one(&mut self, item: T) { + self.push(item) + } + + fn extend_reserve(&mut self, additional: usize) { + match *self { + ThinVec(Some(ref mut vec)) => vec.reserve(additional), + ThinVec(None) => *self = Vec::with_capacity(additional).into(), + } + } +} + +impl<T: HashStable<CTX>, CTX> HashStable<CTX> for ThinVec<T> { + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + (**self).hash_stable(hcx, hasher) + } +} + +impl<T> Default for ThinVec<T> { + fn default() -> Self { + Self(None) + } +} + +#[cfg(test)] +mod tests; diff --git a/compiler/rustc_data_structures/src/thin_vec/tests.rs b/compiler/rustc_data_structures/src/thin_vec/tests.rs new file mode 100644 index 000000000..0221b9912 --- /dev/null +++ b/compiler/rustc_data_structures/src/thin_vec/tests.rs @@ -0,0 +1,42 @@ +use super::*; + +impl<T> ThinVec<T> { + fn into_vec(self) -> Vec<T> { + self.into() + } +} + +#[test] +fn test_from_iterator() { + assert_eq!(std::iter::empty().collect::<ThinVec<String>>().into_vec(), Vec::<String>::new()); + assert_eq!(std::iter::once(42).collect::<ThinVec<_>>().into_vec(), vec![42]); + assert_eq!([1, 2].into_iter().collect::<ThinVec<_>>().into_vec(), vec![1, 2]); + assert_eq!([1, 2, 3].into_iter().collect::<ThinVec<_>>().into_vec(), vec![1, 2, 3]); +} + +#[test] +fn test_into_iterator_owned() { + assert_eq!(ThinVec::new().into_iter().collect::<Vec<String>>(), Vec::<String>::new()); + assert_eq!(ThinVec::from(vec![1]).into_iter().collect::<Vec<_>>(), vec![1]); + assert_eq!(ThinVec::from(vec![1, 2]).into_iter().collect::<Vec<_>>(), vec![1, 2]); + assert_eq!(ThinVec::from(vec![1, 2, 3]).into_iter().collect::<Vec<_>>(), vec![1, 2, 3]); +} + +#[test] +fn test_into_iterator_ref() { + assert_eq!(ThinVec::new().iter().collect::<Vec<&String>>(), Vec::<&String>::new()); + assert_eq!(ThinVec::from(vec![1]).iter().collect::<Vec<_>>(), vec![&1]); + assert_eq!(ThinVec::from(vec![1, 2]).iter().collect::<Vec<_>>(), vec![&1, &2]); + assert_eq!(ThinVec::from(vec![1, 2, 3]).iter().collect::<Vec<_>>(), vec![&1, &2, &3]); +} + +#[test] +fn test_into_iterator_ref_mut() { + assert_eq!(ThinVec::new().iter_mut().collect::<Vec<&mut String>>(), Vec::<&mut String>::new()); + assert_eq!(ThinVec::from(vec![1]).iter_mut().collect::<Vec<_>>(), vec![&mut 1]); + assert_eq!(ThinVec::from(vec![1, 2]).iter_mut().collect::<Vec<_>>(), vec![&mut 1, &mut 2]); + assert_eq!( + ThinVec::from(vec![1, 2, 3]).iter_mut().collect::<Vec<_>>(), + vec![&mut 1, &mut 2, &mut 3], + ); +} diff --git a/compiler/rustc_data_structures/src/tiny_list.rs b/compiler/rustc_data_structures/src/tiny_list.rs new file mode 100644 index 000000000..9b07f8684 --- /dev/null +++ b/compiler/rustc_data_structures/src/tiny_list.rs @@ -0,0 +1,81 @@ +//! A singly-linked list. +//! +//! Using this data structure only makes sense under very specific +//! circumstances: +//! +//! - If you have a list that rarely stores more than one element, then this +//! data-structure can store the element without allocating and only uses as +//! much space as an `Option<(T, usize)>`. If T can double as the `Option` +//! discriminant, it will even only be as large as `T, usize`. +//! +//! If you expect to store more than 1 element in the common case, steer clear +//! and use a `Vec<T>`, `Box<[T]>`, or a `SmallVec<T>`. + +#[cfg(test)] +mod tests; + +#[derive(Clone)] +pub struct TinyList<T> { + head: Option<Element<T>>, +} + +impl<T: PartialEq> TinyList<T> { + #[inline] + pub fn new() -> TinyList<T> { + TinyList { head: None } + } + + #[inline] + pub fn new_single(data: T) -> TinyList<T> { + TinyList { head: Some(Element { data, next: None }) } + } + + #[inline] + pub fn insert(&mut self, data: T) { + self.head = Some(Element { data, next: self.head.take().map(Box::new) }); + } + + #[inline] + pub fn remove(&mut self, data: &T) -> bool { + self.head = match self.head { + Some(ref mut head) if head.data == *data => head.next.take().map(|x| *x), + Some(ref mut head) => return head.remove_next(data), + None => return false, + }; + true + } + + #[inline] + pub fn contains(&self, data: &T) -> bool { + let mut elem = self.head.as_ref(); + while let Some(ref e) = elem { + if &e.data == data { + return true; + } + elem = e.next.as_deref(); + } + false + } +} + +#[derive(Clone)] +struct Element<T> { + data: T, + next: Option<Box<Element<T>>>, +} + +impl<T: PartialEq> Element<T> { + fn remove_next(&mut self, data: &T) -> bool { + let mut n = self; + loop { + match n.next { + Some(ref mut next) if next.data == *data => { + n.next = next.next.take(); + return true; + } + Some(ref mut next) => n = next, + None => return false, + } + } + } +} diff --git a/compiler/rustc_data_structures/src/tiny_list/tests.rs b/compiler/rustc_data_structures/src/tiny_list/tests.rs new file mode 100644 index 000000000..c0334d2e2 --- /dev/null +++ b/compiler/rustc_data_structures/src/tiny_list/tests.rs @@ -0,0 +1,155 @@ +use super::*; + +extern crate test; +use test::{black_box, Bencher}; + +impl<T> TinyList<T> { + fn len(&self) -> usize { + let (mut elem, mut count) = (self.head.as_ref(), 0); + while let Some(ref e) = elem { + count += 1; + elem = e.next.as_deref(); + } + count + } +} + +#[test] +fn test_contains_and_insert() { + fn do_insert(i: u32) -> bool { + i % 2 == 0 + } + + let mut list = TinyList::new(); + + for i in 0..10 { + for j in 0..i { + if do_insert(j) { + assert!(list.contains(&j)); + } else { + assert!(!list.contains(&j)); + } + } + + assert!(!list.contains(&i)); + + if do_insert(i) { + list.insert(i); + assert!(list.contains(&i)); + } + } +} + +#[test] +fn test_remove_first() { + let mut list = TinyList::new(); + list.insert(1); + list.insert(2); + list.insert(3); + list.insert(4); + assert_eq!(list.len(), 4); + + assert!(list.remove(&4)); + assert!(!list.contains(&4)); + + assert_eq!(list.len(), 3); + assert!(list.contains(&1)); + assert!(list.contains(&2)); + assert!(list.contains(&3)); +} + +#[test] +fn test_remove_last() { + let mut list = TinyList::new(); + list.insert(1); + list.insert(2); + list.insert(3); + list.insert(4); + assert_eq!(list.len(), 4); + + assert!(list.remove(&1)); + assert!(!list.contains(&1)); + + assert_eq!(list.len(), 3); + assert!(list.contains(&2)); + assert!(list.contains(&3)); + assert!(list.contains(&4)); +} + +#[test] +fn test_remove_middle() { + let mut list = TinyList::new(); + list.insert(1); + list.insert(2); + list.insert(3); + list.insert(4); + assert_eq!(list.len(), 4); + + assert!(list.remove(&2)); + assert!(!list.contains(&2)); + + assert_eq!(list.len(), 3); + assert!(list.contains(&1)); + assert!(list.contains(&3)); + assert!(list.contains(&4)); +} + +#[test] +fn test_remove_single() { + let mut list = TinyList::new(); + list.insert(1); + assert_eq!(list.len(), 1); + + assert!(list.remove(&1)); + assert!(!list.contains(&1)); + + assert_eq!(list.len(), 0); +} + +#[bench] +fn bench_insert_empty(b: &mut Bencher) { + b.iter(|| { + let mut list = black_box(TinyList::new()); + list.insert(1); + list + }) +} + +#[bench] +fn bench_insert_one(b: &mut Bencher) { + b.iter(|| { + let mut list = black_box(TinyList::new_single(0)); + list.insert(1); + list + }) +} + +#[bench] +fn bench_contains_empty(b: &mut Bencher) { + b.iter(|| black_box(TinyList::new()).contains(&1)); +} + +#[bench] +fn bench_contains_unknown(b: &mut Bencher) { + b.iter(|| black_box(TinyList::new_single(0)).contains(&1)); +} + +#[bench] +fn bench_contains_one(b: &mut Bencher) { + b.iter(|| black_box(TinyList::new_single(1)).contains(&1)); +} + +#[bench] +fn bench_remove_empty(b: &mut Bencher) { + b.iter(|| black_box(TinyList::new()).remove(&1)); +} + +#[bench] +fn bench_remove_unknown(b: &mut Bencher) { + b.iter(|| black_box(TinyList::new_single(0)).remove(&1)); +} + +#[bench] +fn bench_remove_one(b: &mut Bencher) { + b.iter(|| black_box(TinyList::new_single(1)).remove(&1)); +} diff --git a/compiler/rustc_data_structures/src/transitive_relation.rs b/compiler/rustc_data_structures/src/transitive_relation.rs new file mode 100644 index 000000000..0ff64969b --- /dev/null +++ b/compiler/rustc_data_structures/src/transitive_relation.rs @@ -0,0 +1,392 @@ +use crate::fx::FxIndexSet; +use crate::sync::Lock; +use rustc_index::bit_set::BitMatrix; +use std::fmt::Debug; +use std::hash::Hash; +use std::mem; + +#[cfg(test)] +mod tests; + +#[derive(Clone, Debug)] +pub struct TransitiveRelation<T> { + // List of elements. This is used to map from a T to a usize. + elements: FxIndexSet<T>, + + // List of base edges in the graph. Require to compute transitive + // closure. + edges: Vec<Edge>, + + // This is a cached transitive closure derived from the edges. + // Currently, we build it lazily and just throw out any existing + // copy whenever a new edge is added. (The Lock is to permit + // the lazy computation.) This is kind of silly, except for the + // fact its size is tied to `self.elements.len()`, so I wanted to + // wait before building it up to avoid reallocating as new edges + // are added with new elements. Perhaps better would be to ask the + // user for a batch of edges to minimize this effect, but I + // already wrote the code this way. :P -nmatsakis + closure: Lock<Option<BitMatrix<usize, usize>>>, +} + +// HACK(eddyb) manual impl avoids `Default` bound on `T`. +impl<T: Eq + Hash> Default for TransitiveRelation<T> { + fn default() -> Self { + TransitiveRelation { + elements: Default::default(), + edges: Default::default(), + closure: Default::default(), + } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +struct Index(usize); + +#[derive(Clone, PartialEq, Eq, Debug)] +struct Edge { + source: Index, + target: Index, +} + +impl<T: Eq + Hash + Copy> TransitiveRelation<T> { + pub fn is_empty(&self) -> bool { + self.edges.is_empty() + } + + pub fn elements(&self) -> impl Iterator<Item = &T> { + self.elements.iter() + } + + fn index(&self, a: T) -> Option<Index> { + self.elements.get_index_of(&a).map(Index) + } + + fn add_index(&mut self, a: T) -> Index { + let (index, added) = self.elements.insert_full(a); + if added { + // if we changed the dimensions, clear the cache + *self.closure.get_mut() = None; + } + Index(index) + } + + /// Applies the (partial) function to each edge and returns a new + /// relation. If `f` returns `None` for any end-point, returns + /// `None`. + pub fn maybe_map<F, U>(&self, mut f: F) -> Option<TransitiveRelation<U>> + where + F: FnMut(T) -> Option<U>, + U: Clone + Debug + Eq + Hash + Copy, + { + let mut result = TransitiveRelation::default(); + for edge in &self.edges { + result.add(f(self.elements[edge.source.0])?, f(self.elements[edge.target.0])?); + } + Some(result) + } + + /// Indicate that `a < b` (where `<` is this relation) + pub fn add(&mut self, a: T, b: T) { + let a = self.add_index(a); + let b = self.add_index(b); + let edge = Edge { source: a, target: b }; + if !self.edges.contains(&edge) { + self.edges.push(edge); + + // added an edge, clear the cache + *self.closure.get_mut() = None; + } + } + + /// Checks whether `a < target` (transitively) + pub fn contains(&self, a: T, b: T) -> bool { + match (self.index(a), self.index(b)) { + (Some(a), Some(b)) => self.with_closure(|closure| closure.contains(a.0, b.0)), + (None, _) | (_, None) => false, + } + } + + /// Thinking of `x R y` as an edge `x -> y` in a graph, this + /// returns all things reachable from `a`. + /// + /// Really this probably ought to be `impl Iterator<Item = &T>`, but + /// I'm too lazy to make that work, and -- given the caching + /// strategy -- it'd be a touch tricky anyhow. + pub fn reachable_from(&self, a: T) -> Vec<T> { + match self.index(a) { + Some(a) => { + self.with_closure(|closure| closure.iter(a.0).map(|i| self.elements[i]).collect()) + } + None => vec![], + } + } + + /// Picks what I am referring to as the "postdominating" + /// upper-bound for `a` and `b`. This is usually the least upper + /// bound, but in cases where there is no single least upper + /// bound, it is the "mutual immediate postdominator", if you + /// imagine a graph where `a < b` means `a -> b`. + /// + /// This function is needed because region inference currently + /// requires that we produce a single "UB", and there is no best + /// choice for the LUB. Rather than pick arbitrarily, I pick a + /// less good, but predictable choice. This should help ensure + /// that region inference yields predictable results (though it + /// itself is not fully sufficient). + /// + /// Examples are probably clearer than any prose I could write + /// (there are corresponding tests below, btw). In each case, + /// the query is `postdom_upper_bound(a, b)`: + /// + /// ```text + /// // Returns Some(x), which is also LUB. + /// a -> a1 -> x + /// ^ + /// | + /// b -> b1 ---+ + /// + /// // Returns `Some(x)`, which is not LUB (there is none) + /// // diagonal edges run left-to-right. + /// a -> a1 -> x + /// \/ ^ + /// /\ | + /// b -> b1 ---+ + /// + /// // Returns `None`. + /// a -> a1 + /// b -> b1 + /// ``` + pub fn postdom_upper_bound(&self, a: T, b: T) -> Option<T> { + let mubs = self.minimal_upper_bounds(a, b); + self.mutual_immediate_postdominator(mubs) + } + + /// Viewing the relation as a graph, computes the "mutual + /// immediate postdominator" of a set of points (if one + /// exists). See `postdom_upper_bound` for details. + pub fn mutual_immediate_postdominator<'a>(&'a self, mut mubs: Vec<T>) -> Option<T> { + loop { + match mubs.len() { + 0 => return None, + 1 => return Some(mubs[0]), + _ => { + let m = mubs.pop().unwrap(); + let n = mubs.pop().unwrap(); + mubs.extend(self.minimal_upper_bounds(n, m)); + } + } + } + } + + /// Returns the set of bounds `X` such that: + /// + /// - `a < X` and `b < X` + /// - there is no `Y != X` such that `a < Y` and `Y < X` + /// - except for the case where `X < a` (i.e., a strongly connected + /// component in the graph). In that case, the smallest + /// representative of the SCC is returned (as determined by the + /// internal indices). + /// + /// Note that this set can, in principle, have any size. + pub fn minimal_upper_bounds(&self, a: T, b: T) -> Vec<T> { + let (Some(mut a), Some(mut b)) = (self.index(a), self.index(b)) else { + return vec![]; + }; + + // in some cases, there are some arbitrary choices to be made; + // it doesn't really matter what we pick, as long as we pick + // the same thing consistently when queried, so ensure that + // (a, b) are in a consistent relative order + if a > b { + mem::swap(&mut a, &mut b); + } + + let lub_indices = self.with_closure(|closure| { + // Easy case is when either a < b or b < a: + if closure.contains(a.0, b.0) { + return vec![b.0]; + } + if closure.contains(b.0, a.0) { + return vec![a.0]; + } + + // Otherwise, the tricky part is that there may be some c + // where a < c and b < c. In fact, there may be many such + // values. So here is what we do: + // + // 1. Find the vector `[X | a < X && b < X]` of all values + // `X` where `a < X` and `b < X`. In terms of the + // graph, this means all values reachable from both `a` + // and `b`. Note that this vector is also a set, but we + // use the term vector because the order matters + // to the steps below. + // - This vector contains upper bounds, but they are + // not minimal upper bounds. So you may have e.g. + // `[x, y, tcx, z]` where `x < tcx` and `y < tcx` and + // `z < x` and `z < y`: + // + // z --+---> x ----+----> tcx + // | | + // | | + // +---> y ----+ + // + // In this case, we really want to return just `[z]`. + // The following steps below achieve this by gradually + // reducing the list. + // 2. Pare down the vector using `pare_down`. This will + // remove elements from the vector that can be reached + // by an earlier element. + // - In the example above, this would convert `[x, y, + // tcx, z]` to `[x, y, z]`. Note that `x` and `y` are + // still in the vector; this is because while `z < x` + // (and `z < y`) holds, `z` comes after them in the + // vector. + // 3. Reverse the vector and repeat the pare down process. + // - In the example above, we would reverse to + // `[z, y, x]` and then pare down to `[z]`. + // 4. Reverse once more just so that we yield a vector in + // increasing order of index. Not necessary, but why not. + // + // I believe this algorithm yields a minimal set. The + // argument is that, after step 2, we know that no element + // can reach its successors (in the vector, not the graph). + // After step 3, we know that no element can reach any of + // its predecessors (because of step 2) nor successors + // (because we just called `pare_down`) + // + // This same algorithm is used in `parents` below. + + let mut candidates = closure.intersect_rows(a.0, b.0); // (1) + pare_down(&mut candidates, closure); // (2) + candidates.reverse(); // (3a) + pare_down(&mut candidates, closure); // (3b) + candidates + }); + + lub_indices + .into_iter() + .rev() // (4) + .map(|i| self.elements[i]) + .collect() + } + + /// Given an element A, returns the maximal set {B} of elements B + /// such that + /// + /// - A != B + /// - A R B is true + /// - for each i, j: `B[i]` R `B[j]` does not hold + /// + /// The intuition is that this moves "one step up" through a lattice + /// (where the relation is encoding the `<=` relation for the lattice). + /// So e.g., if the relation is `->` and we have + /// + /// ```text + /// a -> b -> d -> f + /// | ^ + /// +--> c -> e ---+ + /// ``` + /// + /// then `parents(a)` returns `[b, c]`. The `postdom_parent` function + /// would further reduce this to just `f`. + pub fn parents(&self, a: T) -> Vec<T> { + let Some(a) = self.index(a) else { + return vec![]; + }; + + // Steal the algorithm for `minimal_upper_bounds` above, but + // with a slight tweak. In the case where `a R a`, we remove + // that from the set of candidates. + let ancestors = self.with_closure(|closure| { + let mut ancestors = closure.intersect_rows(a.0, a.0); + + // Remove anything that can reach `a`. If this is a + // reflexive relation, this will include `a` itself. + ancestors.retain(|&e| !closure.contains(e, a.0)); + + pare_down(&mut ancestors, closure); // (2) + ancestors.reverse(); // (3a) + pare_down(&mut ancestors, closure); // (3b) + ancestors + }); + + ancestors + .into_iter() + .rev() // (4) + .map(|i| self.elements[i]) + .collect() + } + + fn with_closure<OP, R>(&self, op: OP) -> R + where + OP: FnOnce(&BitMatrix<usize, usize>) -> R, + { + let mut closure_cell = self.closure.borrow_mut(); + let mut closure = closure_cell.take(); + if closure.is_none() { + closure = Some(self.compute_closure()); + } + let result = op(closure.as_ref().unwrap()); + *closure_cell = closure; + result + } + + fn compute_closure(&self) -> BitMatrix<usize, usize> { + let mut matrix = BitMatrix::new(self.elements.len(), self.elements.len()); + let mut changed = true; + while changed { + changed = false; + for edge in &self.edges { + // add an edge from S -> T + changed |= matrix.insert(edge.source.0, edge.target.0); + + // add all outgoing edges from T into S + changed |= matrix.union_rows(edge.target.0, edge.source.0); + } + } + matrix + } + + /// Lists all the base edges in the graph: the initial _non-transitive_ set of element + /// relations, which will be later used as the basis for the transitive closure computation. + pub fn base_edges(&self) -> impl Iterator<Item = (T, T)> + '_ { + self.edges + .iter() + .map(move |edge| (self.elements[edge.source.0], self.elements[edge.target.0])) + } +} + +/// Pare down is used as a step in the LUB computation. It edits the +/// candidates array in place by removing any element j for which +/// there exists an earlier element i<j such that i -> j. That is, +/// after you run `pare_down`, you know that for all elements that +/// remain in candidates, they cannot reach any of the elements that +/// come after them. +/// +/// Examples follow. Assume that a -> b -> c and x -> y -> z. +/// +/// - Input: `[a, b, x]`. Output: `[a, x]`. +/// - Input: `[b, a, x]`. Output: `[b, a, x]`. +/// - Input: `[a, x, b, y]`. Output: `[a, x]`. +fn pare_down(candidates: &mut Vec<usize>, closure: &BitMatrix<usize, usize>) { + let mut i = 0; + while let Some(&candidate_i) = candidates.get(i) { + i += 1; + + let mut j = i; + let mut dead = 0; + while let Some(&candidate_j) = candidates.get(j) { + if closure.contains(candidate_i, candidate_j) { + // If `i` can reach `j`, then we can remove `j`. So just + // mark it as dead and move on; subsequent indices will be + // shifted into its place. + dead += 1; + } else { + candidates[j - dead] = candidate_j; + } + j += 1; + } + candidates.truncate(j - dead); + } +} diff --git a/compiler/rustc_data_structures/src/transitive_relation/tests.rs b/compiler/rustc_data_structures/src/transitive_relation/tests.rs new file mode 100644 index 000000000..e1f4c7ee0 --- /dev/null +++ b/compiler/rustc_data_structures/src/transitive_relation/tests.rs @@ -0,0 +1,362 @@ +use super::*; + +impl<T: Eq + Hash + Copy> TransitiveRelation<T> { + /// A "best" parent in some sense. See `parents` and + /// `postdom_upper_bound` for more details. + fn postdom_parent(&self, a: T) -> Option<T> { + self.mutual_immediate_postdominator(self.parents(a)) + } +} + +#[test] +fn test_one_step() { + let mut relation = TransitiveRelation::default(); + relation.add("a", "b"); + relation.add("a", "c"); + assert!(relation.contains("a", "c")); + assert!(relation.contains("a", "b")); + assert!(!relation.contains("b", "a")); + assert!(!relation.contains("a", "d")); +} + +#[test] +fn test_many_steps() { + let mut relation = TransitiveRelation::default(); + relation.add("a", "b"); + relation.add("a", "c"); + relation.add("a", "f"); + + relation.add("b", "c"); + relation.add("b", "d"); + relation.add("b", "e"); + + relation.add("e", "g"); + + assert!(relation.contains("a", "b")); + assert!(relation.contains("a", "c")); + assert!(relation.contains("a", "d")); + assert!(relation.contains("a", "e")); + assert!(relation.contains("a", "f")); + assert!(relation.contains("a", "g")); + + assert!(relation.contains("b", "g")); + + assert!(!relation.contains("a", "x")); + assert!(!relation.contains("b", "f")); +} + +#[test] +fn mubs_triangle() { + // a -> tcx + // ^ + // | + // b + let mut relation = TransitiveRelation::default(); + relation.add("a", "tcx"); + relation.add("b", "tcx"); + assert_eq!(relation.minimal_upper_bounds("a", "b"), vec!["tcx"]); + assert_eq!(relation.parents("a"), vec!["tcx"]); + assert_eq!(relation.parents("b"), vec!["tcx"]); +} + +#[test] +fn mubs_best_choice1() { + // 0 -> 1 <- 3 + // | ^ | + // | | | + // +--> 2 <--+ + // + // mubs(0,3) = [1] + + // This tests a particular state in the algorithm, in which we + // need the second pare down call to get the right result (after + // intersection, we have [1, 2], but 2 -> 1). + + let mut relation = TransitiveRelation::default(); + relation.add("0", "1"); + relation.add("0", "2"); + + relation.add("2", "1"); + + relation.add("3", "1"); + relation.add("3", "2"); + + assert_eq!(relation.minimal_upper_bounds("0", "3"), vec!["2"]); + assert_eq!(relation.parents("0"), vec!["2"]); + assert_eq!(relation.parents("2"), vec!["1"]); + assert!(relation.parents("1").is_empty()); +} + +#[test] +fn mubs_best_choice2() { + // 0 -> 1 <- 3 + // | | | + // | v | + // +--> 2 <--+ + // + // mubs(0,3) = [2] + + // Like the preceding test, but in this case intersection is [2, + // 1], and hence we rely on the first pare down call. + + let mut relation = TransitiveRelation::default(); + relation.add("0", "1"); + relation.add("0", "2"); + + relation.add("1", "2"); + + relation.add("3", "1"); + relation.add("3", "2"); + + assert_eq!(relation.minimal_upper_bounds("0", "3"), vec!["1"]); + assert_eq!(relation.parents("0"), vec!["1"]); + assert_eq!(relation.parents("1"), vec!["2"]); + assert!(relation.parents("2").is_empty()); +} + +#[test] +fn mubs_no_best_choice() { + // in this case, the intersection yields [1, 2], and the "pare + // down" calls find nothing to remove. + let mut relation = TransitiveRelation::default(); + relation.add("0", "1"); + relation.add("0", "2"); + + relation.add("3", "1"); + relation.add("3", "2"); + + assert_eq!(relation.minimal_upper_bounds("0", "3"), vec!["1", "2"]); + assert_eq!(relation.parents("0"), vec!["1", "2"]); + assert_eq!(relation.parents("3"), vec!["1", "2"]); +} + +#[test] +fn mubs_best_choice_scc() { + // in this case, 1 and 2 form a cycle; we pick arbitrarily (but + // consistently). + + let mut relation = TransitiveRelation::default(); + relation.add("0", "1"); + relation.add("0", "2"); + + relation.add("1", "2"); + relation.add("2", "1"); + + relation.add("3", "1"); + relation.add("3", "2"); + + assert_eq!(relation.minimal_upper_bounds("0", "3"), vec!["1"]); + assert_eq!(relation.parents("0"), vec!["1"]); +} + +#[test] +fn pdub_crisscross() { + // diagonal edges run left-to-right + // a -> a1 -> x + // \/ ^ + // /\ | + // b -> b1 ---+ + + let mut relation = TransitiveRelation::default(); + relation.add("a", "a1"); + relation.add("a", "b1"); + relation.add("b", "a1"); + relation.add("b", "b1"); + relation.add("a1", "x"); + relation.add("b1", "x"); + + assert_eq!(relation.minimal_upper_bounds("a", "b"), vec!["a1", "b1"]); + assert_eq!(relation.postdom_upper_bound("a", "b"), Some("x")); + assert_eq!(relation.postdom_parent("a"), Some("x")); + assert_eq!(relation.postdom_parent("b"), Some("x")); +} + +#[test] +fn pdub_crisscross_more() { + // diagonal edges run left-to-right + // a -> a1 -> a2 -> a3 -> x + // \/ \/ ^ + // /\ /\ | + // b -> b1 -> b2 ---------+ + + let mut relation = TransitiveRelation::default(); + relation.add("a", "a1"); + relation.add("a", "b1"); + relation.add("b", "a1"); + relation.add("b", "b1"); + + relation.add("a1", "a2"); + relation.add("a1", "b2"); + relation.add("b1", "a2"); + relation.add("b1", "b2"); + + relation.add("a2", "a3"); + + relation.add("a3", "x"); + relation.add("b2", "x"); + + assert_eq!(relation.minimal_upper_bounds("a", "b"), vec!["a1", "b1"]); + assert_eq!(relation.minimal_upper_bounds("a1", "b1"), vec!["a2", "b2"]); + assert_eq!(relation.postdom_upper_bound("a", "b"), Some("x")); + + assert_eq!(relation.postdom_parent("a"), Some("x")); + assert_eq!(relation.postdom_parent("b"), Some("x")); +} + +#[test] +fn pdub_lub() { + // a -> a1 -> x + // ^ + // | + // b -> b1 ---+ + + let mut relation = TransitiveRelation::default(); + relation.add("a", "a1"); + relation.add("b", "b1"); + relation.add("a1", "x"); + relation.add("b1", "x"); + + assert_eq!(relation.minimal_upper_bounds("a", "b"), vec!["x"]); + assert_eq!(relation.postdom_upper_bound("a", "b"), Some("x")); + + assert_eq!(relation.postdom_parent("a"), Some("a1")); + assert_eq!(relation.postdom_parent("b"), Some("b1")); + assert_eq!(relation.postdom_parent("a1"), Some("x")); + assert_eq!(relation.postdom_parent("b1"), Some("x")); +} + +#[test] +fn mubs_intermediate_node_on_one_side_only() { + // a -> c -> d + // ^ + // | + // b + + // "digraph { a -> c -> d; b -> d; }", + let mut relation = TransitiveRelation::default(); + relation.add("a", "c"); + relation.add("c", "d"); + relation.add("b", "d"); + + assert_eq!(relation.minimal_upper_bounds("a", "b"), vec!["d"]); +} + +#[test] +fn mubs_scc_1() { + // +-------------+ + // | +----+ | + // | v | | + // a -> c -> d <-+ + // ^ + // | + // b + + // "digraph { a -> c -> d; d -> c; a -> d; b -> d; }", + let mut relation = TransitiveRelation::default(); + relation.add("a", "c"); + relation.add("c", "d"); + relation.add("d", "c"); + relation.add("a", "d"); + relation.add("b", "d"); + + assert_eq!(relation.minimal_upper_bounds("a", "b"), vec!["c"]); +} + +#[test] +fn mubs_scc_2() { + // +----+ + // v | + // a -> c -> d + // ^ ^ + // | | + // +--- b + + // "digraph { a -> c -> d; d -> c; b -> d; b -> c; }", + let mut relation = TransitiveRelation::default(); + relation.add("a", "c"); + relation.add("c", "d"); + relation.add("d", "c"); + relation.add("b", "d"); + relation.add("b", "c"); + + assert_eq!(relation.minimal_upper_bounds("a", "b"), vec!["c"]); +} + +#[test] +fn mubs_scc_3() { + // +---------+ + // v | + // a -> c -> d -> e + // ^ ^ + // | | + // b ---+ + + // "digraph { a -> c -> d -> e -> c; b -> d; b -> e; }", + let mut relation = TransitiveRelation::default(); + relation.add("a", "c"); + relation.add("c", "d"); + relation.add("d", "e"); + relation.add("e", "c"); + relation.add("b", "d"); + relation.add("b", "e"); + + assert_eq!(relation.minimal_upper_bounds("a", "b"), vec!["c"]); +} + +#[test] +fn mubs_scc_4() { + // +---------+ + // v | + // a -> c -> d -> e + // | ^ ^ + // +---------+ | + // | + // b ---+ + + // "digraph { a -> c -> d -> e -> c; a -> d; b -> e; }" + let mut relation = TransitiveRelation::default(); + relation.add("a", "c"); + relation.add("c", "d"); + relation.add("d", "e"); + relation.add("e", "c"); + relation.add("a", "d"); + relation.add("b", "e"); + + assert_eq!(relation.minimal_upper_bounds("a", "b"), vec!["c"]); +} + +#[test] +fn parent() { + // An example that was misbehaving in the compiler. + // + // 4 -> 1 -> 3 + // \ | / + // \ v / + // 2 -> 0 + // + // plus a bunch of self-loops + // + // Here `->` represents `<=` and `0` is `'static`. + + let pairs = vec![ + (2, /*->*/ 0), + (2, /*->*/ 2), + (0, /*->*/ 0), + (0, /*->*/ 0), + (1, /*->*/ 0), + (1, /*->*/ 1), + (3, /*->*/ 0), + (3, /*->*/ 3), + (4, /*->*/ 0), + (4, /*->*/ 1), + (1, /*->*/ 3), + ]; + + let mut relation = TransitiveRelation::default(); + for (a, b) in pairs { + relation.add(a, b); + } + + let p = relation.postdom_parent(3); + assert_eq!(p, Some(0)); +} diff --git a/compiler/rustc_data_structures/src/unhash.rs b/compiler/rustc_data_structures/src/unhash.rs new file mode 100644 index 000000000..48e21a9da --- /dev/null +++ b/compiler/rustc_data_structures/src/unhash.rs @@ -0,0 +1,29 @@ +use std::collections::{HashMap, HashSet}; +use std::hash::{BuildHasherDefault, Hasher}; + +pub type UnhashMap<K, V> = HashMap<K, V, BuildHasherDefault<Unhasher>>; +pub type UnhashSet<V> = HashSet<V, BuildHasherDefault<Unhasher>>; + +/// This no-op hasher expects only a single `write_u64` call. It's intended for +/// map keys that already have hash-like quality, like `Fingerprint`. +#[derive(Default)] +pub struct Unhasher { + value: u64, +} + +impl Hasher for Unhasher { + #[inline] + fn finish(&self) -> u64 { + self.value + } + + fn write(&mut self, _bytes: &[u8]) { + unimplemented!("use write_u64"); + } + + #[inline] + fn write_u64(&mut self, value: u64) { + debug_assert_eq!(0, self.value, "Unhasher doesn't mix values!"); + self.value = value; + } +} diff --git a/compiler/rustc_data_structures/src/vec_linked_list.rs b/compiler/rustc_data_structures/src/vec_linked_list.rs new file mode 100644 index 000000000..ce60d40b2 --- /dev/null +++ b/compiler/rustc_data_structures/src/vec_linked_list.rs @@ -0,0 +1,70 @@ +use rustc_index::vec::{Idx, IndexVec}; + +pub fn iter<Ls>( + first: Option<Ls::LinkIndex>, + links: &Ls, +) -> impl Iterator<Item = Ls::LinkIndex> + '_ +where + Ls: Links, +{ + VecLinkedListIterator { links, current: first } +} + +pub struct VecLinkedListIterator<Ls> +where + Ls: Links, +{ + links: Ls, + current: Option<Ls::LinkIndex>, +} + +impl<Ls> Iterator for VecLinkedListIterator<Ls> +where + Ls: Links, +{ + type Item = Ls::LinkIndex; + + fn next(&mut self) -> Option<Ls::LinkIndex> { + if let Some(c) = self.current { + self.current = <Ls as Links>::next(&self.links, c); + Some(c) + } else { + None + } + } +} + +pub trait Links { + type LinkIndex: Copy; + + fn next(links: &Self, index: Self::LinkIndex) -> Option<Self::LinkIndex>; +} + +impl<Ls> Links for &Ls +where + Ls: Links, +{ + type LinkIndex = Ls::LinkIndex; + + fn next(links: &Self, index: Ls::LinkIndex) -> Option<Ls::LinkIndex> { + <Ls as Links>::next(links, index) + } +} + +pub trait LinkElem { + type LinkIndex: Copy; + + fn next(elem: &Self) -> Option<Self::LinkIndex>; +} + +impl<L, E> Links for IndexVec<L, E> +where + E: LinkElem<LinkIndex = L>, + L: Idx, +{ + type LinkIndex = L; + + fn next(links: &Self, index: L) -> Option<L> { + <E as LinkElem>::next(&links[index]) + } +} diff --git a/compiler/rustc_data_structures/src/vec_map.rs b/compiler/rustc_data_structures/src/vec_map.rs new file mode 100644 index 000000000..86be0bd87 --- /dev/null +++ b/compiler/rustc_data_structures/src/vec_map.rs @@ -0,0 +1,194 @@ +use std::borrow::Borrow; +use std::fmt::Debug; +use std::iter::FromIterator; +use std::slice::Iter; +use std::vec::IntoIter; + +use crate::stable_hasher::{HashStable, StableHasher}; + +/// A map type implemented as a vector of pairs `K` (key) and `V` (value). +/// It currently provides a subset of all the map operations, the rest could be added as needed. +#[derive(Clone, Encodable, Decodable, Debug)] +pub struct VecMap<K, V>(Vec<(K, V)>); + +impl<K, V> VecMap<K, V> +where + K: Debug + PartialEq, + V: Debug, +{ + pub fn new() -> Self { + VecMap(Default::default()) + } + + /// Sets the value of the entry, and returns the entry's old value. + pub fn insert(&mut self, k: K, v: V) -> Option<V> { + if let Some(elem) = self.0.iter_mut().find(|(key, _)| *key == k) { + Some(std::mem::replace(&mut elem.1, v)) + } else { + self.0.push((k, v)); + None + } + } + + /// Removes the entry from the map and returns the removed value + pub fn remove(&mut self, k: &K) -> Option<V> { + self.0.iter().position(|(k2, _)| k2 == k).map(|pos| self.0.remove(pos).1) + } + + /// Gets a reference to the value in the entry. + pub fn get<Q: ?Sized>(&self, k: &Q) -> Option<&V> + where + K: Borrow<Q>, + Q: Eq, + { + self.0.iter().find(|(key, _)| k == key.borrow()).map(|elem| &elem.1) + } + + /// Gets a mutable reference to the value in the entry. + pub fn get_mut<Q: ?Sized>(&mut self, k: &Q) -> Option<&mut V> + where + K: Borrow<Q>, + Q: Eq, + { + self.0.iter_mut().find(|(key, _)| k == key.borrow()).map(|elem| &mut elem.1) + } + + /// Returns the any value corresponding to the supplied predicate filter. + /// + /// The supplied predicate will be applied to each (key, value) pair and it will return a + /// reference to the values where the predicate returns `true`. + pub fn any_value_matching(&self, mut predicate: impl FnMut(&(K, V)) -> bool) -> Option<&V> { + self.0.iter().find(|kv| predicate(kv)).map(|elem| &elem.1) + } + + /// Returns the value corresponding to the supplied predicate filter. It crashes if there's + /// more than one matching element. + /// + /// The supplied predicate will be applied to each (key, value) pair and it will return a + /// reference to the value where the predicate returns `true`. + pub fn get_value_matching(&self, mut predicate: impl FnMut(&(K, V)) -> bool) -> Option<&V> { + let mut filter = self.0.iter().filter(|kv| predicate(kv)); + let (_, value) = filter.next()?; + // This should return just one element, otherwise it's a bug + assert!( + filter.next().is_none(), + "Collection {:#?} should have just one matching element", + self + ); + Some(value) + } + + /// Returns `true` if the map contains a value for the specified key. + /// + /// The key may be any borrowed form of the map's key type, + /// [`Eq`] on the borrowed form *must* match those for + /// the key type. + pub fn contains_key<Q: ?Sized>(&self, k: &Q) -> bool + where + K: Borrow<Q>, + Q: Eq, + { + self.get(k).is_some() + } + + /// Returns `true` if the map contains no elements. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn iter(&self) -> Iter<'_, (K, V)> { + self.into_iter() + } + + pub fn iter_mut(&mut self) -> impl Iterator<Item = (&K, &mut V)> { + self.into_iter() + } + + pub fn retain(&mut self, f: impl Fn(&(K, V)) -> bool) { + self.0.retain(f) + } +} + +impl<K, V> Default for VecMap<K, V> { + #[inline] + fn default() -> Self { + Self(Default::default()) + } +} + +impl<K, V> From<Vec<(K, V)>> for VecMap<K, V> { + fn from(vec: Vec<(K, V)>) -> Self { + Self(vec) + } +} + +impl<K, V> Into<Vec<(K, V)>> for VecMap<K, V> { + fn into(self) -> Vec<(K, V)> { + self.0 + } +} + +impl<K, V> FromIterator<(K, V)> for VecMap<K, V> { + fn from_iter<I: IntoIterator<Item = (K, V)>>(iter: I) -> Self { + Self(iter.into_iter().collect()) + } +} + +impl<'a, K, V> IntoIterator for &'a VecMap<K, V> { + type Item = &'a (K, V); + type IntoIter = Iter<'a, (K, V)>; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.0.iter() + } +} + +impl<'a, K: 'a, V: 'a> IntoIterator for &'a mut VecMap<K, V> { + type Item = (&'a K, &'a mut V); + type IntoIter = impl Iterator<Item = Self::Item>; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.0.iter_mut().map(|(k, v)| (&*k, v)) + } +} + +impl<K, V> IntoIterator for VecMap<K, V> { + type Item = (K, V); + type IntoIter = IntoIter<(K, V)>; + + #[inline] + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl<K: PartialEq + Debug, V: Debug> Extend<(K, V)> for VecMap<K, V> { + fn extend<I: IntoIterator<Item = (K, V)>>(&mut self, iter: I) { + for (k, v) in iter { + self.insert(k, v); + } + } + + fn extend_one(&mut self, (k, v): (K, V)) { + self.insert(k, v); + } + + fn extend_reserve(&mut self, additional: usize) { + self.0.extend_reserve(additional); + } +} + +impl<K, V, CTX> HashStable<CTX> for VecMap<K, V> +where + K: HashStable<CTX> + Eq, + V: HashStable<CTX>, +{ + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + self.0.hash_stable(hcx, hasher) + } +} + +#[cfg(test)] +mod tests; diff --git a/compiler/rustc_data_structures/src/vec_map/tests.rs b/compiler/rustc_data_structures/src/vec_map/tests.rs new file mode 100644 index 000000000..458b60077 --- /dev/null +++ b/compiler/rustc_data_structures/src/vec_map/tests.rs @@ -0,0 +1,48 @@ +use super::*; + +impl<K, V> VecMap<K, V> { + fn into_vec(self) -> Vec<(K, V)> { + self.0.into() + } +} + +#[test] +fn test_from_iterator() { + assert_eq!( + std::iter::empty().collect::<VecMap<i32, bool>>().into_vec(), + Vec::<(i32, bool)>::new() + ); + assert_eq!(std::iter::once((42, true)).collect::<VecMap<_, _>>().into_vec(), vec![(42, true)]); + assert_eq!( + [(1, true), (2, false)].into_iter().collect::<VecMap<_, _>>().into_vec(), + vec![(1, true), (2, false)] + ); +} + +#[test] +fn test_into_iterator_owned() { + assert_eq!(VecMap::new().into_iter().collect::<Vec<(i32, bool)>>(), Vec::<(i32, bool)>::new()); + assert_eq!(VecMap::from(vec![(1, true)]).into_iter().collect::<Vec<_>>(), vec![(1, true)]); + assert_eq!( + VecMap::from(vec![(1, true), (2, false)]).into_iter().collect::<Vec<_>>(), + vec![(1, true), (2, false)] + ); +} + +#[test] +fn test_insert() { + let mut v = VecMap::new(); + assert_eq!(v.insert(1, true), None); + assert_eq!(v.insert(2, false), None); + assert_eq!(v.clone().into_vec(), vec![(1, true), (2, false)]); + assert_eq!(v.insert(1, false), Some(true)); + assert_eq!(v.into_vec(), vec![(1, false), (2, false)]); +} + +#[test] +fn test_get() { + let v = [(1, true), (2, false)].into_iter().collect::<VecMap<_, _>>(); + assert_eq!(v.get(&1), Some(&true)); + assert_eq!(v.get(&2), Some(&false)); + assert_eq!(v.get(&3), None); +} diff --git a/compiler/rustc_data_structures/src/work_queue.rs b/compiler/rustc_data_structures/src/work_queue.rs new file mode 100644 index 000000000..10317f1af --- /dev/null +++ b/compiler/rustc_data_structures/src/work_queue.rs @@ -0,0 +1,44 @@ +use rustc_index::bit_set::BitSet; +use rustc_index::vec::Idx; +use std::collections::VecDeque; + +/// A work queue is a handy data structure for tracking work left to +/// do. (For example, basic blocks left to process.) It is basically a +/// de-duplicating queue; so attempting to insert X if X is already +/// enqueued has no effect. This implementation assumes that the +/// elements are dense indices, so it can allocate the queue to size +/// and also use a bit set to track occupancy. +pub struct WorkQueue<T: Idx> { + deque: VecDeque<T>, + set: BitSet<T>, +} + +impl<T: Idx> WorkQueue<T> { + /// Creates a new work queue that starts empty, where elements range from (0..len). + #[inline] + pub fn with_none(len: usize) -> Self { + WorkQueue { deque: VecDeque::with_capacity(len), set: BitSet::new_empty(len) } + } + + /// Attempt to enqueue `element` in the work queue. Returns false if it was already present. + #[inline] + pub fn insert(&mut self, element: T) -> bool { + if self.set.insert(element) { + self.deque.push_back(element); + true + } else { + false + } + } + + /// Attempt to pop an element from the work queue. + #[inline] + pub fn pop(&mut self) -> Option<T> { + if let Some(element) = self.deque.pop_front() { + self.set.remove(element); + Some(element) + } else { + None + } + } +} |