diff options
Diffstat (limited to 'library/std/src/sys_common')
| -rw-r--r-- | library/std/src/sys_common/net.rs | 8 | ||||
| -rw-r--r-- | library/std/src/sys_common/remutex.rs | 46 | ||||
| -rw-r--r-- | library/std/src/sys_common/remutex/tests.rs | 37 | ||||
| -rw-r--r-- | library/std/src/sys_common/thread_local_key.rs | 2 | ||||
| -rw-r--r-- | library/std/src/sys_common/thread_local_key/tests.rs | 9 | ||||
| -rw-r--r-- | library/std/src/sys_common/thread_parker/mod.rs | 1 | ||||
| -rw-r--r-- | library/std/src/sys_common/wtf8.rs | 95 | ||||
| -rw-r--r-- | library/std/src/sys_common/wtf8/tests.rs | 295 |
8 files changed, 390 insertions, 103 deletions
diff --git a/library/std/src/sys_common/net.rs b/library/std/src/sys_common/net.rs index 33d336c43..3ad802afa 100644 --- a/library/std/src/sys_common/net.rs +++ b/library/std/src/sys_common/net.rs @@ -10,7 +10,7 @@ use crate::net::{Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr}; use crate::ptr; use crate::sys::net::netc as c; use crate::sys::net::{cvt, cvt_gai, cvt_r, init, wrlen_t, Socket}; -use crate::sys_common::{FromInner, IntoInner}; +use crate::sys_common::{AsInner, FromInner, IntoInner}; use crate::time::Duration; use libc::{c_int, c_void}; @@ -345,6 +345,12 @@ impl TcpStream { } } +impl AsInner<Socket> for TcpStream { + fn as_inner(&self) -> &Socket { + &self.inner + } +} + impl FromInner<Socket> for TcpStream { fn from_inner(socket: Socket) -> TcpStream { TcpStream { inner: socket } diff --git a/library/std/src/sys_common/remutex.rs b/library/std/src/sys_common/remutex.rs index 8921af311..b448ae3a9 100644 --- a/library/std/src/sys_common/remutex.rs +++ b/library/std/src/sys_common/remutex.rs @@ -1,13 +1,11 @@ #[cfg(all(test, not(target_os = "emscripten")))] mod tests; +use super::mutex as sys; use crate::cell::UnsafeCell; -use crate::marker::PhantomPinned; use crate::ops::Deref; use crate::panic::{RefUnwindSafe, UnwindSafe}; -use crate::pin::Pin; use crate::sync::atomic::{AtomicUsize, Ordering::Relaxed}; -use crate::sys::locks as sys; /// A re-entrant mutual exclusion /// @@ -41,11 +39,10 @@ use crate::sys::locks as sys; /// synchronization is left to the mutex, making relaxed memory ordering for /// the `owner` field fine in all cases. pub struct ReentrantMutex<T> { - mutex: sys::Mutex, + mutex: sys::MovableMutex, owner: AtomicUsize, lock_count: UnsafeCell<u32>, data: T, - _pinned: PhantomPinned, } unsafe impl<T: Send> Send for ReentrantMutex<T> {} @@ -68,39 +65,22 @@ impl<T> RefUnwindSafe for ReentrantMutex<T> {} /// guarded data. #[must_use = "if unused the ReentrantMutex will immediately unlock"] pub struct ReentrantMutexGuard<'a, T: 'a> { - lock: Pin<&'a ReentrantMutex<T>>, + lock: &'a ReentrantMutex<T>, } impl<T> !Send for ReentrantMutexGuard<'_, T> {} impl<T> ReentrantMutex<T> { /// Creates a new reentrant mutex in an unlocked state. - /// - /// # Unsafety - /// - /// This function is unsafe because it is required that `init` is called - /// once this mutex is in its final resting place, and only then are the - /// lock/unlock methods safe. - pub const unsafe fn new(t: T) -> ReentrantMutex<T> { + pub const fn new(t: T) -> ReentrantMutex<T> { ReentrantMutex { - mutex: sys::Mutex::new(), + mutex: sys::MovableMutex::new(), owner: AtomicUsize::new(0), lock_count: UnsafeCell::new(0), data: t, - _pinned: PhantomPinned, } } - /// Initializes this mutex so it's ready for use. - /// - /// # Unsafety - /// - /// Unsafe to call more than once, and must be called after this will no - /// longer move in memory. - pub unsafe fn init(self: Pin<&mut Self>) { - self.get_unchecked_mut().mutex.init() - } - /// Acquires a mutex, blocking the current thread until it is able to do so. /// /// This function will block the caller until it is available to acquire the mutex. @@ -113,15 +93,14 @@ impl<T> ReentrantMutex<T> { /// If another user of this mutex panicked while holding the mutex, then /// this call will return failure if the mutex would otherwise be /// acquired. - pub fn lock(self: Pin<&Self>) -> ReentrantMutexGuard<'_, T> { + pub fn lock(&self) -> ReentrantMutexGuard<'_, T> { let this_thread = current_thread_unique_ptr(); - // Safety: We only touch lock_count when we own the lock, - // and since self is pinned we can safely call the lock() on the mutex. + // Safety: We only touch lock_count when we own the lock. unsafe { if self.owner.load(Relaxed) == this_thread { self.increment_lock_count(); } else { - self.mutex.lock(); + self.mutex.raw_lock(); self.owner.store(this_thread, Relaxed); debug_assert_eq!(*self.lock_count.get(), 0); *self.lock_count.get() = 1; @@ -142,10 +121,9 @@ impl<T> ReentrantMutex<T> { /// If another user of this mutex panicked while holding the mutex, then /// this call will return failure if the mutex would otherwise be /// acquired. - pub fn try_lock(self: Pin<&Self>) -> Option<ReentrantMutexGuard<'_, T>> { + pub fn try_lock(&self) -> Option<ReentrantMutexGuard<'_, T>> { let this_thread = current_thread_unique_ptr(); - // Safety: We only touch lock_count when we own the lock, - // and since self is pinned we can safely call the try_lock on the mutex. + // Safety: We only touch lock_count when we own the lock. unsafe { if self.owner.load(Relaxed) == this_thread { self.increment_lock_count(); @@ -179,12 +157,12 @@ impl<T> Deref for ReentrantMutexGuard<'_, T> { impl<T> Drop for ReentrantMutexGuard<'_, T> { #[inline] fn drop(&mut self) { - // Safety: We own the lock, and the lock is pinned. + // Safety: We own the lock. unsafe { *self.lock.lock_count.get() -= 1; if *self.lock.lock_count.get() == 0 { self.lock.owner.store(0, Relaxed); - self.lock.mutex.unlock(); + self.lock.mutex.raw_unlock(); } } } diff --git a/library/std/src/sys_common/remutex/tests.rs b/library/std/src/sys_common/remutex/tests.rs index 64873b850..8e97ce11c 100644 --- a/library/std/src/sys_common/remutex/tests.rs +++ b/library/std/src/sys_common/remutex/tests.rs @@ -1,18 +1,11 @@ -use crate::boxed::Box; use crate::cell::RefCell; -use crate::pin::Pin; use crate::sync::Arc; use crate::sys_common::remutex::{ReentrantMutex, ReentrantMutexGuard}; use crate::thread; #[test] fn smoke() { - let m = unsafe { - let mut m = Box::pin(ReentrantMutex::new(())); - m.as_mut().init(); - m - }; - let m = m.as_ref(); + let m = ReentrantMutex::new(()); { let a = m.lock(); { @@ -29,20 +22,15 @@ fn smoke() { #[test] fn is_mutex() { - let m = unsafe { - // FIXME: Simplify this if Arc gets an Arc::get_pin_mut. - let mut m = Arc::new(ReentrantMutex::new(RefCell::new(0))); - Pin::new_unchecked(Arc::get_mut_unchecked(&mut m)).init(); - Pin::new_unchecked(m) - }; + let m = Arc::new(ReentrantMutex::new(RefCell::new(0))); let m2 = m.clone(); - let lock = m.as_ref().lock(); + let lock = m.lock(); let child = thread::spawn(move || { - let lock = m2.as_ref().lock(); + let lock = m2.lock(); assert_eq!(*lock.borrow(), 4950); }); for i in 0..100 { - let lock = m.as_ref().lock(); + let lock = m.lock(); *lock.borrow_mut() += i; } drop(lock); @@ -51,22 +39,17 @@ fn is_mutex() { #[test] fn trylock_works() { - let m = unsafe { - // FIXME: Simplify this if Arc gets an Arc::get_pin_mut. - let mut m = Arc::new(ReentrantMutex::new(())); - Pin::new_unchecked(Arc::get_mut_unchecked(&mut m)).init(); - Pin::new_unchecked(m) - }; + let m = Arc::new(ReentrantMutex::new(())); let m2 = m.clone(); - let _lock = m.as_ref().try_lock(); - let _lock2 = m.as_ref().try_lock(); + let _lock = m.try_lock(); + let _lock2 = m.try_lock(); thread::spawn(move || { - let lock = m2.as_ref().try_lock(); + let lock = m2.try_lock(); assert!(lock.is_none()); }) .join() .unwrap(); - let _lock3 = m.as_ref().try_lock(); + let _lock3 = m.try_lock(); } pub struct Answer<'a>(pub ReentrantMutexGuard<'a, RefCell<u32>>); diff --git a/library/std/src/sys_common/thread_local_key.rs b/library/std/src/sys_common/thread_local_key.rs index 70beebe86..032bf604d 100644 --- a/library/std/src/sys_common/thread_local_key.rs +++ b/library/std/src/sys_common/thread_local_key.rs @@ -69,8 +69,10 @@ use crate::sys_common::mutex::StaticMutex; /// ```ignore (cannot-doctest-private-modules) /// use tls::os::{StaticKey, INIT}; /// +/// // Use a regular global static to store the key. /// static KEY: StaticKey = INIT; /// +/// // The state provided via `get` and `set` is thread-local. /// unsafe { /// assert!(KEY.get().is_null()); /// KEY.set(1 as *mut u8); diff --git a/library/std/src/sys_common/thread_local_key/tests.rs b/library/std/src/sys_common/thread_local_key/tests.rs index 968738a41..6f32b858f 100644 --- a/library/std/src/sys_common/thread_local_key/tests.rs +++ b/library/std/src/sys_common/thread_local_key/tests.rs @@ -1,4 +1,5 @@ use super::{Key, StaticKey}; +use core::ptr; fn assert_sync<T: Sync>() {} fn assert_send<T: Send>() {} @@ -12,8 +13,8 @@ fn smoke() { let k2 = Key::new(None); assert!(k1.get().is_null()); assert!(k2.get().is_null()); - k1.set(1 as *mut _); - k2.set(2 as *mut _); + k1.set(ptr::invalid_mut(1)); + k2.set(ptr::invalid_mut(2)); assert_eq!(k1.get() as usize, 1); assert_eq!(k2.get() as usize, 2); } @@ -26,8 +27,8 @@ fn statik() { unsafe { assert!(K1.get().is_null()); assert!(K2.get().is_null()); - K1.set(1 as *mut _); - K2.set(2 as *mut _); + K1.set(ptr::invalid_mut(1)); + K2.set(ptr::invalid_mut(2)); assert_eq!(K1.get() as usize, 1); assert_eq!(K2.get() as usize, 2); } diff --git a/library/std/src/sys_common/thread_parker/mod.rs b/library/std/src/sys_common/thread_parker/mod.rs index cbd7832eb..f86a9a555 100644 --- a/library/std/src/sys_common/thread_parker/mod.rs +++ b/library/std/src/sys_common/thread_parker/mod.rs @@ -7,6 +7,7 @@ cfg_if::cfg_if! { target_os = "openbsd", target_os = "dragonfly", target_os = "fuchsia", + target_os = "hermit", ))] { mod futex; pub use futex::Parker; diff --git a/library/std/src/sys_common/wtf8.rs b/library/std/src/sys_common/wtf8.rs index 57fa49893..dd53767d4 100644 --- a/library/std/src/sys_common/wtf8.rs +++ b/library/std/src/sys_common/wtf8.rs @@ -89,6 +89,24 @@ impl CodePoint { self.value } + /// Returns the numeric value of the code point if it is a leading surrogate. + #[inline] + pub fn to_lead_surrogate(&self) -> Option<u16> { + match self.value { + lead @ 0xD800..=0xDBFF => Some(lead as u16), + _ => None, + } + } + + /// Returns the numeric value of the code point if it is a trailing surrogate. + #[inline] + pub fn to_trail_surrogate(&self) -> Option<u16> { + match self.value { + trail @ 0xDC00..=0xDFFF => Some(trail as u16), + _ => None, + } + } + /// Optionally returns a Unicode scalar value for the code point. /// /// Returns `None` if the code point is a surrogate (from U+D800 to U+DFFF). @@ -117,6 +135,14 @@ impl CodePoint { #[derive(Eq, PartialEq, Ord, PartialOrd, Clone)] pub struct Wtf8Buf { bytes: Vec<u8>, + + /// Do we know that `bytes` holds a valid UTF-8 encoding? We can easily + /// know this if we're constructed from a `String` or `&str`. + /// + /// It is possible for `bytes` to have valid UTF-8 without this being + /// set, such as when we're concatenating `&Wtf8`'s and surrogates become + /// paired, as we don't bother to rescan the entire string. + is_known_utf8: bool, } impl ops::Deref for Wtf8Buf { @@ -147,13 +173,13 @@ impl Wtf8Buf { /// Creates a new, empty WTF-8 string. #[inline] pub fn new() -> Wtf8Buf { - Wtf8Buf { bytes: Vec::new() } + Wtf8Buf { bytes: Vec::new(), is_known_utf8: true } } /// Creates a new, empty WTF-8 string with pre-allocated capacity for `capacity` bytes. #[inline] pub fn with_capacity(capacity: usize) -> Wtf8Buf { - Wtf8Buf { bytes: Vec::with_capacity(capacity) } + Wtf8Buf { bytes: Vec::with_capacity(capacity), is_known_utf8: true } } /// Creates a WTF-8 string from a UTF-8 `String`. @@ -163,7 +189,7 @@ impl Wtf8Buf { /// Since WTF-8 is a superset of UTF-8, this always succeeds. #[inline] pub fn from_string(string: String) -> Wtf8Buf { - Wtf8Buf { bytes: string.into_bytes() } + Wtf8Buf { bytes: string.into_bytes(), is_known_utf8: true } } /// Creates a WTF-8 string from a UTF-8 `&str` slice. @@ -173,11 +199,12 @@ impl Wtf8Buf { /// Since WTF-8 is a superset of UTF-8, this always succeeds. #[inline] pub fn from_str(str: &str) -> Wtf8Buf { - Wtf8Buf { bytes: <[_]>::to_vec(str.as_bytes()) } + Wtf8Buf { bytes: <[_]>::to_vec(str.as_bytes()), is_known_utf8: true } } pub fn clear(&mut self) { - self.bytes.clear() + self.bytes.clear(); + self.is_known_utf8 = true; } /// Creates a WTF-8 string from a potentially ill-formed UTF-16 slice of 16-bit code units. @@ -193,9 +220,11 @@ impl Wtf8Buf { let surrogate = surrogate.unpaired_surrogate(); // Surrogates are known to be in the code point range. let code_point = unsafe { CodePoint::from_u32_unchecked(surrogate as u32) }; + // The string will now contain an unpaired surrogate. + string.is_known_utf8 = false; // Skip the WTF-8 concatenation check, // surrogate pairs are already decoded by decode_utf16 - string.push_code_point_unchecked(code_point) + string.push_code_point_unchecked(code_point); } } } @@ -203,7 +232,7 @@ impl Wtf8Buf { } /// Copied from String::push - /// This does **not** include the WTF-8 concatenation check. + /// This does **not** include the WTF-8 concatenation check or `is_known_utf8` check. fn push_code_point_unchecked(&mut self, code_point: CodePoint) { let mut bytes = [0; 4]; let bytes = char::encode_utf8_raw(code_point.value, &mut bytes); @@ -217,6 +246,9 @@ impl Wtf8Buf { #[inline] pub fn as_mut_slice(&mut self) -> &mut Wtf8 { + // Safety: `Wtf8` doesn't expose any way to mutate the bytes that would + // cause them to change from well-formed UTF-8 to ill-formed UTF-8, + // which would break the assumptions of the `is_known_utf8` field. unsafe { Wtf8::from_mut_bytes_unchecked(&mut self.bytes) } } @@ -236,7 +268,8 @@ impl Wtf8Buf { /// in the given `Wtf8Buf`. The `Wtf8Buf` may reserve more space to avoid /// frequent reallocations. After calling `try_reserve`, capacity will be /// greater than or equal to `self.len() + additional`. Does nothing if - /// capacity is already sufficient. + /// capacity is already sufficient. This method preserves the contents even + /// if an error occurs. /// /// # Errors /// @@ -313,7 +346,15 @@ impl Wtf8Buf { self.push_char(decode_surrogate_pair(lead, trail)); self.bytes.extend_from_slice(other_without_trail_surrogate); } - _ => self.bytes.extend_from_slice(&other.bytes), + _ => { + // If we'll be pushing a string containing a surrogate, we may + // no longer have UTF-8. + if other.next_surrogate(0).is_some() { + self.is_known_utf8 = false; + } + + self.bytes.extend_from_slice(&other.bytes); + } } } @@ -330,13 +371,19 @@ impl Wtf8Buf { /// like concatenating ill-formed UTF-16 strings effectively would. #[inline] pub fn push(&mut self, code_point: CodePoint) { - if let trail @ 0xDC00..=0xDFFF = code_point.to_u32() { + if let Some(trail) = code_point.to_trail_surrogate() { if let Some(lead) = (&*self).final_lead_surrogate() { let len_without_lead_surrogate = self.len() - 3; self.bytes.truncate(len_without_lead_surrogate); - self.push_char(decode_surrogate_pair(lead, trail as u16)); + self.push_char(decode_surrogate_pair(lead, trail)); return; } + + // We're pushing a trailing surrogate. + self.is_known_utf8 = false; + } else if code_point.to_lead_surrogate().is_some() { + // We're pushing a leading surrogate. + self.is_known_utf8 = false; } // No newly paired surrogates at the boundary. @@ -363,9 +410,10 @@ impl Wtf8Buf { /// (that is, if the string contains surrogates), /// the original WTF-8 string is returned instead. pub fn into_string(self) -> Result<String, Wtf8Buf> { - match self.next_surrogate(0) { - None => Ok(unsafe { String::from_utf8_unchecked(self.bytes) }), - Some(_) => Err(self), + if self.is_known_utf8 || self.next_surrogate(0).is_none() { + Ok(unsafe { String::from_utf8_unchecked(self.bytes) }) + } else { + Err(self) } } @@ -375,6 +423,11 @@ impl Wtf8Buf { /// /// Surrogates are replaced with `"\u{FFFD}"` (the replacement character āļæ½ā) pub fn into_string_lossy(mut self) -> String { + // Fast path: If we already have UTF-8, we can return it immediately. + if self.is_known_utf8 { + return unsafe { String::from_utf8_unchecked(self.bytes) }; + } + let mut pos = 0; loop { match self.next_surrogate(pos) { @@ -397,7 +450,7 @@ impl Wtf8Buf { /// Converts a `Box<Wtf8>` into a `Wtf8Buf`. pub fn from_box(boxed: Box<Wtf8>) -> Wtf8Buf { let bytes: Box<[u8]> = unsafe { mem::transmute(boxed) }; - Wtf8Buf { bytes: bytes.into_vec() } + Wtf8Buf { bytes: bytes.into_vec(), is_known_utf8: false } } } @@ -575,6 +628,11 @@ impl Wtf8 { } } + /// Creates an owned `Wtf8Buf` from a borrowed `Wtf8`. + pub fn to_owned(&self) -> Wtf8Buf { + Wtf8Buf { bytes: self.bytes.to_vec(), is_known_utf8: false } + } + /// Lossily converts the string to UTF-8. /// Returns a UTF-8 `&str` slice if the contents are well-formed in UTF-8. /// @@ -664,7 +722,8 @@ impl Wtf8 { } pub fn clone_into(&self, buf: &mut Wtf8Buf) { - self.bytes.clone_into(&mut buf.bytes) + buf.is_known_utf8 = false; + self.bytes.clone_into(&mut buf.bytes); } /// Boxes this `Wtf8`. @@ -704,12 +763,12 @@ impl Wtf8 { #[inline] pub fn to_ascii_lowercase(&self) -> Wtf8Buf { - Wtf8Buf { bytes: self.bytes.to_ascii_lowercase() } + Wtf8Buf { bytes: self.bytes.to_ascii_lowercase(), is_known_utf8: false } } #[inline] pub fn to_ascii_uppercase(&self) -> Wtf8Buf { - Wtf8Buf { bytes: self.bytes.to_ascii_uppercase() } + Wtf8Buf { bytes: self.bytes.to_ascii_uppercase(), is_known_utf8: false } } #[inline] diff --git a/library/std/src/sys_common/wtf8/tests.rs b/library/std/src/sys_common/wtf8/tests.rs index 931996791..1a302d646 100644 --- a/library/std/src/sys_common/wtf8/tests.rs +++ b/library/std/src/sys_common/wtf8/tests.rs @@ -20,6 +20,36 @@ fn code_point_to_u32() { } #[test] +fn code_point_to_lead_surrogate() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0).to_lead_surrogate(), None); + assert_eq!(c(0xE9).to_lead_surrogate(), None); + assert_eq!(c(0xD800).to_lead_surrogate(), Some(0xD800)); + assert_eq!(c(0xDBFF).to_lead_surrogate(), Some(0xDBFF)); + assert_eq!(c(0xDC00).to_lead_surrogate(), None); + assert_eq!(c(0xDFFF).to_lead_surrogate(), None); + assert_eq!(c(0x1F4A9).to_lead_surrogate(), None); + assert_eq!(c(0x10FFFF).to_lead_surrogate(), None); +} + +#[test] +fn code_point_to_trail_surrogate() { + fn c(value: u32) -> CodePoint { + CodePoint::from_u32(value).unwrap() + } + assert_eq!(c(0).to_trail_surrogate(), None); + assert_eq!(c(0xE9).to_trail_surrogate(), None); + assert_eq!(c(0xD800).to_trail_surrogate(), None); + assert_eq!(c(0xDBFF).to_trail_surrogate(), None); + assert_eq!(c(0xDC00).to_trail_surrogate(), Some(0xDC00)); + assert_eq!(c(0xDFFF).to_trail_surrogate(), Some(0xDFFF)); + assert_eq!(c(0x1F4A9).to_trail_surrogate(), None); + assert_eq!(c(0x10FFFF).to_trail_surrogate(), None); +} + +#[test] fn code_point_from_char() { assert_eq!(CodePoint::from_char('a').to_u32(), 0x61); assert_eq!(CodePoint::from_char('š©').to_u32(), 0x1F4A9); @@ -70,35 +100,66 @@ fn wtf8buf_from_string() { #[test] fn wtf8buf_from_wide() { - assert_eq!(Wtf8Buf::from_wide(&[]).bytes, b""); - assert_eq!( - Wtf8Buf::from_wide(&[0x61, 0xE9, 0x20, 0xD83D, 0xD83D, 0xDCA9]).bytes, - b"a\xC3\xA9 \xED\xA0\xBD\xF0\x9F\x92\xA9" - ); + let buf = Wtf8Buf::from_wide(&[]); + assert_eq!(buf.bytes, b""); + assert!(buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0x61, 0xE9, 0x20, 0xD83D, 0xDCA9]); + assert_eq!(buf.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0x61, 0xE9, 0x20, 0xD83D, 0xD83D, 0xDCA9]); + assert_eq!(buf.bytes, b"a\xC3\xA9 \xED\xA0\xBD\xF0\x9F\x92\xA9"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xD800]); + assert_eq!(buf.bytes, b"\xED\xA0\x80"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xDBFF]); + assert_eq!(buf.bytes, b"\xED\xAF\xBF"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xDC00]); + assert_eq!(buf.bytes, b"\xED\xB0\x80"); + assert!(!buf.is_known_utf8); + + let buf = Wtf8Buf::from_wide(&[0xDFFF]); + assert_eq!(buf.bytes, b"\xED\xBF\xBF"); + assert!(!buf.is_known_utf8); } #[test] fn wtf8buf_push_str() { let mut string = Wtf8Buf::new(); assert_eq!(string.bytes, b""); + assert!(string.is_known_utf8); + string.push_str("aĆ© š©"); assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); } #[test] fn wtf8buf_push_char() { let mut string = Wtf8Buf::from_str("aĆ© "); assert_eq!(string.bytes, b"a\xC3\xA9 "); + assert!(string.is_known_utf8); + string.push_char('š©'); assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); } #[test] fn wtf8buf_push() { let mut string = Wtf8Buf::from_str("aĆ© "); assert_eq!(string.bytes, b"a\xC3\xA9 "); + assert!(string.is_known_utf8); + string.push(CodePoint::from_char('š©')); assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); fn c(value: u32) -> CodePoint { CodePoint::from_u32(value).unwrap() @@ -106,37 +167,46 @@ fn wtf8buf_push() { let mut string = Wtf8Buf::new(); string.push(c(0xD83D)); // lead + assert!(!string.is_known_utf8); string.push(c(0xDCA9)); // trail assert_eq!(string.bytes, b"\xF0\x9F\x92\xA9"); // Magic! let mut string = Wtf8Buf::new(); string.push(c(0xD83D)); // lead + assert!(!string.is_known_utf8); string.push(c(0x20)); // not surrogate string.push(c(0xDCA9)); // trail assert_eq!(string.bytes, b"\xED\xA0\xBD \xED\xB2\xA9"); let mut string = Wtf8Buf::new(); string.push(c(0xD800)); // lead + assert!(!string.is_known_utf8); string.push(c(0xDBFF)); // lead assert_eq!(string.bytes, b"\xED\xA0\x80\xED\xAF\xBF"); let mut string = Wtf8Buf::new(); string.push(c(0xD800)); // lead + assert!(!string.is_known_utf8); string.push(c(0xE000)); // not surrogate assert_eq!(string.bytes, b"\xED\xA0\x80\xEE\x80\x80"); let mut string = Wtf8Buf::new(); string.push(c(0xD7FF)); // not surrogate + assert!(string.is_known_utf8); string.push(c(0xDC00)); // trail + assert!(!string.is_known_utf8); assert_eq!(string.bytes, b"\xED\x9F\xBF\xED\xB0\x80"); let mut string = Wtf8Buf::new(); string.push(c(0x61)); // not surrogate, < 3 bytes + assert!(string.is_known_utf8); string.push(c(0xDC00)); // trail + assert!(!string.is_known_utf8); assert_eq!(string.bytes, b"\x61\xED\xB0\x80"); let mut string = Wtf8Buf::new(); string.push(c(0xDC00)); // trail + assert!(!string.is_known_utf8); assert_eq!(string.bytes, b"\xED\xB0\x80"); } @@ -146,6 +216,7 @@ fn wtf8buf_push_wtf8() { assert_eq!(string.bytes, b"a\xC3\xA9"); string.push_wtf8(Wtf8::from_str(" š©")); assert_eq!(string.bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); fn w(v: &[u8]) -> &Wtf8 { unsafe { Wtf8::from_bytes_unchecked(v) } @@ -161,37 +232,68 @@ fn wtf8buf_push_wtf8() { string.push_wtf8(w(b" ")); // not surrogate string.push_wtf8(w(b"\xED\xB2\xA9")); // trail assert_eq!(string.bytes, b"\xED\xA0\xBD \xED\xB2\xA9"); + assert!(!string.is_known_utf8); let mut string = Wtf8Buf::new(); string.push_wtf8(w(b"\xED\xA0\x80")); // lead string.push_wtf8(w(b"\xED\xAF\xBF")); // lead assert_eq!(string.bytes, b"\xED\xA0\x80\xED\xAF\xBF"); + assert!(!string.is_known_utf8); let mut string = Wtf8Buf::new(); string.push_wtf8(w(b"\xED\xA0\x80")); // lead string.push_wtf8(w(b"\xEE\x80\x80")); // not surrogate assert_eq!(string.bytes, b"\xED\xA0\x80\xEE\x80\x80"); + assert!(!string.is_known_utf8); let mut string = Wtf8Buf::new(); string.push_wtf8(w(b"\xED\x9F\xBF")); // not surrogate string.push_wtf8(w(b"\xED\xB0\x80")); // trail assert_eq!(string.bytes, b"\xED\x9F\xBF\xED\xB0\x80"); + assert!(!string.is_known_utf8); let mut string = Wtf8Buf::new(); string.push_wtf8(w(b"a")); // not surrogate, < 3 bytes string.push_wtf8(w(b"\xED\xB0\x80")); // trail assert_eq!(string.bytes, b"\x61\xED\xB0\x80"); + assert!(!string.is_known_utf8); let mut string = Wtf8Buf::new(); string.push_wtf8(w(b"\xED\xB0\x80")); // trail assert_eq!(string.bytes, b"\xED\xB0\x80"); + assert!(!string.is_known_utf8); } #[test] fn wtf8buf_truncate() { let mut string = Wtf8Buf::from_str("aĆ©"); + assert!(string.is_known_utf8); + + string.truncate(3); + assert_eq!(string.bytes, b"a\xC3\xA9"); + assert!(string.is_known_utf8); + string.truncate(1); assert_eq!(string.bytes, b"a"); + assert!(string.is_known_utf8); + + string.truncate(0); + assert_eq!(string.bytes, b""); + assert!(string.is_known_utf8); +} + +#[test] +fn wtf8buf_truncate_around_non_bmp() { + let mut string = Wtf8Buf::from_str("š©"); + assert!(string.is_known_utf8); + + string.truncate(4); + assert_eq!(string.bytes, b"\xF0\x9F\x92\xA9"); + assert!(string.is_known_utf8); + + string.truncate(0); + assert_eq!(string.bytes, b""); + assert!(string.is_known_utf8); } #[test] @@ -209,10 +311,36 @@ fn wtf8buf_truncate_fail_longer() { } #[test] +#[should_panic] +fn wtf8buf_truncate_splitting_non_bmp3() { + let mut string = Wtf8Buf::from_str("š©"); + assert!(string.is_known_utf8); + string.truncate(3); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_splitting_non_bmp2() { + let mut string = Wtf8Buf::from_str("š©"); + assert!(string.is_known_utf8); + string.truncate(2); +} + +#[test] +#[should_panic] +fn wtf8buf_truncate_splitting_non_bmp1() { + let mut string = Wtf8Buf::from_str("š©"); + assert!(string.is_known_utf8); + string.truncate(1); +} + +#[test] fn wtf8buf_into_string() { let mut string = Wtf8Buf::from_str("aĆ© š©"); + assert!(string.is_known_utf8); assert_eq!(string.clone().into_string(), Ok(String::from("aĆ© š©"))); string.push(CodePoint::from_u32(0xD800).unwrap()); + assert!(!string.is_known_utf8); assert_eq!(string.clone().into_string(), Err(string)); } @@ -229,15 +357,33 @@ fn wtf8buf_from_iterator() { fn f(values: &[u32]) -> Wtf8Buf { values.iter().map(|&c| CodePoint::from_u32(c).unwrap()).collect::<Wtf8Buf>() } - assert_eq!(f(&[0x61, 0xE9, 0x20, 0x1F4A9]).bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert_eq!( + f(&[0x61, 0xE9, 0x20, 0x1F4A9]), + Wtf8Buf { bytes: b"a\xC3\xA9 \xF0\x9F\x92\xA9".to_vec(), is_known_utf8: true } + ); assert_eq!(f(&[0xD83D, 0xDCA9]).bytes, b"\xF0\x9F\x92\xA9"); // Magic! - assert_eq!(f(&[0xD83D, 0x20, 0xDCA9]).bytes, b"\xED\xA0\xBD \xED\xB2\xA9"); - assert_eq!(f(&[0xD800, 0xDBFF]).bytes, b"\xED\xA0\x80\xED\xAF\xBF"); - assert_eq!(f(&[0xD800, 0xE000]).bytes, b"\xED\xA0\x80\xEE\x80\x80"); - assert_eq!(f(&[0xD7FF, 0xDC00]).bytes, b"\xED\x9F\xBF\xED\xB0\x80"); - assert_eq!(f(&[0x61, 0xDC00]).bytes, b"\x61\xED\xB0\x80"); - assert_eq!(f(&[0xDC00]).bytes, b"\xED\xB0\x80"); + assert_eq!( + f(&[0xD83D, 0x20, 0xDCA9]), + Wtf8Buf { bytes: b"\xED\xA0\xBD \xED\xB2\xA9".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0xD800, 0xDBFF]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xED\xAF\xBF".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0xD800, 0xE000]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xEE\x80\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0xD7FF, 0xDC00]), + Wtf8Buf { bytes: b"\xED\x9F\xBF\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + f(&[0x61, 0xDC00]), + Wtf8Buf { bytes: b"\x61\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!(f(&[0xDC00]), Wtf8Buf { bytes: b"\xED\xB0\x80".to_vec(), is_known_utf8: false }); } #[test] @@ -251,15 +397,36 @@ fn wtf8buf_extend() { string } - assert_eq!(e(&[0x61, 0xE9], &[0x20, 0x1F4A9]).bytes, b"a\xC3\xA9 \xF0\x9F\x92\xA9"); + assert_eq!( + e(&[0x61, 0xE9], &[0x20, 0x1F4A9]), + Wtf8Buf { bytes: b"a\xC3\xA9 \xF0\x9F\x92\xA9".to_vec(), is_known_utf8: true } + ); assert_eq!(e(&[0xD83D], &[0xDCA9]).bytes, b"\xF0\x9F\x92\xA9"); // Magic! - assert_eq!(e(&[0xD83D, 0x20], &[0xDCA9]).bytes, b"\xED\xA0\xBD \xED\xB2\xA9"); - assert_eq!(e(&[0xD800], &[0xDBFF]).bytes, b"\xED\xA0\x80\xED\xAF\xBF"); - assert_eq!(e(&[0xD800], &[0xE000]).bytes, b"\xED\xA0\x80\xEE\x80\x80"); - assert_eq!(e(&[0xD7FF], &[0xDC00]).bytes, b"\xED\x9F\xBF\xED\xB0\x80"); - assert_eq!(e(&[0x61], &[0xDC00]).bytes, b"\x61\xED\xB0\x80"); - assert_eq!(e(&[], &[0xDC00]).bytes, b"\xED\xB0\x80"); + assert_eq!( + e(&[0xD83D, 0x20], &[0xDCA9]), + Wtf8Buf { bytes: b"\xED\xA0\xBD \xED\xB2\xA9".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0xD800], &[0xDBFF]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xED\xAF\xBF".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0xD800], &[0xE000]), + Wtf8Buf { bytes: b"\xED\xA0\x80\xEE\x80\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0xD7FF], &[0xDC00]), + Wtf8Buf { bytes: b"\xED\x9F\xBF\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[0x61], &[0xDC00]), + Wtf8Buf { bytes: b"\x61\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); + assert_eq!( + e(&[], &[0xDC00]), + Wtf8Buf { bytes: b"\xED\xB0\x80".to_vec(), is_known_utf8: false } + ); } #[test] @@ -407,3 +574,93 @@ fn wtf8_encode_wide_size_hint() { assert_eq!((0, Some(0)), iter.size_hint()); assert!(iter.next().is_none()); } + +#[test] +fn wtf8_clone_into() { + let mut string = Wtf8Buf::new(); + Wtf8::from_str("green").clone_into(&mut string); + assert_eq!(string.bytes, b"green"); + + let mut string = Wtf8Buf::from_str("green"); + Wtf8::from_str("").clone_into(&mut string); + assert_eq!(string.bytes, b""); + + let mut string = Wtf8Buf::from_str("red"); + Wtf8::from_str("green").clone_into(&mut string); + assert_eq!(string.bytes, b"green"); + + let mut string = Wtf8Buf::from_str("green"); + Wtf8::from_str("red").clone_into(&mut string); + assert_eq!(string.bytes, b"red"); + + let mut string = Wtf8Buf::from_str("green"); + assert!(string.is_known_utf8); + unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").clone_into(&mut string) }; + assert_eq!(string.bytes, b"\xED\xA0\x80"); + assert!(!string.is_known_utf8); +} + +#[test] +fn wtf8_to_ascii_lowercase() { + let lowercase = Wtf8::from_str("").to_ascii_lowercase(); + assert_eq!(lowercase.bytes, b""); + + let lowercase = Wtf8::from_str("GrEeN gRaPeS! š").to_ascii_lowercase(); + assert_eq!(lowercase.bytes, b"green grapes! \xf0\x9f\x8d\x87"); + + let lowercase = unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").to_ascii_lowercase() }; + assert_eq!(lowercase.bytes, b"\xED\xA0\x80"); + assert!(!lowercase.is_known_utf8); +} + +#[test] +fn wtf8_to_ascii_uppercase() { + let uppercase = Wtf8::from_str("").to_ascii_uppercase(); + assert_eq!(uppercase.bytes, b""); + + let uppercase = Wtf8::from_str("GrEeN gRaPeS! š").to_ascii_uppercase(); + assert_eq!(uppercase.bytes, b"GREEN GRAPES! \xf0\x9f\x8d\x87"); + + let uppercase = unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").to_ascii_uppercase() }; + assert_eq!(uppercase.bytes, b"\xED\xA0\x80"); + assert!(!uppercase.is_known_utf8); +} + +#[test] +fn wtf8_make_ascii_lowercase() { + let mut lowercase = Wtf8Buf::from_str(""); + lowercase.make_ascii_lowercase(); + assert_eq!(lowercase.bytes, b""); + + let mut lowercase = Wtf8Buf::from_str("GrEeN gRaPeS! š"); + lowercase.make_ascii_lowercase(); + assert_eq!(lowercase.bytes, b"green grapes! \xf0\x9f\x8d\x87"); + + let mut lowercase = unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").to_owned() }; + lowercase.make_ascii_lowercase(); + assert_eq!(lowercase.bytes, b"\xED\xA0\x80"); + assert!(!lowercase.is_known_utf8); +} + +#[test] +fn wtf8_make_ascii_uppercase() { + let mut uppercase = Wtf8Buf::from_str(""); + uppercase.make_ascii_uppercase(); + assert_eq!(uppercase.bytes, b""); + + let mut uppercase = Wtf8Buf::from_str("GrEeN gRaPeS! š"); + uppercase.make_ascii_uppercase(); + assert_eq!(uppercase.bytes, b"GREEN GRAPES! \xf0\x9f\x8d\x87"); + + let mut uppercase = unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").to_owned() }; + uppercase.make_ascii_uppercase(); + assert_eq!(uppercase.bytes, b"\xED\xA0\x80"); + assert!(!uppercase.is_known_utf8); +} + +#[test] +fn wtf8_to_owned() { + let string = unsafe { Wtf8::from_bytes_unchecked(b"\xED\xA0\x80").to_owned() }; + assert_eq!(string.bytes, b"\xED\xA0\x80"); + assert!(!string.is_known_utf8); +} |