diff options
Diffstat (limited to 'library/std/src/sys')
45 files changed, 1468 insertions, 252 deletions
diff --git a/library/std/src/sys/common/alloc.rs b/library/std/src/sys/common/alloc.rs index a5fcbdf39..d58aa6c27 100644 --- a/library/std/src/sys/common/alloc.rs +++ b/library/std/src/sys/common/alloc.rs @@ -8,7 +8,9 @@ use crate::ptr; target_arch = "x86", target_arch = "arm", target_arch = "m68k", + target_arch = "csky", target_arch = "mips", + target_arch = "mips32r6", target_arch = "powerpc", target_arch = "powerpc64", target_arch = "sparc", @@ -24,6 +26,7 @@ pub const MIN_ALIGN: usize = 8; target_arch = "aarch64", target_arch = "loongarch64", target_arch = "mips64", + target_arch = "mips64r6", target_arch = "s390x", target_arch = "sparc64", target_arch = "riscv64", diff --git a/library/std/src/sys/common/thread_local/fast_local.rs b/library/std/src/sys/common/thread_local/fast_local.rs index bc5da1a18..c0a9619bf 100644 --- a/library/std/src/sys/common/thread_local/fast_local.rs +++ b/library/std/src/sys/common/thread_local/fast_local.rs @@ -87,10 +87,6 @@ pub macro thread_local_inner { static __KEY: $crate::thread::local_impl::Key<$t> = $crate::thread::local_impl::Key::<$t>::new(); - // FIXME: remove the #[allow(...)] marker when macros don't - // raise warning for missing/extraneous unsafe blocks anymore. - // See https://github.com/rust-lang/rust/issues/74838. - #[allow(unused_unsafe)] unsafe { __KEY.get(move || { if let $crate::option::Option::Some(init) = init { diff --git a/library/std/src/sys/common/thread_local/os_local.rs b/library/std/src/sys/common/thread_local/os_local.rs index 5d48ce1e0..7cf291921 100644 --- a/library/std/src/sys/common/thread_local/os_local.rs +++ b/library/std/src/sys/common/thread_local/os_local.rs @@ -24,7 +24,6 @@ pub macro thread_local_inner { const fn __init() -> $t { INIT_EXPR } static __KEY: $crate::thread::local_impl::Key<$t> = $crate::thread::local_impl::Key::new(); - #[allow(unused_unsafe)] unsafe { __KEY.get(move || { if let $crate::option::Option::Some(init) = _init { @@ -59,10 +58,6 @@ pub macro thread_local_inner { static __KEY: $crate::thread::local_impl::Key<$t> = $crate::thread::local_impl::Key::new(); - // FIXME: remove the #[allow(...)] marker when macros don't - // raise warning for missing/extraneous unsafe blocks anymore. - // See https://github.com/rust-lang/rust/issues/74838. - #[allow(unused_unsafe)] unsafe { __KEY.get(move || { if let $crate::option::Option::Some(init) = init { diff --git a/library/std/src/sys/common/thread_local/static_local.rs b/library/std/src/sys/common/thread_local/static_local.rs index 80322a978..5cb6c541a 100644 --- a/library/std/src/sys/common/thread_local/static_local.rs +++ b/library/std/src/sys/common/thread_local/static_local.rs @@ -43,10 +43,6 @@ pub macro thread_local_inner { static __KEY: $crate::thread::local_impl::Key<$t> = $crate::thread::local_impl::Key::new(); - // FIXME: remove the #[allow(...)] marker when macros don't - // raise warning for missing/extraneous unsafe blocks anymore. - // See https://github.com/rust-lang/rust/issues/74838. - #[allow(unused_unsafe)] unsafe { __KEY.get(move || { if let $crate::option::Option::Some(init) = init { diff --git a/library/std/src/sys/hermit/fs.rs b/library/std/src/sys/hermit/fs.rs index 4bb735668..6aa4ea7f5 100644 --- a/library/std/src/sys/hermit/fs.rs +++ b/library/std/src/sys/hermit/fs.rs @@ -335,6 +335,7 @@ impl File { false } + #[inline] pub fn flush(&self) -> io::Result<()> { Ok(()) } diff --git a/library/std/src/sys/hermit/os.rs b/library/std/src/sys/hermit/os.rs index e53dbae61..c79197a9a 100644 --- a/library/std/src/sys/hermit/os.rs +++ b/library/std/src/sys/hermit/os.rs @@ -112,6 +112,34 @@ pub struct Env { iter: vec::IntoIter<(OsString, OsString)>, } +// FIXME(https://github.com/rust-lang/rust/issues/114583): Remove this when <OsStr as Debug>::fmt matches <str as Debug>::fmt. +pub struct EnvStrDebug<'a> { + slice: &'a [(OsString, OsString)], +} + +impl fmt::Debug for EnvStrDebug<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { slice } = self; + f.debug_list() + .entries(slice.iter().map(|(a, b)| (a.to_str().unwrap(), b.to_str().unwrap()))) + .finish() + } +} + +impl Env { + pub fn str_debug(&self) -> impl fmt::Debug + '_ { + let Self { iter } = self; + EnvStrDebug { slice: iter.as_slice() } + } +} + +impl fmt::Debug for Env { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { iter } = self; + f.debug_list().entries(iter.as_slice()).finish() + } +} + impl !Send for Env {} impl !Sync for Env {} diff --git a/library/std/src/sys/hermit/thread.rs b/library/std/src/sys/hermit/thread.rs index 2507f7069..332151e40 100644 --- a/library/std/src/sys/hermit/thread.rs +++ b/library/std/src/sys/hermit/thread.rs @@ -1,6 +1,5 @@ #![allow(dead_code)] -use super::unsupported; use crate::ffi::CStr; use crate::io; use crate::mem; @@ -99,7 +98,7 @@ impl Thread { } pub fn available_parallelism() -> io::Result<NonZeroUsize> { - unsupported() + unsafe { Ok(NonZeroUsize::new_unchecked(abi::get_processor_count())) } } pub mod guard { diff --git a/library/std/src/sys/hermit/time.rs b/library/std/src/sys/hermit/time.rs index 5440d85df..7d91460ab 100644 --- a/library/std/src/sys/hermit/time.rs +++ b/library/std/src/sys/hermit/time.rs @@ -40,7 +40,7 @@ impl Timespec { } fn checked_add_duration(&self, other: &Duration) -> Option<Timespec> { - let mut secs = self.tv_sec.checked_add_unsigned(other.as_secs())?; + let mut secs = self.t.tv_sec.checked_add_unsigned(other.as_secs())?; // Nano calculations can't overflow because nanos are <1B which fit // in a u32. @@ -53,7 +53,7 @@ impl Timespec { } fn checked_sub_duration(&self, other: &Duration) -> Option<Timespec> { - let mut secs = self.tv_sec.checked_sub_unsigned(other.as_secs())?; + let mut secs = self.t.tv_sec.checked_sub_unsigned(other.as_secs())?; // Similar to above, nanos can't overflow. let mut nsec = self.t.tv_nsec as i32 - other.subsec_nanos() as i32; diff --git a/library/std/src/sys/mod.rs b/library/std/src/sys/mod.rs index c72be1380..beea3f23c 100644 --- a/library/std/src/sys/mod.rs +++ b/library/std/src/sys/mod.rs @@ -23,6 +23,7 @@ #![allow(missing_debug_implementations)] pub mod common; +mod personality; cfg_if::cfg_if! { if #[cfg(unix)] { @@ -60,3 +61,52 @@ cfg_if::cfg_if! { pub const FULL_BACKTRACE_DEFAULT: bool = false; } } + +#[cfg(not(test))] +cfg_if::cfg_if! { + if #[cfg(target_os = "android")] { + pub use self::android::log2f32; + pub use self::android::log2f64; + } else { + #[inline] + pub fn log2f32(n: f32) -> f32 { + unsafe { crate::intrinsics::log2f32(n) } + } + + #[inline] + pub fn log2f64(n: f64) -> f64 { + unsafe { crate::intrinsics::log2f64(n) } + } + } +} + +// Solaris/Illumos requires a wrapper around log, log2, and log10 functions +// because of their non-standard behavior (e.g., log(-n) returns -Inf instead +// of expected NaN). +#[cfg(not(test))] +#[cfg(any(target_os = "solaris", target_os = "illumos"))] +#[inline] +pub fn log_wrapper<F: Fn(f64) -> f64>(n: f64, log_fn: F) -> f64 { + if n.is_finite() { + if n > 0.0 { + log_fn(n) + } else if n == 0.0 { + f64::NEG_INFINITY // log(0) = -Inf + } else { + f64::NAN // log(-n) = NaN + } + } else if n.is_nan() { + n // log(NaN) = NaN + } else if n > 0.0 { + n // log(Inf) = Inf + } else { + f64::NAN // log(-Inf) = NaN + } +} + +#[cfg(not(test))] +#[cfg(not(any(target_os = "solaris", target_os = "illumos")))] +#[inline] +pub fn log_wrapper<F: Fn(f64) -> f64>(n: f64, log_fn: F) -> f64 { + log_fn(n) +} diff --git a/library/std/src/sys/personality/dwarf/eh.rs b/library/std/src/sys/personality/dwarf/eh.rs new file mode 100644 index 000000000..79624703a --- /dev/null +++ b/library/std/src/sys/personality/dwarf/eh.rs @@ -0,0 +1,208 @@ +//! Parsing of GCC-style Language-Specific Data Area (LSDA) +//! For details see: +//! * <https://refspecs.linuxfoundation.org/LSB_3.0.0/LSB-PDA/LSB-PDA/ehframechpt.html> +//! * <https://itanium-cxx-abi.github.io/cxx-abi/exceptions.pdf> +//! * <https://www.airs.com/blog/archives/460> +//! * <https://www.airs.com/blog/archives/464> +//! +//! A reference implementation may be found in the GCC source tree +//! (`<root>/libgcc/unwind-c.c` as of this writing). + +#![allow(non_upper_case_globals)] +#![allow(unused)] + +use super::DwarfReader; +use core::mem; +use core::ptr; + +pub const DW_EH_PE_omit: u8 = 0xFF; +pub const DW_EH_PE_absptr: u8 = 0x00; + +pub const DW_EH_PE_uleb128: u8 = 0x01; +pub const DW_EH_PE_udata2: u8 = 0x02; +pub const DW_EH_PE_udata4: u8 = 0x03; +pub const DW_EH_PE_udata8: u8 = 0x04; +pub const DW_EH_PE_sleb128: u8 = 0x09; +pub const DW_EH_PE_sdata2: u8 = 0x0A; +pub const DW_EH_PE_sdata4: u8 = 0x0B; +pub const DW_EH_PE_sdata8: u8 = 0x0C; + +pub const DW_EH_PE_pcrel: u8 = 0x10; +pub const DW_EH_PE_textrel: u8 = 0x20; +pub const DW_EH_PE_datarel: u8 = 0x30; +pub const DW_EH_PE_funcrel: u8 = 0x40; +pub const DW_EH_PE_aligned: u8 = 0x50; + +pub const DW_EH_PE_indirect: u8 = 0x80; + +#[derive(Copy, Clone)] +pub struct EHContext<'a> { + pub ip: usize, // Current instruction pointer + pub func_start: usize, // Address of the current function + pub get_text_start: &'a dyn Fn() -> usize, // Get address of the code section + pub get_data_start: &'a dyn Fn() -> usize, // Get address of the data section +} + +pub enum EHAction { + None, + Cleanup(usize), + Catch(usize), + Filter(usize), + Terminate, +} + +pub const USING_SJLJ_EXCEPTIONS: bool = cfg!(all(target_os = "ios", target_arch = "arm")); + +pub unsafe fn find_eh_action(lsda: *const u8, context: &EHContext<'_>) -> Result<EHAction, ()> { + if lsda.is_null() { + return Ok(EHAction::None); + } + + let func_start = context.func_start; + let mut reader = DwarfReader::new(lsda); + + let start_encoding = reader.read::<u8>(); + // base address for landing pad offsets + let lpad_base = if start_encoding != DW_EH_PE_omit { + read_encoded_pointer(&mut reader, context, start_encoding)? + } else { + func_start + }; + + let ttype_encoding = reader.read::<u8>(); + if ttype_encoding != DW_EH_PE_omit { + // Rust doesn't analyze exception types, so we don't care about the type table + reader.read_uleb128(); + } + + let call_site_encoding = reader.read::<u8>(); + let call_site_table_length = reader.read_uleb128(); + let action_table = reader.ptr.add(call_site_table_length as usize); + let ip = context.ip; + + if !USING_SJLJ_EXCEPTIONS { + while reader.ptr < action_table { + let cs_start = read_encoded_pointer(&mut reader, context, call_site_encoding)?; + let cs_len = read_encoded_pointer(&mut reader, context, call_site_encoding)?; + let cs_lpad = read_encoded_pointer(&mut reader, context, call_site_encoding)?; + let cs_action_entry = reader.read_uleb128(); + // Callsite table is sorted by cs_start, so if we've passed the ip, we + // may stop searching. + if ip < func_start + cs_start { + break; + } + if ip < func_start + cs_start + cs_len { + if cs_lpad == 0 { + return Ok(EHAction::None); + } else { + let lpad = lpad_base + cs_lpad; + return Ok(interpret_cs_action(action_table as *mut u8, cs_action_entry, lpad)); + } + } + } + // Ip is not present in the table. This indicates a nounwind call. + Ok(EHAction::Terminate) + } else { + // SjLj version: + // The "IP" is an index into the call-site table, with two exceptions: + // -1 means 'no-action', and 0 means 'terminate'. + match ip as isize { + -1 => return Ok(EHAction::None), + 0 => return Ok(EHAction::Terminate), + _ => (), + } + let mut idx = ip; + loop { + let cs_lpad = reader.read_uleb128(); + let cs_action_entry = reader.read_uleb128(); + idx -= 1; + if idx == 0 { + // Can never have null landing pad for sjlj -- that would have + // been indicated by a -1 call site index. + let lpad = (cs_lpad + 1) as usize; + return Ok(interpret_cs_action(action_table as *mut u8, cs_action_entry, lpad)); + } + } + } +} + +unsafe fn interpret_cs_action( + action_table: *mut u8, + cs_action_entry: u64, + lpad: usize, +) -> EHAction { + if cs_action_entry == 0 { + // If cs_action_entry is 0 then this is a cleanup (Drop::drop). We run these + // for both Rust panics and foreign exceptions. + EHAction::Cleanup(lpad) + } else { + // If lpad != 0 and cs_action_entry != 0, we have to check ttype_index. + // If ttype_index == 0 under the condition, we take cleanup action. + let action_record = (action_table as *mut u8).offset(cs_action_entry as isize - 1); + let mut action_reader = DwarfReader::new(action_record); + let ttype_index = action_reader.read_sleb128(); + if ttype_index == 0 { + EHAction::Cleanup(lpad) + } else if ttype_index > 0 { + // Stop unwinding Rust panics at catch_unwind. + EHAction::Catch(lpad) + } else { + EHAction::Filter(lpad) + } + } +} + +#[inline] +fn round_up(unrounded: usize, align: usize) -> Result<usize, ()> { + if align.is_power_of_two() { Ok((unrounded + align - 1) & !(align - 1)) } else { Err(()) } +} + +unsafe fn read_encoded_pointer( + reader: &mut DwarfReader, + context: &EHContext<'_>, + encoding: u8, +) -> Result<usize, ()> { + if encoding == DW_EH_PE_omit { + return Err(()); + } + + // DW_EH_PE_aligned implies it's an absolute pointer value + if encoding == DW_EH_PE_aligned { + reader.ptr = reader.ptr.with_addr(round_up(reader.ptr.addr(), mem::size_of::<usize>())?); + return Ok(reader.read::<usize>()); + } + + let mut result = match encoding & 0x0F { + DW_EH_PE_absptr => reader.read::<usize>(), + DW_EH_PE_uleb128 => reader.read_uleb128() as usize, + DW_EH_PE_udata2 => reader.read::<u16>() as usize, + DW_EH_PE_udata4 => reader.read::<u32>() as usize, + DW_EH_PE_udata8 => reader.read::<u64>() as usize, + DW_EH_PE_sleb128 => reader.read_sleb128() as usize, + DW_EH_PE_sdata2 => reader.read::<i16>() as usize, + DW_EH_PE_sdata4 => reader.read::<i32>() as usize, + DW_EH_PE_sdata8 => reader.read::<i64>() as usize, + _ => return Err(()), + }; + + result += match encoding & 0x70 { + DW_EH_PE_absptr => 0, + // relative to address of the encoded value, despite the name + DW_EH_PE_pcrel => reader.ptr.expose_addr(), + DW_EH_PE_funcrel => { + if context.func_start == 0 { + return Err(()); + } + context.func_start + } + DW_EH_PE_textrel => (*context.get_text_start)(), + DW_EH_PE_datarel => (*context.get_data_start)(), + _ => return Err(()), + }; + + if encoding & DW_EH_PE_indirect != 0 { + result = *ptr::from_exposed_addr::<usize>(result); + } + + Ok(result) +} diff --git a/library/std/src/sys/personality/dwarf/mod.rs b/library/std/src/sys/personality/dwarf/mod.rs new file mode 100644 index 000000000..652fbe95a --- /dev/null +++ b/library/std/src/sys/personality/dwarf/mod.rs @@ -0,0 +1,73 @@ +//! Utilities for parsing DWARF-encoded data streams. +//! See <http://www.dwarfstd.org>, +//! DWARF-4 standard, Section 7 - "Data Representation" + +// This module is used only by x86_64-pc-windows-gnu for now, but we +// are compiling it everywhere to avoid regressions. +#![allow(unused)] + +#[cfg(test)] +mod tests; + +pub mod eh; + +use core::mem; + +pub struct DwarfReader { + pub ptr: *const u8, +} + +#[repr(C, packed)] +struct Unaligned<T>(T); + +impl DwarfReader { + pub fn new(ptr: *const u8) -> DwarfReader { + DwarfReader { ptr } + } + + // DWARF streams are packed, so e.g., a u32 would not necessarily be aligned + // on a 4-byte boundary. This may cause problems on platforms with strict + // alignment requirements. By wrapping data in a "packed" struct, we are + // telling the backend to generate "misalignment-safe" code. + pub unsafe fn read<T: Copy>(&mut self) -> T { + let Unaligned(result) = *(self.ptr as *const Unaligned<T>); + self.ptr = self.ptr.add(mem::size_of::<T>()); + result + } + + // ULEB128 and SLEB128 encodings are defined in Section 7.6 - "Variable + // Length Data". + pub unsafe fn read_uleb128(&mut self) -> u64 { + let mut shift: usize = 0; + let mut result: u64 = 0; + let mut byte: u8; + loop { + byte = self.read::<u8>(); + result |= ((byte & 0x7F) as u64) << shift; + shift += 7; + if byte & 0x80 == 0 { + break; + } + } + result + } + + pub unsafe fn read_sleb128(&mut self) -> i64 { + let mut shift: u32 = 0; + let mut result: u64 = 0; + let mut byte: u8; + loop { + byte = self.read::<u8>(); + result |= ((byte & 0x7F) as u64) << shift; + shift += 7; + if byte & 0x80 == 0 { + break; + } + } + // sign-extend + if shift < u64::BITS && (byte & 0x40) != 0 { + result |= (!0 as u64) << shift; + } + result as i64 + } +} diff --git a/library/std/src/sys/personality/dwarf/tests.rs b/library/std/src/sys/personality/dwarf/tests.rs new file mode 100644 index 000000000..1644f3708 --- /dev/null +++ b/library/std/src/sys/personality/dwarf/tests.rs @@ -0,0 +1,19 @@ +use super::*; + +#[test] +fn dwarf_reader() { + let encoded: &[u8] = &[1, 2, 3, 4, 5, 6, 7, 0xE5, 0x8E, 0x26, 0x9B, 0xF1, 0x59, 0xFF, 0xFF]; + + let mut reader = DwarfReader::new(encoded.as_ptr()); + + unsafe { + assert!(reader.read::<u8>() == u8::to_be(1u8)); + assert!(reader.read::<u16>() == u16::to_be(0x0203)); + assert!(reader.read::<u32>() == u32::to_be(0x04050607)); + + assert!(reader.read_uleb128() == 624485); + assert!(reader.read_sleb128() == -624485); + + assert!(reader.read::<i8>() == i8::to_be(-1)); + } +} diff --git a/library/std/src/sys/personality/emcc.rs b/library/std/src/sys/personality/emcc.rs new file mode 100644 index 000000000..cb52ae89b --- /dev/null +++ b/library/std/src/sys/personality/emcc.rs @@ -0,0 +1,20 @@ +//! On Emscripten Rust panics are wrapped in C++ exceptions, so we just forward +//! to `__gxx_personality_v0` which is provided by Emscripten. + +use crate::ffi::c_int; +use unwind as uw; + +// This is required by the compiler to exist (e.g., it's a lang item), but it's +// never actually called by the compiler. Emscripten EH doesn't use a +// personality function at all, it instead uses __cxa_find_matching_catch. +// Wasm error handling would use __gxx_personality_wasm0. +#[lang = "eh_personality"] +unsafe extern "C" fn rust_eh_personality( + _version: c_int, + _actions: uw::_Unwind_Action, + _exception_class: uw::_Unwind_Exception_Class, + _exception_object: *mut uw::_Unwind_Exception, + _context: *mut uw::_Unwind_Context, +) -> uw::_Unwind_Reason_Code { + core::intrinsics::abort() +} diff --git a/library/std/src/sys/personality/gcc.rs b/library/std/src/sys/personality/gcc.rs new file mode 100644 index 000000000..e477a0cd7 --- /dev/null +++ b/library/std/src/sys/personality/gcc.rs @@ -0,0 +1,294 @@ +//! Implementation of panics backed by libgcc/libunwind (in some form). +//! +//! For background on exception handling and stack unwinding please see +//! "Exception Handling in LLVM" (llvm.org/docs/ExceptionHandling.html) and +//! documents linked from it. +//! These are also good reads: +//! * <https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html> +//! * <https://monoinfinito.wordpress.com/series/exception-handling-in-c/> +//! * <https://www.airs.com/blog/index.php?s=exception+frames> +//! +//! ## A brief summary +//! +//! Exception handling happens in two phases: a search phase and a cleanup +//! phase. +//! +//! In both phases the unwinder walks stack frames from top to bottom using +//! information from the stack frame unwind sections of the current process's +//! modules ("module" here refers to an OS module, i.e., an executable or a +//! dynamic library). +//! +//! For each stack frame, it invokes the associated "personality routine", whose +//! address is also stored in the unwind info section. +//! +//! In the search phase, the job of a personality routine is to examine +//! exception object being thrown, and to decide whether it should be caught at +//! that stack frame. Once the handler frame has been identified, cleanup phase +//! begins. +//! +//! In the cleanup phase, the unwinder invokes each personality routine again. +//! This time it decides which (if any) cleanup code needs to be run for +//! the current stack frame. If so, the control is transferred to a special +//! branch in the function body, the "landing pad", which invokes destructors, +//! frees memory, etc. At the end of the landing pad, control is transferred +//! back to the unwinder and unwinding resumes. +//! +//! Once stack has been unwound down to the handler frame level, unwinding stops +//! and the last personality routine transfers control to the catch block. + +use super::dwarf::eh::{self, EHAction, EHContext}; +use crate::ffi::c_int; +use libc::uintptr_t; +use unwind as uw; + +// Register ids were lifted from LLVM's TargetLowering::getExceptionPointerRegister() +// and TargetLowering::getExceptionSelectorRegister() for each architecture, +// then mapped to DWARF register numbers via register definition tables +// (typically <arch>RegisterInfo.td, search for "DwarfRegNum"). +// See also https://llvm.org/docs/WritingAnLLVMBackend.html#defining-a-register. + +#[cfg(target_arch = "x86")] +const UNWIND_DATA_REG: (i32, i32) = (0, 2); // EAX, EDX + +#[cfg(target_arch = "x86_64")] +const UNWIND_DATA_REG: (i32, i32) = (0, 1); // RAX, RDX + +#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] +const UNWIND_DATA_REG: (i32, i32) = (0, 1); // R0, R1 / X0, X1 + +#[cfg(target_arch = "m68k")] +const UNWIND_DATA_REG: (i32, i32) = (0, 1); // D0, D1 + +#[cfg(any( + target_arch = "mips", + target_arch = "mips32r6", + target_arch = "mips64", + target_arch = "mips64r6" +))] +const UNWIND_DATA_REG: (i32, i32) = (4, 5); // A0, A1 + +#[cfg(target_arch = "csky")] +const UNWIND_DATA_REG: (i32, i32) = (0, 1); // R0, R1 + +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +const UNWIND_DATA_REG: (i32, i32) = (3, 4); // R3, R4 / X3, X4 + +#[cfg(target_arch = "s390x")] +const UNWIND_DATA_REG: (i32, i32) = (6, 7); // R6, R7 + +#[cfg(any(target_arch = "sparc", target_arch = "sparc64"))] +const UNWIND_DATA_REG: (i32, i32) = (24, 25); // I0, I1 + +#[cfg(target_arch = "hexagon")] +const UNWIND_DATA_REG: (i32, i32) = (0, 1); // R0, R1 + +#[cfg(any(target_arch = "riscv64", target_arch = "riscv32"))] +const UNWIND_DATA_REG: (i32, i32) = (10, 11); // x10, x11 + +#[cfg(target_arch = "loongarch64")] +const UNWIND_DATA_REG: (i32, i32) = (4, 5); // a0, a1 + +// The following code is based on GCC's C and C++ personality routines. For reference, see: +// https://github.com/gcc-mirror/gcc/blob/master/libstdc++-v3/libsupc++/eh_personality.cc +// https://github.com/gcc-mirror/gcc/blob/trunk/libgcc/unwind-c.c + +cfg_if::cfg_if! { + if #[cfg(all(target_arch = "arm", not(target_os = "ios"), not(target_os = "tvos"), not(target_os = "watchos"), not(target_os = "netbsd")))] { + // ARM EHABI personality routine. + // https://infocenter.arm.com/help/topic/com.arm.doc.ihi0038b/IHI0038B_ehabi.pdf + // + // iOS uses the default routine instead since it uses SjLj unwinding. + #[lang = "eh_personality"] + unsafe extern "C" fn rust_eh_personality( + state: uw::_Unwind_State, + exception_object: *mut uw::_Unwind_Exception, + context: *mut uw::_Unwind_Context, + ) -> uw::_Unwind_Reason_Code { + let state = state as c_int; + let action = state & uw::_US_ACTION_MASK as c_int; + let search_phase = if action == uw::_US_VIRTUAL_UNWIND_FRAME as c_int { + // Backtraces on ARM will call the personality routine with + // state == _US_VIRTUAL_UNWIND_FRAME | _US_FORCE_UNWIND. In those cases + // we want to continue unwinding the stack, otherwise all our backtraces + // would end at __rust_try + if state & uw::_US_FORCE_UNWIND as c_int != 0 { + return continue_unwind(exception_object, context); + } + true + } else if action == uw::_US_UNWIND_FRAME_STARTING as c_int { + false + } else if action == uw::_US_UNWIND_FRAME_RESUME as c_int { + return continue_unwind(exception_object, context); + } else { + return uw::_URC_FAILURE; + }; + + // The DWARF unwinder assumes that _Unwind_Context holds things like the function + // and LSDA pointers, however ARM EHABI places them into the exception object. + // To preserve signatures of functions like _Unwind_GetLanguageSpecificData(), which + // take only the context pointer, GCC personality routines stash a pointer to + // exception_object in the context, using location reserved for ARM's + // "scratch register" (r12). + uw::_Unwind_SetGR(context, uw::UNWIND_POINTER_REG, exception_object as uw::_Unwind_Ptr); + // ...A more principled approach would be to provide the full definition of ARM's + // _Unwind_Context in our libunwind bindings and fetch the required data from there + // directly, bypassing DWARF compatibility functions. + + let eh_action = match find_eh_action(context) { + Ok(action) => action, + Err(_) => return uw::_URC_FAILURE, + }; + if search_phase { + match eh_action { + EHAction::None | EHAction::Cleanup(_) => { + return continue_unwind(exception_object, context); + } + EHAction::Catch(_) | EHAction::Filter(_) => { + // EHABI requires the personality routine to update the + // SP value in the barrier cache of the exception object. + (*exception_object).private[5] = + uw::_Unwind_GetGR(context, uw::UNWIND_SP_REG); + return uw::_URC_HANDLER_FOUND; + } + EHAction::Terminate => return uw::_URC_FAILURE, + } + } else { + match eh_action { + EHAction::None => return continue_unwind(exception_object, context), + EHAction::Filter(_) if state & uw::_US_FORCE_UNWIND as c_int != 0 => return continue_unwind(exception_object, context), + EHAction::Cleanup(lpad) | EHAction::Catch(lpad) | EHAction::Filter(lpad) => { + uw::_Unwind_SetGR( + context, + UNWIND_DATA_REG.0, + exception_object as uintptr_t, + ); + uw::_Unwind_SetGR(context, UNWIND_DATA_REG.1, 0); + uw::_Unwind_SetIP(context, lpad); + return uw::_URC_INSTALL_CONTEXT; + } + EHAction::Terminate => return uw::_URC_FAILURE, + } + } + + // On ARM EHABI the personality routine is responsible for actually + // unwinding a single stack frame before returning (ARM EHABI Sec. 6.1). + unsafe fn continue_unwind( + exception_object: *mut uw::_Unwind_Exception, + context: *mut uw::_Unwind_Context, + ) -> uw::_Unwind_Reason_Code { + if __gnu_unwind_frame(exception_object, context) == uw::_URC_NO_REASON { + uw::_URC_CONTINUE_UNWIND + } else { + uw::_URC_FAILURE + } + } + // defined in libgcc + extern "C" { + fn __gnu_unwind_frame( + exception_object: *mut uw::_Unwind_Exception, + context: *mut uw::_Unwind_Context, + ) -> uw::_Unwind_Reason_Code; + } + } + } else { + // Default personality routine, which is used directly on most targets + // and indirectly on Windows x86_64 via SEH. + unsafe extern "C" fn rust_eh_personality_impl( + version: c_int, + actions: uw::_Unwind_Action, + _exception_class: uw::_Unwind_Exception_Class, + exception_object: *mut uw::_Unwind_Exception, + context: *mut uw::_Unwind_Context, + ) -> uw::_Unwind_Reason_Code { + if version != 1 { + return uw::_URC_FATAL_PHASE1_ERROR; + } + let eh_action = match find_eh_action(context) { + Ok(action) => action, + Err(_) => return uw::_URC_FATAL_PHASE1_ERROR, + }; + if actions as i32 & uw::_UA_SEARCH_PHASE as i32 != 0 { + match eh_action { + EHAction::None | EHAction::Cleanup(_) => uw::_URC_CONTINUE_UNWIND, + EHAction::Catch(_) | EHAction::Filter(_) => uw::_URC_HANDLER_FOUND, + EHAction::Terminate => uw::_URC_FATAL_PHASE1_ERROR, + } + } else { + match eh_action { + EHAction::None => uw::_URC_CONTINUE_UNWIND, + // Forced unwinding hits a terminate action. + EHAction::Filter(_) if actions as i32 & uw::_UA_FORCE_UNWIND as i32 != 0 => uw::_URC_CONTINUE_UNWIND, + EHAction::Cleanup(lpad) | EHAction::Catch(lpad) | EHAction::Filter(lpad) => { + uw::_Unwind_SetGR( + context, + UNWIND_DATA_REG.0, + exception_object as uintptr_t, + ); + uw::_Unwind_SetGR(context, UNWIND_DATA_REG.1, 0); + uw::_Unwind_SetIP(context, lpad); + uw::_URC_INSTALL_CONTEXT + } + EHAction::Terminate => uw::_URC_FATAL_PHASE2_ERROR, + } + } + } + + cfg_if::cfg_if! { + if #[cfg(all(windows, any(target_arch = "aarch64", target_arch = "x86_64"), target_env = "gnu"))] { + // On x86_64 MinGW targets, the unwinding mechanism is SEH however the unwind + // handler data (aka LSDA) uses GCC-compatible encoding. + #[lang = "eh_personality"] + #[allow(nonstandard_style)] + unsafe extern "C" fn rust_eh_personality( + exceptionRecord: *mut uw::EXCEPTION_RECORD, + establisherFrame: uw::LPVOID, + contextRecord: *mut uw::CONTEXT, + dispatcherContext: *mut uw::DISPATCHER_CONTEXT, + ) -> uw::EXCEPTION_DISPOSITION { + uw::_GCC_specific_handler( + exceptionRecord, + establisherFrame, + contextRecord, + dispatcherContext, + rust_eh_personality_impl, + ) + } + } else { + // The personality routine for most of our targets. + #[lang = "eh_personality"] + unsafe extern "C" fn rust_eh_personality( + version: c_int, + actions: uw::_Unwind_Action, + exception_class: uw::_Unwind_Exception_Class, + exception_object: *mut uw::_Unwind_Exception, + context: *mut uw::_Unwind_Context, + ) -> uw::_Unwind_Reason_Code { + rust_eh_personality_impl( + version, + actions, + exception_class, + exception_object, + context, + ) + } + } + } + } +} + +unsafe fn find_eh_action(context: *mut uw::_Unwind_Context) -> Result<EHAction, ()> { + let lsda = uw::_Unwind_GetLanguageSpecificData(context) as *const u8; + let mut ip_before_instr: c_int = 0; + let ip = uw::_Unwind_GetIPInfo(context, &mut ip_before_instr); + let eh_context = EHContext { + // The return address points 1 byte past the call instruction, + // which could be in the next IP range in LSDA range table. + // + // `ip = -1` has special meaning, so use wrapping sub to allow for that + ip: if ip_before_instr != 0 { ip } else { ip.wrapping_sub(1) }, + func_start: uw::_Unwind_GetRegionStart(context), + get_text_start: &|| uw::_Unwind_GetTextRelBase(context), + get_data_start: &|| uw::_Unwind_GetDataRelBase(context), + }; + eh::find_eh_action(lsda, &eh_context) +} diff --git a/library/std/src/sys/personality/mod.rs b/library/std/src/sys/personality/mod.rs new file mode 100644 index 000000000..386a399f5 --- /dev/null +++ b/library/std/src/sys/personality/mod.rs @@ -0,0 +1,46 @@ +//! This module contains the implementation of the `eh_personality` lang item. +//! +//! The actual implementation is heavily dependent on the target since Rust +//! tries to use the native stack unwinding mechanism whenever possible. +//! +//! This personality function is still required with `-C panic=abort` because +//! it is used to catch foreign exceptions from `extern "C-unwind"` and turn +//! them into aborts. +//! +//! Additionally, ARM EHABI uses the personality function when generating +//! backtraces. + +mod dwarf; + +#[cfg(not(test))] +cfg_if::cfg_if! { + if #[cfg(target_os = "emscripten")] { + mod emcc; + } else if #[cfg(target_env = "msvc")] { + // This is required by the compiler to exist (e.g., it's a lang item), + // but it's never actually called by the compiler because + // _CxxFrameHandler3 is the personality function that is always used. + // Hence this is just an aborting stub. + #[lang = "eh_personality"] + fn rust_eh_personality() { + core::intrinsics::abort() + } + } else if #[cfg(any( + all(target_family = "windows", target_env = "gnu"), + target_os = "psp", + target_os = "solid_asp3", + all(target_family = "unix", not(target_os = "espidf"), not(target_os = "l4re")), + all(target_vendor = "fortanix", target_env = "sgx"), + ))] { + mod gcc; + } else { + // Targets that don't support unwinding. + // - family=wasm + // - os=none ("bare metal" targets) + // - os=uefi + // - os=espidf + // - os=hermit + // - nvptx64-nvidia-cuda + // - arch=avr + } +} diff --git a/library/std/src/sys/sgx/os.rs b/library/std/src/sys/sgx/os.rs index 5da0257f3..86f4c7d3d 100644 --- a/library/std/src/sys/sgx/os.rs +++ b/library/std/src/sys/sgx/os.rs @@ -96,14 +96,61 @@ fn create_env_store() -> &'static EnvStore { unsafe { &*(ENV.load(Ordering::Relaxed) as *const EnvStore) } } -pub type Env = vec::IntoIter<(OsString, OsString)>; +pub struct Env { + iter: vec::IntoIter<(OsString, OsString)>, +} + +// FIXME(https://github.com/rust-lang/rust/issues/114583): Remove this when <OsStr as Debug>::fmt matches <str as Debug>::fmt. +pub struct EnvStrDebug<'a> { + slice: &'a [(OsString, OsString)], +} + +impl fmt::Debug for EnvStrDebug<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { slice } = self; + f.debug_list() + .entries(slice.iter().map(|(a, b)| (a.to_str().unwrap(), b.to_str().unwrap()))) + .finish() + } +} + +impl Env { + pub fn str_debug(&self) -> impl fmt::Debug + '_ { + let Self { iter } = self; + EnvStrDebug { slice: iter.as_slice() } + } +} + +impl fmt::Debug for Env { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { iter } = self; + f.debug_list().entries(iter.as_slice()).finish() + } +} + +impl !Send for Env {} +impl !Sync for Env {} + +impl Iterator for Env { + type Item = (OsString, OsString); + fn next(&mut self) -> Option<(OsString, OsString)> { + self.iter.next() + } + fn size_hint(&self) -> (usize, Option<usize>) { + self.iter.size_hint() + } +} pub fn env() -> Env { let clone_to_vec = |map: &HashMap<OsString, OsString>| -> Vec<_> { map.iter().map(|(k, v)| (k.clone(), v.clone())).collect() }; - get_env_store().map(|env| clone_to_vec(&env.lock().unwrap())).unwrap_or_default().into_iter() + let iter = get_env_store() + .map(|env| clone_to_vec(&env.lock().unwrap())) + .unwrap_or_default() + .into_iter(); + Env { iter } } pub fn getenv(k: &OsStr) -> Option<OsString> { diff --git a/library/std/src/sys/sgx/thread.rs b/library/std/src/sys/sgx/thread.rs index 1608b8cb6..7ac9d1d64 100644 --- a/library/std/src/sys/sgx/thread.rs +++ b/library/std/src/sys/sgx/thread.rs @@ -121,8 +121,16 @@ impl Thread { rtassert!(wait_error.kind() == io::ErrorKind::WouldBlock); } + /// SGX should protect in-enclave data from the outside (attacker), + /// so there should be no data leakage to the OS, + /// and therefore also no 1-1 mapping between SGX thread names and OS thread names. + /// + /// This is why the method is intentionally No-Op. pub fn set_name(_name: &CStr) { - // FIXME: could store this pointer in TLS somewhere + // Note that the internally visible SGX thread name is already provided + // by the platform-agnostic (target-agnostic) Rust thread code. + // This can be observed in the [`std::thread::tests::test_named_thread`] test, + // which succeeds as-is with the SGX target. } pub fn sleep(dur: Duration) { diff --git a/library/std/src/sys/solid/os.rs b/library/std/src/sys/solid/os.rs index 6135921f0..9f4e66d62 100644 --- a/library/std/src/sys/solid/os.rs +++ b/library/std/src/sys/solid/os.rs @@ -81,10 +81,42 @@ pub fn current_exe() -> io::Result<PathBuf> { static ENV_LOCK: RwLock<()> = RwLock::new(()); +pub fn env_read_lock() -> impl Drop { + ENV_LOCK.read().unwrap_or_else(PoisonError::into_inner) +} + pub struct Env { iter: vec::IntoIter<(OsString, OsString)>, } +// FIXME(https://github.com/rust-lang/rust/issues/114583): Remove this when <OsStr as Debug>::fmt matches <str as Debug>::fmt. +pub struct EnvStrDebug<'a> { + slice: &'a [(OsString, OsString)], +} + +impl fmt::Debug for EnvStrDebug<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { slice } = self; + f.debug_list() + .entries(slice.iter().map(|(a, b)| (a.to_str().unwrap(), b.to_str().unwrap()))) + .finish() + } +} + +impl Env { + pub fn str_debug(&self) -> impl fmt::Debug + '_ { + let Self { iter } = self; + EnvStrDebug { slice: iter.as_slice() } + } +} + +impl fmt::Debug for Env { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { iter } = self; + f.debug_list().entries(iter.as_slice()).finish() + } +} + impl !Send for Env {} impl !Sync for Env {} @@ -106,7 +138,7 @@ pub fn env() -> Env { } unsafe { - let _guard = ENV_LOCK.read(); + let _guard = env_read_lock(); let mut result = Vec::new(); if !environ.is_null() { while !(*environ).is_null() { @@ -140,17 +172,21 @@ pub fn env() -> Env { pub fn getenv(k: &OsStr) -> Option<OsString> { // environment variables with a nul byte can't be set, so their value is // always None as well - let s = run_with_cstr(k.as_bytes(), |k| { - let _guard = ENV_LOCK.read(); - Ok(unsafe { libc::getenv(k.as_ptr()) } as *const libc::c_char) - }) - .ok()?; + run_with_cstr(k.as_bytes(), |k| { + let _guard = env_read_lock(); + let v = unsafe { libc::getenv(k.as_ptr()) } as *const libc::c_char; - if s.is_null() { - None - } else { - Some(OsStringExt::from_vec(unsafe { CStr::from_ptr(s) }.to_bytes().to_vec())) - } + if v.is_null() { + Ok(None) + } else { + // SAFETY: `v` cannot be mutated while executing this line since we've a read lock + let bytes = unsafe { CStr::from_ptr(v) }.to_bytes().to_vec(); + + Ok(Some(OsStringExt::from_vec(bytes))) + } + }) + .ok() + .flatten() } pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { diff --git a/library/std/src/sys/unix/cmath.rs b/library/std/src/sys/unix/cmath.rs index 2bf80d7a4..5346d2291 100644 --- a/library/std/src/sys/unix/cmath.rs +++ b/library/std/src/sys/unix/cmath.rs @@ -30,4 +30,8 @@ extern "C" { pub fn tanf(n: f32) -> f32; pub fn tanh(n: f64) -> f64; pub fn tanhf(n: f32) -> f32; + pub fn tgamma(n: f64) -> f64; + pub fn tgammaf(n: f32) -> f32; + pub fn lgamma_r(n: f64, s: &mut i32) -> f64; + pub fn lgammaf_r(n: f32, s: &mut i32) -> f32; } diff --git a/library/std/src/sys/unix/fs.rs b/library/std/src/sys/unix/fs.rs index fbc7f04ce..a5604c92a 100644 --- a/library/std/src/sys/unix/fs.rs +++ b/library/std/src/sys/unix/fs.rs @@ -7,17 +7,6 @@ use crate::ffi::{CStr, OsStr, OsString}; use crate::fmt; use crate::io::{self, BorrowedCursor, Error, IoSlice, IoSliceMut, SeekFrom}; use crate::mem; -#[cfg(any( - target_os = "android", - target_os = "linux", - target_os = "solaris", - target_os = "fuchsia", - target_os = "redox", - target_os = "illumos", - target_os = "nto", - target_os = "vita", -))] -use crate::mem::MaybeUninit; use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd}; use crate::path::{Path, PathBuf}; use crate::ptr; @@ -712,22 +701,10 @@ impl Iterator for ReadDir { // requires the full extent of *entry_ptr to be in bounds of the same // allocation, which is not necessarily the case here. // - // Absent any other way to obtain a pointer to `(*entry_ptr).d_name` - // legally in Rust analogously to how it would be done in C, we instead - // need to make our own non-libc allocation that conforms to the weird - // imaginary definition of dirent64, and use that for a field offset - // computation. + // Instead we must access fields individually through their offsets. macro_rules! offset_ptr { ($entry_ptr:expr, $field:ident) => {{ - const OFFSET: isize = { - let delusion = MaybeUninit::<dirent64>::uninit(); - let entry_ptr = delusion.as_ptr(); - unsafe { - ptr::addr_of!((*entry_ptr).$field) - .cast::<u8>() - .offset_from(entry_ptr.cast::<u8>()) - } - }; + const OFFSET: isize = mem::offset_of!(dirent64, $field) as isize; if true { // Cast to the same type determined by the else branch. $entry_ptr.byte_offset(OFFSET).cast::<_>() @@ -1227,6 +1204,7 @@ impl File { self.0.write_vectored_at(bufs, offset) } + #[inline] pub fn flush(&self) -> io::Result<()> { Ok(()) } diff --git a/library/std/src/sys/unix/kernel_copy.rs b/library/std/src/sys/unix/kernel_copy.rs index 7d49bbdcb..4d17a1b00 100644 --- a/library/std/src/sys/unix/kernel_copy.rs +++ b/library/std/src/sys/unix/kernel_copy.rs @@ -89,6 +89,12 @@ enum FdMeta { NoneObtained, } +#[derive(PartialEq)] +enum FdHandle { + Input, + Output, +} + impl FdMeta { fn maybe_fifo(&self) -> bool { match self { @@ -114,12 +120,14 @@ impl FdMeta { } } - fn copy_file_range_candidate(&self) -> bool { + fn copy_file_range_candidate(&self, f: FdHandle) -> bool { match self { // copy_file_range will fail on empty procfs files. `read` can determine whether EOF has been reached // without extra cost and skip the write, thus there is no benefit in attempting copy_file_range - FdMeta::Metadata(meta) if meta.is_file() && meta.len() > 0 => true, - FdMeta::NoneObtained => true, + FdMeta::Metadata(meta) if f == FdHandle::Input && meta.is_file() && meta.len() > 0 => { + true + } + FdMeta::Metadata(meta) if f == FdHandle::Output && meta.is_file() => true, _ => false, } } @@ -197,7 +205,9 @@ impl<R: CopyRead, W: CopyWrite> SpecCopy for Copier<'_, '_, R, W> { written += flush()?; let max_write = reader.min_limit(); - if input_meta.copy_file_range_candidate() && output_meta.copy_file_range_candidate() { + if input_meta.copy_file_range_candidate(FdHandle::Input) + && output_meta.copy_file_range_candidate(FdHandle::Output) + { let result = copy_regular_files(readfd, writefd, max_write); result.update_take(reader); diff --git a/library/std/src/sys/unix/mod.rs b/library/std/src/sys/unix/mod.rs index 326f1481e..77ef086f2 100644 --- a/library/std/src/sys/unix/mod.rs +++ b/library/std/src/sys/unix/mod.rs @@ -110,6 +110,11 @@ pub unsafe fn init(argc: isize, argv: *const *const u8, sigpipe: u8) { while libc::poll(pfds.as_mut_ptr(), 3, 0) == -1 { match errno() { libc::EINTR => continue, + #[cfg(target_vendor = "unikraft")] + libc::ENOSYS => { + // Not all configurations of Unikraft enable `LIBPOSIX_EVENT`. + break 'poll; + } libc::EINVAL | libc::EAGAIN | libc::ENOMEM => { // RLIMIT_NOFILE or temporary allocation failures // may be preventing use of poll(), fall back to fcntl @@ -165,7 +170,14 @@ pub unsafe fn init(argc: isize, argv: *const *const u8, sigpipe: u8) { } unsafe fn reset_sigpipe(#[allow(unused_variables)] sigpipe: u8) { - #[cfg(not(any(target_os = "emscripten", target_os = "fuchsia", target_os = "horizon")))] + #[cfg(not(any( + target_os = "emscripten", + target_os = "fuchsia", + target_os = "horizon", + // Unikraft's `signal` implementation is currently broken: + // https://github.com/unikraft/lib-musl/issues/57 + target_vendor = "unikraft", + )))] { // We don't want to add this as a public type to std, nor do we // want to `include!` a file from the compiler (which would break diff --git a/library/std/src/sys/unix/os.rs b/library/std/src/sys/unix/os.rs index a68c14758..57e1a36da 100644 --- a/library/std/src/sys/unix/os.rs +++ b/library/std/src/sys/unix/os.rs @@ -495,6 +495,34 @@ pub struct Env { iter: vec::IntoIter<(OsString, OsString)>, } +// FIXME(https://github.com/rust-lang/rust/issues/114583): Remove this when <OsStr as Debug>::fmt matches <str as Debug>::fmt. +pub struct EnvStrDebug<'a> { + slice: &'a [(OsString, OsString)], +} + +impl fmt::Debug for EnvStrDebug<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { slice } = self; + f.debug_list() + .entries(slice.iter().map(|(a, b)| (a.to_str().unwrap(), b.to_str().unwrap()))) + .finish() + } +} + +impl Env { + pub fn str_debug(&self) -> impl fmt::Debug + '_ { + let Self { iter } = self; + EnvStrDebug { slice: iter.as_slice() } + } +} + +impl fmt::Debug for Env { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { iter } = self; + f.debug_list().entries(iter.as_slice()).finish() + } +} + impl !Send for Env {} impl !Sync for Env {} @@ -566,16 +594,21 @@ pub fn env() -> Env { pub fn getenv(k: &OsStr) -> Option<OsString> { // environment variables with a nul byte can't be set, so their value is // always None as well - let s = run_with_cstr(k.as_bytes(), |k| { + run_with_cstr(k.as_bytes(), |k| { let _guard = env_read_lock(); - Ok(unsafe { libc::getenv(k.as_ptr()) } as *const libc::c_char) + let v = unsafe { libc::getenv(k.as_ptr()) } as *const libc::c_char; + + if v.is_null() { + Ok(None) + } else { + // SAFETY: `v` cannot be mutated while executing this line since we've a read lock + let bytes = unsafe { CStr::from_ptr(v) }.to_bytes().to_vec(); + + Ok(Some(OsStringExt::from_vec(bytes))) + } }) - .ok()?; - if s.is_null() { - None - } else { - Some(OsStringExt::from_vec(unsafe { CStr::from_ptr(s) }.to_bytes().to_vec())) - } + .ok() + .flatten() } pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { diff --git a/library/std/src/sys/unix/os_str.rs b/library/std/src/sys/unix/os_str.rs index f7333fd5a..463b0a275 100644 --- a/library/std/src/sys/unix/os_str.rs +++ b/library/std/src/sys/unix/os_str.rs @@ -96,6 +96,16 @@ impl AsInner<[u8]> for Buf { } impl Buf { + #[inline] + pub fn into_os_str_bytes(self) -> Vec<u8> { + self.inner + } + + #[inline] + pub unsafe fn from_os_str_bytes_unchecked(s: Vec<u8>) -> Self { + Self { inner: s } + } + pub fn from_string(s: String) -> Buf { Buf { inner: s.into_bytes() } } diff --git a/library/std/src/sys/unix/process/process_fuchsia.rs b/library/std/src/sys/unix/process/process_fuchsia.rs index e45c380a0..9931c2af2 100644 --- a/library/std/src/sys/unix/process/process_fuchsia.rs +++ b/library/std/src/sys/unix/process/process_fuchsia.rs @@ -235,7 +235,7 @@ impl Process { } } -#[derive(PartialEq, Eq, Clone, Copy, Debug)] +#[derive(PartialEq, Eq, Clone, Copy, Debug, Default)] pub struct ExitStatus(i64); impl ExitStatus { diff --git a/library/std/src/sys/unix/process/process_unix.rs b/library/std/src/sys/unix/process/process_unix.rs index 0ce93af66..3963e7f52 100644 --- a/library/std/src/sys/unix/process/process_unix.rs +++ b/library/std/src/sys/unix/process/process_unix.rs @@ -10,9 +10,6 @@ use core::ffi::NonZero_c_int; #[cfg(target_os = "linux")] use crate::os::linux::process::PidFd; -#[cfg(target_os = "linux")] -use crate::sys::weak::raw_syscall; - #[cfg(any( target_os = "macos", target_os = "watchos", @@ -91,6 +88,11 @@ impl Command { if let Some(ret) = self.posix_spawn(&theirs, envp.as_ref())? { return Ok((ret, ours)); } + + #[cfg(target_os = "linux")] + let (input, output) = sys::net::Socket::new_pair(libc::AF_UNIX, libc::SOCK_SEQPACKET)?; + + #[cfg(not(target_os = "linux"))] let (input, output) = sys::pipe::anon_pipe()?; // Whatever happens after the fork is almost for sure going to touch or @@ -104,12 +106,16 @@ impl Command { // The child calls `mem::forget` to leak the lock, which is crucial because // releasing a lock is not async-signal-safe. let env_lock = sys::os::env_read_lock(); - let (pid, pidfd) = unsafe { self.do_fork()? }; + let pid = unsafe { self.do_fork()? }; if pid == 0 { crate::panic::always_abort(); mem::forget(env_lock); // avoid non-async-signal-safe unlocking drop(input); + #[cfg(target_os = "linux")] + if self.get_create_pidfd() { + self.send_pidfd(&output); + } let Err(err) = unsafe { self.do_exec(theirs, envp.as_ref()) }; let errno = err.raw_os_error().unwrap_or(libc::EINVAL) as u32; let errno = errno.to_be_bytes(); @@ -133,6 +139,12 @@ impl Command { drop(env_lock); drop(output); + #[cfg(target_os = "linux")] + let pidfd = if self.get_create_pidfd() { self.recv_pidfd(&input) } else { -1 }; + + #[cfg(not(target_os = "linux"))] + let pidfd = -1; + // Safety: We obtained the pidfd from calling `clone3` with // `CLONE_PIDFD` so it's valid an otherwise unowned. let mut p = unsafe { Process::new(pid, pidfd) }; @@ -160,6 +172,7 @@ impl Command { } Ok(..) => { // pipe I/O up to PIPE_BUF bytes should be atomic + // similarly SOCK_SEQPACKET messages should arrive whole assert!(p.wait().is_ok(), "wait() should either return Ok or panic"); panic!("short read on the CLOEXEC pipe") } @@ -185,20 +198,19 @@ impl Command { ); #[cfg(any(target_os = "tvos", target_os = "watchos"))] - unsafe fn do_fork(&mut self) -> Result<(pid_t, pid_t), io::Error> { + unsafe fn do_fork(&mut self) -> Result<pid_t, io::Error> { return Err(Self::ERR_APPLE_TV_WATCH_NO_FORK_EXEC); } // Attempts to fork the process. If successful, returns Ok((0, -1)) // in the child, and Ok((child_pid, -1)) in the parent. #[cfg(not(any( - target_os = "linux", target_os = "watchos", target_os = "tvos", all(target_os = "nto", target_env = "nto71"), )))] - unsafe fn do_fork(&mut self) -> Result<(pid_t, pid_t), io::Error> { - cvt(libc::fork()).map(|res| (res, -1)) + unsafe fn do_fork(&mut self) -> Result<pid_t, io::Error> { + cvt(libc::fork()) } // On QNX Neutrino, fork can fail with EBADF in case "another thread might have opened @@ -206,7 +218,7 @@ impl Command { // Documentation says "... or try calling fork() again". This is what we do here. // See also https://www.qnx.com/developers/docs/7.1/#com.qnx.doc.neutrino.lib_ref/topic/f/fork.html #[cfg(all(target_os = "nto", target_env = "nto71"))] - unsafe fn do_fork(&mut self) -> Result<(pid_t, pid_t), io::Error> { + unsafe fn do_fork(&mut self) -> Result<pid_t, io::Error> { use crate::sys::os::errno; let mut delay = MIN_FORKSPAWN_SLEEP; @@ -229,91 +241,11 @@ impl Command { delay *= 2; continue; } else { - return cvt(r).map(|res| (res, -1)); + return cvt(r); } } } - // Attempts to fork the process. If successful, returns Ok((0, -1)) - // in the child, and Ok((child_pid, child_pidfd)) in the parent. - #[cfg(target_os = "linux")] - unsafe fn do_fork(&mut self) -> Result<(pid_t, pid_t), io::Error> { - use crate::sync::atomic::{AtomicBool, Ordering}; - - static HAS_CLONE3: AtomicBool = AtomicBool::new(true); - const CLONE_PIDFD: u64 = 0x00001000; - - #[repr(C)] - struct clone_args { - flags: u64, - pidfd: u64, - child_tid: u64, - parent_tid: u64, - exit_signal: u64, - stack: u64, - stack_size: u64, - tls: u64, - set_tid: u64, - set_tid_size: u64, - cgroup: u64, - } - - raw_syscall! { - fn clone3(cl_args: *mut clone_args, len: libc::size_t) -> libc::c_long - } - - // Bypassing libc for `clone3` can make further libc calls unsafe, - // so we use it sparingly for now. See #89522 for details. - // Some tools (e.g. sandboxing tools) may also expect `fork` - // rather than `clone3`. - let want_clone3_pidfd = self.get_create_pidfd(); - - // If we fail to create a pidfd for any reason, this will - // stay as -1, which indicates an error. - let mut pidfd: pid_t = -1; - - // Attempt to use the `clone3` syscall, which supports more arguments - // (in particular, the ability to create a pidfd). If this fails, - // we will fall through this block to a call to `fork()` - if want_clone3_pidfd && HAS_CLONE3.load(Ordering::Relaxed) { - let mut args = clone_args { - flags: CLONE_PIDFD, - pidfd: &mut pidfd as *mut pid_t as u64, - child_tid: 0, - parent_tid: 0, - exit_signal: libc::SIGCHLD as u64, - stack: 0, - stack_size: 0, - tls: 0, - set_tid: 0, - set_tid_size: 0, - cgroup: 0, - }; - - let args_ptr = &mut args as *mut clone_args; - let args_size = crate::mem::size_of::<clone_args>(); - - let res = cvt(clone3(args_ptr, args_size)); - match res { - Ok(n) => return Ok((n as pid_t, pidfd)), - Err(e) => match e.raw_os_error() { - // Multiple threads can race to execute this store, - // but that's fine - that just means that multiple threads - // will have tried and failed to execute the same syscall, - // with no other side effects. - Some(libc::ENOSYS) => HAS_CLONE3.store(false, Ordering::Relaxed), - // Fallback to fork if `EPERM` is returned. (e.g. blocked by seccomp) - Some(libc::EPERM) => {} - _ => return Err(e), - }, - } - } - - // Generally, we just call `fork`. If we get here after wanting `clone3`, - // then the syscall does not exist or we do not have permission to call it. - cvt(libc::fork()).map(|res| (res, pidfd)) - } - pub fn exec(&mut self, default: Stdio) -> io::Error { let envp = self.capture_env(); @@ -722,6 +654,115 @@ impl Command { Ok(Some(p)) } } + + #[cfg(target_os = "linux")] + fn send_pidfd(&self, sock: &crate::sys::net::Socket) { + use crate::io::IoSlice; + use crate::os::fd::RawFd; + use crate::sys::cvt_r; + use libc::{CMSG_DATA, CMSG_FIRSTHDR, CMSG_LEN, CMSG_SPACE, SCM_RIGHTS, SOL_SOCKET}; + + unsafe { + let child_pid = libc::getpid(); + // pidfd_open sets CLOEXEC by default + let pidfd = libc::syscall(libc::SYS_pidfd_open, child_pid, 0); + + let fds: [c_int; 1] = [pidfd as RawFd]; + + const SCM_MSG_LEN: usize = mem::size_of::<[c_int; 1]>(); + + #[repr(C)] + union Cmsg { + buf: [u8; unsafe { CMSG_SPACE(SCM_MSG_LEN as u32) as usize }], + _align: libc::cmsghdr, + } + + let mut cmsg: Cmsg = mem::zeroed(); + + // 0-length message to send through the socket so we can pass along the fd + let mut iov = [IoSlice::new(b"")]; + let mut msg: libc::msghdr = mem::zeroed(); + + msg.msg_iov = &mut iov as *mut _ as *mut _; + msg.msg_iovlen = 1; + msg.msg_controllen = mem::size_of_val(&cmsg.buf) as _; + msg.msg_control = &mut cmsg.buf as *mut _ as *mut _; + + // only attach cmsg if we successfully acquired the pidfd + if pidfd >= 0 { + let hdr = CMSG_FIRSTHDR(&mut msg as *mut _ as *mut _); + (*hdr).cmsg_level = SOL_SOCKET; + (*hdr).cmsg_type = SCM_RIGHTS; + (*hdr).cmsg_len = CMSG_LEN(SCM_MSG_LEN as _) as _; + let data = CMSG_DATA(hdr); + crate::ptr::copy_nonoverlapping( + fds.as_ptr().cast::<u8>(), + data as *mut _, + SCM_MSG_LEN, + ); + } + + // we send the 0-length message even if we failed to acquire the pidfd + // so we get a consistent SEQPACKET order + match cvt_r(|| libc::sendmsg(sock.as_raw(), &msg, 0)) { + Ok(0) => {} + _ => rtabort!("failed to communicate with parent process"), + } + } + } + + #[cfg(target_os = "linux")] + fn recv_pidfd(&self, sock: &crate::sys::net::Socket) -> pid_t { + use crate::io::IoSliceMut; + use crate::sys::cvt_r; + + use libc::{CMSG_DATA, CMSG_FIRSTHDR, CMSG_LEN, CMSG_SPACE, SCM_RIGHTS, SOL_SOCKET}; + + unsafe { + const SCM_MSG_LEN: usize = mem::size_of::<[c_int; 1]>(); + + #[repr(C)] + union Cmsg { + _buf: [u8; unsafe { CMSG_SPACE(SCM_MSG_LEN as u32) as usize }], + _align: libc::cmsghdr, + } + let mut cmsg: Cmsg = mem::zeroed(); + // 0-length read to get the fd + let mut iov = [IoSliceMut::new(&mut [])]; + + let mut msg: libc::msghdr = mem::zeroed(); + + msg.msg_iov = &mut iov as *mut _ as *mut _; + msg.msg_iovlen = 1; + msg.msg_controllen = mem::size_of::<Cmsg>() as _; + msg.msg_control = &mut cmsg as *mut _ as *mut _; + + match cvt_r(|| libc::recvmsg(sock.as_raw(), &mut msg, 0)) { + Err(_) => return -1, + Ok(_) => {} + } + + let hdr = CMSG_FIRSTHDR(&mut msg as *mut _ as *mut _); + if hdr.is_null() + || (*hdr).cmsg_level != SOL_SOCKET + || (*hdr).cmsg_type != SCM_RIGHTS + || (*hdr).cmsg_len != CMSG_LEN(SCM_MSG_LEN as _) as _ + { + return -1; + } + let data = CMSG_DATA(hdr); + + let mut fds = [-1 as c_int]; + + crate::ptr::copy_nonoverlapping( + data as *const _, + fds.as_mut_ptr().cast::<u8>(), + SCM_MSG_LEN, + ); + + fds[0] + } + } } //////////////////////////////////////////////////////////////////////////////// @@ -800,7 +841,7 @@ impl Process { // // This is not actually an "exit status" in Unix terminology. Rather, it is a "wait status". // See the discussion in comments and doc comments for `std::process::ExitStatus`. -#[derive(PartialEq, Eq, Clone, Copy)] +#[derive(PartialEq, Eq, Clone, Copy, Default)] pub struct ExitStatus(c_int); impl fmt::Debug for ExitStatus { diff --git a/library/std/src/sys/unix/process/process_unix/tests.rs b/library/std/src/sys/unix/process/process_unix/tests.rs index e5e1f956b..6aa79e7f9 100644 --- a/library/std/src/sys/unix/process/process_unix/tests.rs +++ b/library/std/src/sys/unix/process/process_unix/tests.rs @@ -60,3 +60,28 @@ fn test_command_fork_no_unwind() { || signal == libc::SIGSEGV ); } + +#[test] +#[cfg(target_os = "linux")] +fn test_command_pidfd() { + use crate::os::fd::RawFd; + use crate::os::linux::process::{ChildExt, CommandExt}; + use crate::process::Command; + + let our_pid = crate::process::id(); + let pidfd = unsafe { libc::syscall(libc::SYS_pidfd_open, our_pid, 0) }; + let pidfd_open_available = if pidfd >= 0 { + unsafe { libc::close(pidfd as RawFd) }; + true + } else { + false + }; + + // always exercise creation attempts + let child = Command::new("echo").create_pidfd(true).spawn().unwrap(); + + // but only check if we know that the kernel supports pidfds + if pidfd_open_available { + assert!(child.pidfd().is_ok()) + } +} diff --git a/library/std/src/sys/unix/process/process_unsupported.rs b/library/std/src/sys/unix/process/process_unsupported.rs index f28ca58d0..8e0b971af 100644 --- a/library/std/src/sys/unix/process/process_unsupported.rs +++ b/library/std/src/sys/unix/process/process_unsupported.rs @@ -55,7 +55,7 @@ impl Process { } } -#[derive(PartialEq, Eq, Clone, Copy, Debug)] +#[derive(PartialEq, Eq, Clone, Copy, Debug, Default)] pub struct ExitStatus(c_int); impl ExitStatus { diff --git a/library/std/src/sys/unix/process/process_vxworks.rs b/library/std/src/sys/unix/process/process_vxworks.rs index f70d3cb39..1ff2b2fb3 100644 --- a/library/std/src/sys/unix/process/process_vxworks.rs +++ b/library/std/src/sys/unix/process/process_vxworks.rs @@ -179,7 +179,7 @@ impl Process { } /// Unix exit statuses -#[derive(PartialEq, Eq, Clone, Copy, Debug)] +#[derive(PartialEq, Eq, Clone, Copy, Debug, Default)] pub struct ExitStatus(c_int); impl ExitStatus { diff --git a/library/std/src/sys/unix/rand.rs b/library/std/src/sys/unix/rand.rs index d471be33e..fbf158f56 100644 --- a/library/std/src/sys/unix/rand.rs +++ b/library/std/src/sys/unix/rand.rs @@ -17,7 +17,6 @@ pub fn hashmap_random_keys() -> (u64, u64) { not(target_os = "tvos"), not(target_os = "watchos"), not(target_os = "openbsd"), - not(target_os = "freebsd"), not(target_os = "netbsd"), not(target_os = "fuchsia"), not(target_os = "redox"), @@ -68,11 +67,25 @@ mod imp { unsafe { libc::getrandom(buf.as_mut_ptr().cast(), buf.len(), 0) } } + #[cfg(target_os = "freebsd")] + fn getrandom(buf: &mut [u8]) -> libc::ssize_t { + // FIXME: using the above when libary std's libc is updated + extern "C" { + fn getrandom( + buffer: *mut libc::c_void, + length: libc::size_t, + flags: libc::c_uint, + ) -> libc::ssize_t; + } + unsafe { getrandom(buf.as_mut_ptr().cast(), buf.len(), 0) } + } + #[cfg(not(any( target_os = "linux", target_os = "android", target_os = "espidf", - target_os = "horizon" + target_os = "horizon", + target_os = "freebsd" )))] fn getrandom_fill_bytes(_buf: &mut [u8]) -> bool { false @@ -82,7 +95,8 @@ mod imp { target_os = "linux", target_os = "android", target_os = "espidf", - target_os = "horizon" + target_os = "horizon", + target_os = "freebsd" ))] fn getrandom_fill_bytes(v: &mut [u8]) -> bool { use crate::sync::atomic::{AtomicBool, Ordering}; @@ -222,7 +236,7 @@ mod imp { } } -#[cfg(any(target_os = "freebsd", target_os = "netbsd"))] +#[cfg(target_os = "netbsd")] mod imp { use crate::ptr; diff --git a/library/std/src/sys/unix/stdio.rs b/library/std/src/sys/unix/stdio.rs index a26f20795..97e75f1b5 100644 --- a/library/std/src/sys/unix/stdio.rs +++ b/library/std/src/sys/unix/stdio.rs @@ -54,6 +54,7 @@ impl io::Write for Stdout { true } + #[inline] fn flush(&mut self) -> io::Result<()> { Ok(()) } @@ -81,6 +82,7 @@ impl io::Write for Stderr { true } + #[inline] fn flush(&mut self) -> io::Result<()> { Ok(()) } diff --git a/library/std/src/sys/unsupported/os.rs b/library/std/src/sys/unsupported/os.rs index e150ae143..248b34829 100644 --- a/library/std/src/sys/unsupported/os.rs +++ b/library/std/src/sys/unsupported/os.rs @@ -65,10 +65,26 @@ pub fn current_exe() -> io::Result<PathBuf> { pub struct Env(!); +impl Env { + // FIXME(https://github.com/rust-lang/rust/issues/114583): Remove this when <OsStr as Debug>::fmt matches <str as Debug>::fmt. + pub fn str_debug(&self) -> impl fmt::Debug + '_ { + let Self(inner) = self; + match *inner {} + } +} + +impl fmt::Debug for Env { + fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self(inner) = self; + match *inner {} + } +} + impl Iterator for Env { type Item = (OsString, OsString); fn next(&mut self) -> Option<(OsString, OsString)> { - self.0 + let Self(inner) = self; + match *inner {} } } diff --git a/library/std/src/sys/unsupported/process.rs b/library/std/src/sys/unsupported/process.rs index a494f2d6b..77b675aaa 100644 --- a/library/std/src/sys/unsupported/process.rs +++ b/library/std/src/sys/unsupported/process.rs @@ -99,58 +99,59 @@ impl fmt::Debug for Command { } } -pub struct ExitStatus(!); +#[derive(PartialEq, Eq, Clone, Copy, Debug, Default)] +#[non_exhaustive] +pub struct ExitStatus(); impl ExitStatus { pub fn exit_ok(&self) -> Result<(), ExitStatusError> { - self.0 + Ok(()) } pub fn code(&self) -> Option<i32> { - self.0 + Some(0) } } -impl Clone for ExitStatus { - fn clone(&self) -> ExitStatus { - self.0 +impl fmt::Display for ExitStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "<dummy exit status>") } } -impl Copy for ExitStatus {} +pub struct ExitStatusError(!); -impl PartialEq for ExitStatus { - fn eq(&self, _other: &ExitStatus) -> bool { +impl Clone for ExitStatusError { + fn clone(&self) -> ExitStatusError { self.0 } } -impl Eq for ExitStatus {} +impl Copy for ExitStatusError {} -impl fmt::Debug for ExitStatus { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl PartialEq for ExitStatusError { + fn eq(&self, _other: &ExitStatusError) -> bool { self.0 } } -impl fmt::Display for ExitStatus { +impl Eq for ExitStatusError {} + +impl fmt::Debug for ExitStatusError { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0 } } -#[derive(PartialEq, Eq, Clone, Copy, Debug)] -pub struct ExitStatusError(ExitStatus); - impl Into<ExitStatus> for ExitStatusError { fn into(self) -> ExitStatus { - self.0.0 + self.0 } } impl ExitStatusError { pub fn code(self) -> Option<NonZeroI32> { - self.0.0 + self.0 } } diff --git a/library/std/src/sys/wasi/fd.rs b/library/std/src/sys/wasi/fd.rs index 1b50c2ea6..d7295a799 100644 --- a/library/std/src/sys/wasi/fd.rs +++ b/library/std/src/sys/wasi/fd.rs @@ -16,14 +16,20 @@ pub struct WasiFd { fn iovec<'a>(a: &'a mut [IoSliceMut<'_>]) -> &'a [wasi::Iovec] { assert_eq!(mem::size_of::<IoSliceMut<'_>>(), mem::size_of::<wasi::Iovec>()); assert_eq!(mem::align_of::<IoSliceMut<'_>>(), mem::align_of::<wasi::Iovec>()); - // SAFETY: `IoSliceMut` and `IoVec` have exactly the same memory layout + // SAFETY: `IoSliceMut` and `IoVec` have exactly the same memory layout. + // We decorate our `IoSliceMut` with `repr(transparent)` (see `io.rs`), and + // `crate::io::IoSliceMut` is a `repr(transparent)` wrapper around our type, so this is + // guaranteed. unsafe { mem::transmute(a) } } fn ciovec<'a>(a: &'a [IoSlice<'_>]) -> &'a [wasi::Ciovec] { assert_eq!(mem::size_of::<IoSlice<'_>>(), mem::size_of::<wasi::Ciovec>()); assert_eq!(mem::align_of::<IoSlice<'_>>(), mem::align_of::<wasi::Ciovec>()); - // SAFETY: `IoSlice` and `CIoVec` have exactly the same memory layout + // SAFETY: `IoSlice` and `CIoVec` have exactly the same memory layout. + // We decorate our `IoSlice` with `repr(transparent)` (see `io.rs`), and + // `crate::io::IoSlice` is a `repr(transparent)` wrapper around our type, so this is + // guaranteed. unsafe { mem::transmute(a) } } diff --git a/library/std/src/sys/wasi/mod.rs b/library/std/src/sys/wasi/mod.rs index a22237080..98517da1d 100644 --- a/library/std/src/sys/wasi/mod.rs +++ b/library/std/src/sys/wasi/mod.rs @@ -29,8 +29,7 @@ pub mod fs; #[path = "../wasm/atomics/futex.rs"] pub mod futex; pub mod io; -#[path = "../unsupported/locks/mod.rs"] -pub mod locks; + pub mod net; pub mod os; #[path = "../unix/os_str.rs"] @@ -47,14 +46,27 @@ pub mod thread; pub mod thread_local_dtor; #[path = "../unsupported/thread_local_key.rs"] pub mod thread_local_key; -#[path = "../unsupported/thread_parking.rs"] -pub mod thread_parking; pub mod time; cfg_if::cfg_if! { - if #[cfg(not(target_feature = "atomics"))] { + if #[cfg(target_feature = "atomics")] { + #[path = "../unix/locks"] + pub mod locks { + #![allow(unsafe_op_in_unsafe_fn)] + mod futex_condvar; + mod futex_mutex; + mod futex_rwlock; + pub(crate) use futex_condvar::Condvar; + pub(crate) use futex_mutex::Mutex; + pub(crate) use futex_rwlock::RwLock; + } + } else { + #[path = "../unsupported/locks/mod.rs"] + pub mod locks; #[path = "../unsupported/once.rs"] pub mod once; + #[path = "../unsupported/thread_parking.rs"] + pub mod thread_parking; } } diff --git a/library/std/src/sys/wasi/os.rs b/library/std/src/sys/wasi/os.rs index 9919dc708..d53bddd8e 100644 --- a/library/std/src/sys/wasi/os.rs +++ b/library/std/src/sys/wasi/os.rs @@ -142,10 +142,39 @@ impl StdError for JoinPathsError { pub fn current_exe() -> io::Result<PathBuf> { unsupported() } + pub struct Env { iter: vec::IntoIter<(OsString, OsString)>, } +// FIXME(https://github.com/rust-lang/rust/issues/114583): Remove this when <OsStr as Debug>::fmt matches <str as Debug>::fmt. +pub struct EnvStrDebug<'a> { + slice: &'a [(OsString, OsString)], +} + +impl fmt::Debug for EnvStrDebug<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { slice } = self; + f.debug_list() + .entries(slice.iter().map(|(a, b)| (a.to_str().unwrap(), b.to_str().unwrap()))) + .finish() + } +} + +impl Env { + pub fn str_debug(&self) -> impl fmt::Debug + '_ { + let Self { iter } = self; + EnvStrDebug { slice: iter.as_slice() } + } +} + +impl fmt::Debug for Env { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { iter } = self; + f.debug_list().entries(iter.as_slice()).finish() + } +} + impl !Send for Env {} impl !Sync for Env {} @@ -196,16 +225,23 @@ pub fn env() -> Env { } pub fn getenv(k: &OsStr) -> Option<OsString> { - let s = run_with_cstr(k.as_bytes(), |k| unsafe { + // environment variables with a nul byte can't be set, so their value is + // always None as well + run_with_cstr(k.as_bytes(), |k| { let _guard = env_read_lock(); - Ok(libc::getenv(k.as_ptr()) as *const libc::c_char) + let v = unsafe { libc::getenv(k.as_ptr()) } as *const libc::c_char; + + if v.is_null() { + Ok(None) + } else { + // SAFETY: `v` cannot be mutated while executing this line since we've a read lock + let bytes = unsafe { CStr::from_ptr(v) }.to_bytes().to_vec(); + + Ok(Some(OsStringExt::from_vec(bytes))) + } }) - .ok()?; - if s.is_null() { - None - } else { - Some(OsStringExt::from_vec(unsafe { CStr::from_ptr(s) }.to_bytes().to_vec())) - } + .ok() + .flatten() } pub fn setenv(k: &OsStr, v: &OsStr) -> io::Result<()> { @@ -224,6 +260,11 @@ pub fn unsetenv(n: &OsStr) -> io::Result<()> { }) } +#[allow(dead_code)] +pub fn page_size() -> usize { + unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize } +} + pub fn temp_dir() -> PathBuf { panic!("no filesystem on wasm") } diff --git a/library/std/src/sys/wasi/thread.rs b/library/std/src/sys/wasi/thread.rs index e7a6ab4be..a0eefa881 100644 --- a/library/std/src/sys/wasi/thread.rs +++ b/library/std/src/sys/wasi/thread.rs @@ -1,5 +1,3 @@ -#![deny(unsafe_op_in_unsafe_fn)] - use crate::ffi::CStr; use crate::io; use crate::mem; @@ -7,14 +5,124 @@ use crate::num::NonZeroUsize; use crate::sys::unsupported; use crate::time::Duration; -pub struct Thread(!); +cfg_if::cfg_if! { + if #[cfg(target_feature = "atomics")] { + use crate::cmp; + use crate::ptr; + use crate::sys::os; + // Add a few symbols not in upstream `libc` just yet. + mod libc { + pub use crate::ffi; + pub use crate::mem; + pub use libc::*; + + // defined in wasi-libc + // https://github.com/WebAssembly/wasi-libc/blob/a6f871343313220b76009827ed0153586361c0d5/libc-top-half/musl/include/alltypes.h.in#L108 + #[repr(C)] + union pthread_attr_union { + __i: [ffi::c_int; if mem::size_of::<ffi::c_long>() == 8 { 14 } else { 9 }], + __vi: [ffi::c_int; if mem::size_of::<ffi::c_long>() == 8 { 14 } else { 9 }], + __s: [ffi::c_ulong; if mem::size_of::<ffi::c_long>() == 8 { 7 } else { 9 }], + } + + #[repr(C)] + pub struct pthread_attr_t { + __u: pthread_attr_union, + } + + #[allow(non_camel_case_types)] + pub type pthread_t = *mut ffi::c_void; + + extern "C" { + pub fn pthread_create( + native: *mut pthread_t, + attr: *const pthread_attr_t, + f: extern "C" fn(*mut ffi::c_void) -> *mut ffi::c_void, + value: *mut ffi::c_void, + ) -> ffi::c_int; + pub fn pthread_join(native: pthread_t, value: *mut *mut ffi::c_void) -> ffi::c_int; + pub fn pthread_attr_init(attrp: *mut pthread_attr_t) -> ffi::c_int; + pub fn pthread_attr_setstacksize( + attr: *mut pthread_attr_t, + stack_size: libc::size_t, + ) -> ffi::c_int; + pub fn pthread_attr_destroy(attr: *mut pthread_attr_t) -> ffi::c_int; + pub fn pthread_detach(thread: pthread_t) -> ffi::c_int; + } + } + + pub struct Thread { + id: libc::pthread_t, + } + + impl Drop for Thread { + fn drop(&mut self) { + let ret = unsafe { libc::pthread_detach(self.id) }; + debug_assert_eq!(ret, 0); + } + } + } else { + pub struct Thread(!); + } +} pub const DEFAULT_MIN_STACK_SIZE: usize = 4096; impl Thread { // unsafe: see thread::Builder::spawn_unchecked for safety requirements - pub unsafe fn new(_stack: usize, _p: Box<dyn FnOnce()>) -> io::Result<Thread> { - unsupported() + cfg_if::cfg_if! { + if #[cfg(target_feature = "atomics")] { + pub unsafe fn new(stack: usize, p: Box<dyn FnOnce()>) -> io::Result<Thread> { + let p = Box::into_raw(Box::new(p)); + let mut native: libc::pthread_t = mem::zeroed(); + let mut attr: libc::pthread_attr_t = mem::zeroed(); + assert_eq!(libc::pthread_attr_init(&mut attr), 0); + + let stack_size = cmp::max(stack, DEFAULT_MIN_STACK_SIZE); + + match libc::pthread_attr_setstacksize(&mut attr, stack_size) { + 0 => {} + n => { + assert_eq!(n, libc::EINVAL); + // EINVAL means |stack_size| is either too small or not a + // multiple of the system page size. Because it's definitely + // >= PTHREAD_STACK_MIN, it must be an alignment issue. + // Round up to the nearest page and try again. + let page_size = os::page_size(); + let stack_size = + (stack_size + page_size - 1) & (-(page_size as isize - 1) as usize - 1); + assert_eq!(libc::pthread_attr_setstacksize(&mut attr, stack_size), 0); + } + }; + + let ret = libc::pthread_create(&mut native, &attr, thread_start, p as *mut _); + // Note: if the thread creation fails and this assert fails, then p will + // be leaked. However, an alternative design could cause double-free + // which is clearly worse. + assert_eq!(libc::pthread_attr_destroy(&mut attr), 0); + + return if ret != 0 { + // The thread failed to start and as a result p was not consumed. Therefore, it is + // safe to reconstruct the box so that it gets deallocated. + drop(Box::from_raw(p)); + Err(io::Error::from_raw_os_error(ret)) + } else { + Ok(Thread { id: native }) + }; + + extern "C" fn thread_start(main: *mut libc::c_void) -> *mut libc::c_void { + unsafe { + // Finally, let's run some code. + Box::from_raw(main as *mut Box<dyn FnOnce()>)(); + } + ptr::null_mut() + } + } + } else { + pub unsafe fn new(_stack: usize, _p: Box<dyn FnOnce()>) -> io::Result<Thread> { + unsupported() + } + } } pub fn yield_now() { @@ -62,7 +170,19 @@ impl Thread { } pub fn join(self) { - self.0 + cfg_if::cfg_if! { + if #[cfg(target_feature = "atomics")] { + unsafe { + let ret = libc::pthread_join(self.id, ptr::null_mut()); + mem::forget(self); + if ret != 0 { + rtabort!("failed to join thread: {}", io::Error::from_raw_os_error(ret)); + } + } + } else { + self.0 + } + } } } diff --git a/library/std/src/sys/windows/cmath.rs b/library/std/src/sys/windows/cmath.rs index 43ab8c7ee..1b2a86f3c 100644 --- a/library/std/src/sys/windows/cmath.rs +++ b/library/std/src/sys/windows/cmath.rs @@ -1,6 +1,6 @@ #![cfg(not(test))] -use libc::{c_double, c_float}; +use libc::{c_double, c_float, c_int}; extern "C" { pub fn acos(n: c_double) -> c_double; @@ -23,6 +23,10 @@ extern "C" { pub fn sinh(n: c_double) -> c_double; pub fn tan(n: c_double) -> c_double; pub fn tanh(n: c_double) -> c_double; + pub fn tgamma(n: c_double) -> c_double; + pub fn tgammaf(n: c_float) -> c_float; + pub fn lgamma_r(n: c_double, s: &mut c_int) -> c_double; + pub fn lgammaf_r(n: c_float, s: &mut c_int) -> c_float; } pub use self::shims::*; diff --git a/library/std/src/sys/windows/compat.rs b/library/std/src/sys/windows/compat.rs index 4fe95d411..e28dd4935 100644 --- a/library/std/src/sys/windows/compat.rs +++ b/library/std/src/sys/windows/compat.rs @@ -69,10 +69,7 @@ unsafe extern "C" fn init() { /// Helper macro for creating CStrs from literals and symbol names. macro_rules! ansi_str { - (sym $ident:ident) => {{ - #[allow(unused_unsafe)] - crate::sys::compat::const_cstr_from_bytes(concat!(stringify!($ident), "\0").as_bytes()) - }}; + (sym $ident:ident) => {{ crate::sys::compat::const_cstr_from_bytes(concat!(stringify!($ident), "\0").as_bytes()) }}; ($lit:literal) => {{ crate::sys::compat::const_cstr_from_bytes(concat!($lit, "\0").as_bytes()) }}; } diff --git a/library/std/src/sys/windows/os.rs b/library/std/src/sys/windows/os.rs index d7adeb266..58afca088 100644 --- a/library/std/src/sys/windows/os.rs +++ b/library/std/src/sys/windows/os.rs @@ -25,10 +25,6 @@ pub fn errno() -> i32 { /// Gets a detailed string description for the given error number. pub fn error_string(mut errnum: i32) -> String { - // This value is calculated from the macro - // MAKELANGID(LANG_SYSTEM_DEFAULT, SUBLANG_SYS_DEFAULT) - let langId = 0x0800 as c::DWORD; - let mut buf = [0 as c::WCHAR; 2048]; unsafe { @@ -56,13 +52,13 @@ pub fn error_string(mut errnum: i32) -> String { flags | c::FORMAT_MESSAGE_FROM_SYSTEM | c::FORMAT_MESSAGE_IGNORE_INSERTS, module, errnum as c::DWORD, - langId, + 0, buf.as_mut_ptr(), buf.len() as c::DWORD, ptr::null(), ) as usize; if res == 0 { - // Sometimes FormatMessageW can fail e.g., system doesn't like langId, + // Sometimes FormatMessageW can fail e.g., system doesn't like 0 as langId, let fm_err = errno(); return format!("OS Error {errnum} (FormatMessageW() returned error {fm_err})"); } @@ -85,25 +81,69 @@ pub fn error_string(mut errnum: i32) -> String { pub struct Env { base: c::LPWCH, - cur: c::LPWCH, + iter: EnvIterator, +} + +// FIXME(https://github.com/rust-lang/rust/issues/114583): Remove this when <OsStr as Debug>::fmt matches <str as Debug>::fmt. +pub struct EnvStrDebug<'a> { + iter: &'a EnvIterator, +} + +impl fmt::Debug for EnvStrDebug<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { iter } = self; + let iter: EnvIterator = (*iter).clone(); + let mut list = f.debug_list(); + for (a, b) in iter { + list.entry(&(a.to_str().unwrap(), b.to_str().unwrap())); + } + list.finish() + } +} + +impl Env { + pub fn str_debug(&self) -> impl fmt::Debug + '_ { + let Self { base: _, iter } = self; + EnvStrDebug { iter } + } +} + +impl fmt::Debug for Env { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { base: _, iter } = self; + f.debug_list().entries(iter.clone()).finish() + } } impl Iterator for Env { type Item = (OsString, OsString); fn next(&mut self) -> Option<(OsString, OsString)> { + let Self { base: _, iter } = self; + iter.next() + } +} + +#[derive(Clone)] +struct EnvIterator(c::LPWCH); + +impl Iterator for EnvIterator { + type Item = (OsString, OsString); + + fn next(&mut self) -> Option<(OsString, OsString)> { + let Self(cur) = self; loop { unsafe { - if *self.cur == 0 { + if **cur == 0 { return None; } - let p = self.cur as *const u16; + let p = *cur as *const u16; let mut len = 0; while *p.add(len) != 0 { len += 1; } let s = slice::from_raw_parts(p, len); - self.cur = self.cur.add(len + 1); + *cur = cur.add(len + 1); // Windows allows environment variables to start with an equals // symbol (in any other position, this is the separator between @@ -137,7 +177,7 @@ pub fn env() -> Env { if ch.is_null() { panic!("failure getting env string from OS: {}", io::Error::last_os_error()); } - Env { base: ch, cur: ch } + Env { base: ch, iter: EnvIterator(ch) } } } diff --git a/library/std/src/sys/windows/os_str.rs b/library/std/src/sys/windows/os_str.rs index 16c4f55c6..4708657a9 100644 --- a/library/std/src/sys/windows/os_str.rs +++ b/library/std/src/sys/windows/os_str.rs @@ -63,6 +63,16 @@ impl fmt::Display for Slice { } impl Buf { + #[inline] + pub fn into_os_str_bytes(self) -> Vec<u8> { + self.inner.into_bytes() + } + + #[inline] + pub unsafe fn from_os_str_bytes_unchecked(s: Vec<u8>) -> Self { + Self { inner: Wtf8Buf::from_bytes_unchecked(s) } + } + pub fn with_capacity(capacity: usize) -> Buf { Buf { inner: Wtf8Buf::with_capacity(capacity) } } diff --git a/library/std/src/sys/windows/process.rs b/library/std/src/sys/windows/process.rs index e3493cbb8..2dd0c67ac 100644 --- a/library/std/src/sys/windows/process.rs +++ b/library/std/src/sys/windows/process.rs @@ -652,7 +652,7 @@ impl Process { } } -#[derive(PartialEq, Eq, Clone, Copy, Debug)] +#[derive(PartialEq, Eq, Clone, Copy, Debug, Default)] pub struct ExitStatus(c::DWORD); impl ExitStatus { diff --git a/library/std/src/sys/windows/thread_local_dtor.rs b/library/std/src/sys/windows/thread_local_dtor.rs index 9707a95df..cf542d2bf 100644 --- a/library/std/src/sys/windows/thread_local_dtor.rs +++ b/library/std/src/sys/windows/thread_local_dtor.rs @@ -4,29 +4,4 @@ #![unstable(feature = "thread_local_internals", issue = "none")] #![cfg(target_thread_local)] -// Using a per-thread list avoids the problems in synchronizing global state. -#[thread_local] -static mut DESTRUCTORS: Vec<(*mut u8, unsafe extern "C" fn(*mut u8))> = Vec::new(); - -// Ensure this can never be inlined because otherwise this may break in dylibs. -// See #44391. -#[inline(never)] -pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - DESTRUCTORS.push((t, dtor)); -} - -#[inline(never)] // See comment above -/// Runs destructors. This should not be called until thread exit. -pub unsafe fn run_keyless_dtors() { - // Drop all the destructors. - // - // Note: While this is potentially an infinite loop, it *should* be - // the case that this loop always terminates because we provide the - // guarantee that a TLS key cannot be set after it is flagged for - // destruction. - while let Some((ptr, dtor)) = DESTRUCTORS.pop() { - (dtor)(ptr); - } - // We're done so free the memory. - DESTRUCTORS = Vec::new(); -} +pub use super::thread_local_key::register_keyless_dtor as register_dtor; diff --git a/library/std/src/sys/windows/thread_local_key.rs b/library/std/src/sys/windows/thread_local_key.rs index 17628b757..036d96596 100644 --- a/library/std/src/sys/windows/thread_local_key.rs +++ b/library/std/src/sys/windows/thread_local_key.rs @@ -1,7 +1,7 @@ use crate::cell::UnsafeCell; use crate::ptr; use crate::sync::atomic::{ - AtomicPtr, AtomicU32, + AtomicBool, AtomicPtr, AtomicU32, Ordering::{AcqRel, Acquire, Relaxed, Release}, }; use crate::sys::c; @@ -9,6 +9,41 @@ use crate::sys::c; #[cfg(test)] mod tests; +/// An optimization hint. The compiler is often smart enough to know if an atomic +/// is never set and can remove dead code based on that fact. +static HAS_DTORS: AtomicBool = AtomicBool::new(false); + +// Using a per-thread list avoids the problems in synchronizing global state. +#[thread_local] +#[cfg(target_thread_local)] +static mut DESTRUCTORS: Vec<(*mut u8, unsafe extern "C" fn(*mut u8))> = Vec::new(); + +// Ensure this can never be inlined because otherwise this may break in dylibs. +// See #44391. +#[inline(never)] +#[cfg(target_thread_local)] +pub unsafe fn register_keyless_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { + DESTRUCTORS.push((t, dtor)); + HAS_DTORS.store(true, Relaxed); +} + +#[inline(never)] // See comment above +#[cfg(target_thread_local)] +/// Runs destructors. This should not be called until thread exit. +unsafe fn run_keyless_dtors() { + // Drop all the destructors. + // + // Note: While this is potentially an infinite loop, it *should* be + // the case that this loop always terminates because we provide the + // guarantee that a TLS key cannot be set after it is flagged for + // destruction. + while let Some((ptr, dtor)) = DESTRUCTORS.pop() { + (dtor)(ptr); + } + // We're done so free the memory. + DESTRUCTORS = Vec::new(); +} + type Key = c::DWORD; type Dtor = unsafe extern "C" fn(*mut u8); @@ -156,6 +191,8 @@ static DTORS: AtomicPtr<StaticKey> = AtomicPtr::new(ptr::null_mut()); /// Should only be called once per key, otherwise loops or breaks may occur in /// the linked list. unsafe fn register_dtor(key: &'static StaticKey) { + // Ensure this is never run when native thread locals are available. + assert_eq!(false, cfg!(target_thread_local)); let this = <*const StaticKey>::cast_mut(key); // Use acquire ordering to pass along the changes done by the previously // registered keys when we store the new head with release ordering. @@ -167,6 +204,7 @@ unsafe fn register_dtor(key: &'static StaticKey) { Err(new) => head = new, } } + HAS_DTORS.store(true, Release); } // ------------------------------------------------------------------------- @@ -240,10 +278,14 @@ pub static p_thread_callback: unsafe extern "system" fn(c::LPVOID, c::DWORD, c:: #[allow(dead_code, unused_variables)] unsafe extern "system" fn on_tls_callback(h: c::LPVOID, dwReason: c::DWORD, pv: c::LPVOID) { + if !HAS_DTORS.load(Acquire) { + return; + } if dwReason == c::DLL_THREAD_DETACH || dwReason == c::DLL_PROCESS_DETACH { + #[cfg(not(target_thread_local))] run_dtors(); #[cfg(target_thread_local)] - super::thread_local_dtor::run_keyless_dtors(); + run_keyless_dtors(); } // See comments above for what this is doing. Note that we don't need this diff --git a/library/std/src/sys/windows/thread_local_key/tests.rs b/library/std/src/sys/windows/thread_local_key/tests.rs index c95f383fb..c739f0caf 100644 --- a/library/std/src/sys/windows/thread_local_key/tests.rs +++ b/library/std/src/sys/windows/thread_local_key/tests.rs @@ -1,3 +1,7 @@ +// This file only tests the thread local key fallback. +// Windows targets with native thread local support do not use this. +#![cfg(not(target_thread_local))] + use super::StaticKey; use crate::ptr; |