extern crate libc; use std::fs::File; use std::mem::ManuallyDrop; use std::os::unix::io::{FromRawFd, RawFd}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::{io, ptr}; use crate::advice::Advice; #[cfg(any( all(target_os = "linux", not(target_arch = "mips")), target_os = "freebsd", target_os = "android" ))] const MAP_STACK: libc::c_int = libc::MAP_STACK; #[cfg(not(any( all(target_os = "linux", not(target_arch = "mips")), target_os = "freebsd", target_os = "android" )))] const MAP_STACK: libc::c_int = 0; #[cfg(any(target_os = "linux", target_os = "android"))] const MAP_POPULATE: libc::c_int = libc::MAP_POPULATE; #[cfg(not(any(target_os = "linux", target_os = "android")))] const MAP_POPULATE: libc::c_int = 0; #[cfg(any( target_os = "android", all(target_os = "linux", not(target_env = "musl")) ))] use libc::{mmap64 as mmap, off64_t as off_t}; #[cfg(not(any( target_os = "android", all(target_os = "linux", not(target_env = "musl")) )))] use libc::{mmap, off_t}; pub struct MmapInner { ptr: *mut libc::c_void, len: usize, } impl MmapInner { /// Creates a new `MmapInner`. /// /// This is a thin wrapper around the `mmap` sytem call. fn new( len: usize, prot: libc::c_int, flags: libc::c_int, file: RawFd, offset: u64, ) -> io::Result { let alignment = offset % page_size() as u64; let aligned_offset = offset - alignment; let (map_len, map_offset) = Self::adjust_mmap_params(len as usize, alignment as usize)?; unsafe { let ptr = mmap( ptr::null_mut(), map_len as libc::size_t, prot, flags, file, aligned_offset as off_t, ); if ptr == libc::MAP_FAILED { Err(io::Error::last_os_error()) } else { Ok(Self::from_raw_parts(ptr, len, map_offset)) } } } fn adjust_mmap_params(len: usize, alignment: usize) -> io::Result<(usize, usize)> { use std::isize; // Rust's slice cannot be larger than isize::MAX. // See https://doc.rust-lang.org/std/slice/fn.from_raw_parts.html // // This is not a problem on 64-bit targets, but on 32-bit one // having a file or an anonymous mapping larger than 2GB is quite normal // and we have to prevent it. // // The code below is essentially the same as in Rust's std: // https://github.com/rust-lang/rust/blob/db78ab70a88a0a5e89031d7ee4eccec835dcdbde/library/alloc/src/raw_vec.rs#L495 if std::mem::size_of::() < 8 && len > isize::MAX as usize { return Err(io::Error::new( io::ErrorKind::InvalidData, "memory map length overflows isize", )); } let map_len = len + alignment; let map_offset = alignment; // `libc::mmap` does not support zero-size mappings. POSIX defines: // // https://pubs.opengroup.org/onlinepubs/9699919799/functions/mmap.html // > If `len` is zero, `mmap()` shall fail and no mapping shall be established. // // So if we would create such a mapping, crate a one-byte mapping instead: let map_len = map_len.max(1); // Note that in that case `MmapInner::len` is still set to zero, // and `Mmap` will still dereferences to an empty slice. // // If this mapping is backed by an empty file, we create a mapping larger than the file. // This is unusual but well-defined. On the same man page, POSIX further defines: // // > The `mmap()` function can be used to map a region of memory that is larger // > than the current size of the object. // // (The object here is the file.) // // > Memory access within the mapping but beyond the current end of the underlying // > objects may result in SIGBUS signals being sent to the process. The reason for this // > is that the size of the object can be manipulated by other processes and can change // > at any moment. The implementation should tell the application that a memory reference // > is outside the object where this can be detected; otherwise, written data may be lost // > and read data may not reflect actual data in the object. // // Because `MmapInner::len` is not incremented, this increment of `aligned_len` // will not allow accesses past the end of the file and will not cause SIGBUS. // // (SIGBUS is still possible by mapping a non-empty file and then truncating it // to a shorter size, but that is unrelated to this handling of empty files.) Ok((map_len, map_offset)) } /// Get the current memory mapping as a `(ptr, map_len, offset)` tuple. /// /// Note that `map_len` is the length of the memory mapping itself and /// _not_ the one that would be passed to `from_raw_parts`. fn as_mmap_params(&self) -> (*mut libc::c_void, usize, usize) { let offset = self.ptr as usize % page_size(); let len = self.len + offset; // There are two possible memory layouts we could have, depending on // the length and offset passed when constructing this instance: // // 1. The "normal" memory layout looks like this: // // |<------------------>|<---------------------->| // mmap ptr offset ptr public slice // // That is, we have // - The start of the page-aligned memory mapping returned by mmap, // followed by, // - Some number of bytes that are memory mapped but ignored since // they are before the byte offset requested by the user, followed // by, // - The actual memory mapped slice requested by the user. // // This maps cleanly to a (ptr, len, offset) tuple. // // 2. Then, we have the case where the user requested a zero-length // memory mapping. mmap(2) does not support zero-length mappings so // this crate works around that by actually making a mapping of // length one. This means that we have // - A length zero slice, followed by, // - A single memory mapped byte // // Note that this only happens if the offset within the page is also // zero. Otherwise, we have a memory map of offset bytes and not a // zero-length memory map. // // This doesn't fit cleanly into a (ptr, len, offset) tuple. Instead, // we fudge it slightly: a zero-length memory map turns into a // mapping of length one and can't be told apart outside of this // method without knowing the original length. if len == 0 { (self.ptr, 1, 0) } else { (unsafe { self.ptr.offset(-(offset as isize)) }, len, offset) } } /// Construct this `MmapInner` from its raw components /// /// # Safety /// /// - `ptr` must point to the start of memory mapping that can be freed /// using `munmap(2)` (i.e. returned by `mmap(2)` or `mremap(2)`) /// - The memory mapping at `ptr` must have a length of `len + offset`. /// - If `len + offset == 0` then the memory mapping must be of length 1. /// - `offset` must be less than the current page size. unsafe fn from_raw_parts(ptr: *mut libc::c_void, len: usize, offset: usize) -> Self { debug_assert_eq!(ptr as usize % page_size(), 0, "ptr not page-aligned"); debug_assert!(offset < page_size(), "offset larger than page size"); Self { ptr: ptr.offset(offset as isize), len, } } pub fn map(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result { let populate = if populate { MAP_POPULATE } else { 0 }; MmapInner::new( len, libc::PROT_READ, libc::MAP_SHARED | populate, file, offset, ) } pub fn map_exec(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result { let populate = if populate { MAP_POPULATE } else { 0 }; MmapInner::new( len, libc::PROT_READ | libc::PROT_EXEC, libc::MAP_SHARED | populate, file, offset, ) } pub fn map_mut(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result { let populate = if populate { MAP_POPULATE } else { 0 }; MmapInner::new( len, libc::PROT_READ | libc::PROT_WRITE, libc::MAP_SHARED | populate, file, offset, ) } pub fn map_copy(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result { let populate = if populate { MAP_POPULATE } else { 0 }; MmapInner::new( len, libc::PROT_READ | libc::PROT_WRITE, libc::MAP_PRIVATE | populate, file, offset, ) } pub fn map_copy_read_only( len: usize, file: RawFd, offset: u64, populate: bool, ) -> io::Result { let populate = if populate { MAP_POPULATE } else { 0 }; MmapInner::new( len, libc::PROT_READ, libc::MAP_PRIVATE | populate, file, offset, ) } /// Open an anonymous memory map. pub fn map_anon(len: usize, stack: bool, populate: bool) -> io::Result { let stack = if stack { MAP_STACK } else { 0 }; let populate = if populate { MAP_POPULATE } else { 0 }; MmapInner::new( len, libc::PROT_READ | libc::PROT_WRITE, libc::MAP_PRIVATE | libc::MAP_ANON | stack | populate, -1, 0, ) } pub fn flush(&self, offset: usize, len: usize) -> io::Result<()> { let alignment = (self.ptr as usize + offset) % page_size(); let offset = offset as isize - alignment as isize; let len = len + alignment; let result = unsafe { libc::msync(self.ptr.offset(offset), len as libc::size_t, libc::MS_SYNC) }; if result == 0 { Ok(()) } else { Err(io::Error::last_os_error()) } } pub fn flush_async(&self, offset: usize, len: usize) -> io::Result<()> { let alignment = (self.ptr as usize + offset) % page_size(); let offset = offset as isize - alignment as isize; let len = len + alignment; let result = unsafe { libc::msync(self.ptr.offset(offset), len as libc::size_t, libc::MS_ASYNC) }; if result == 0 { Ok(()) } else { Err(io::Error::last_os_error()) } } fn mprotect(&mut self, prot: libc::c_int) -> io::Result<()> { unsafe { let alignment = self.ptr as usize % page_size(); let ptr = self.ptr.offset(-(alignment as isize)); let len = self.len + alignment; let len = len.max(1); if libc::mprotect(ptr, len, prot) == 0 { Ok(()) } else { Err(io::Error::last_os_error()) } } } pub fn make_read_only(&mut self) -> io::Result<()> { self.mprotect(libc::PROT_READ) } pub fn make_exec(&mut self) -> io::Result<()> { self.mprotect(libc::PROT_READ | libc::PROT_EXEC) } pub fn make_mut(&mut self) -> io::Result<()> { self.mprotect(libc::PROT_READ | libc::PROT_WRITE) } #[inline] pub fn ptr(&self) -> *const u8 { self.ptr as *const u8 } #[inline] pub fn mut_ptr(&mut self) -> *mut u8 { self.ptr as *mut u8 } #[inline] pub fn len(&self) -> usize { self.len } pub fn advise(&self, advice: Advice, offset: usize, len: usize) -> io::Result<()> { let alignment = (self.ptr as usize + offset) % page_size(); let offset = offset as isize - alignment as isize; let len = len + alignment; unsafe { if libc::madvise(self.ptr.offset(offset), len, advice as i32) != 0 { Err(io::Error::last_os_error()) } else { Ok(()) } } } #[cfg(target_os = "linux")] pub fn remap(&mut self, new_len: usize, options: crate::RemapOptions) -> io::Result<()> { let (old_ptr, old_len, offset) = self.as_mmap_params(); let (map_len, offset) = Self::adjust_mmap_params(new_len, offset)?; unsafe { let new_ptr = libc::mremap(old_ptr, old_len, map_len, options.into_flags()); if new_ptr == libc::MAP_FAILED { Err(io::Error::last_os_error()) } else { // We explicitly don't drop self since the pointer within is no longer valid. ptr::write(self, Self::from_raw_parts(new_ptr, new_len, offset)); Ok(()) } } } pub fn lock(&self) -> io::Result<()> { unsafe { if libc::mlock(self.ptr, self.len) != 0 { Err(io::Error::last_os_error()) } else { Ok(()) } } } pub fn unlock(&self) -> io::Result<()> { unsafe { if libc::munlock(self.ptr, self.len) != 0 { Err(io::Error::last_os_error()) } else { Ok(()) } } } } impl Drop for MmapInner { fn drop(&mut self) { let (ptr, len, _) = self.as_mmap_params(); // Any errors during unmapping/closing are ignored as the only way // to report them would be through panicking which is highly discouraged // in Drop impls, c.f. https://github.com/rust-lang/lang-team/issues/97 unsafe { libc::munmap(ptr, len as libc::size_t) }; } } unsafe impl Sync for MmapInner {} unsafe impl Send for MmapInner {} fn page_size() -> usize { static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0); match PAGE_SIZE.load(Ordering::Relaxed) { 0 => { let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize }; PAGE_SIZE.store(page_size, Ordering::Relaxed); page_size } page_size => page_size, } } pub fn file_len(file: RawFd) -> io::Result { // SAFETY: We must not close the passed-in fd by dropping the File we create, // we ensure this by immediately wrapping it in a ManuallyDrop. unsafe { let file = ManuallyDrop::new(File::from_raw_fd(file)); Ok(file.metadata()?.len()) } }