diff options
Diffstat (limited to 'third_party/rust/wgpu-core/src/device')
| -rw-r--r-- | third_party/rust/wgpu-core/src/device/any_device.rs | 102 | ||||
| -rw-r--r-- | third_party/rust/wgpu-core/src/device/bgl.rs | 129 | ||||
| -rw-r--r-- | third_party/rust/wgpu-core/src/device/global.rs | 2559 | ||||
| -rw-r--r-- | third_party/rust/wgpu-core/src/device/life.rs | 831 | ||||
| -rw-r--r-- | third_party/rust/wgpu-core/src/device/mod.rs | 480 | ||||
| -rw-r--r-- | third_party/rust/wgpu-core/src/device/queue.rs | 1569 | ||||
| -rw-r--r-- | third_party/rust/wgpu-core/src/device/resource.rs | 3530 | ||||
| -rw-r--r-- | third_party/rust/wgpu-core/src/device/trace.rs | 235 |
8 files changed, 9435 insertions, 0 deletions
diff --git a/third_party/rust/wgpu-core/src/device/any_device.rs b/third_party/rust/wgpu-core/src/device/any_device.rs new file mode 100644 index 0000000000..693155a753 --- /dev/null +++ b/third_party/rust/wgpu-core/src/device/any_device.rs @@ -0,0 +1,102 @@ +use wgt::Backend; + +use super::Device; +/// The `AnyDevice` type: a pointer to a `Device<A>` for any backend `A`. +use crate::hal_api::HalApi; + +use std::fmt; +use std::mem::ManuallyDrop; +use std::ptr::NonNull; +use std::sync::Arc; + +struct AnyDeviceVtable { + // We oppurtunistically store the backend here, since we now it will be used + // with backend selection and it can be stored in static memory. + backend: Backend, + // Drop glue which knows how to drop the stored data. + drop: unsafe fn(*mut ()), +} + +/// A pointer to a `Device<A>`, for any backend `A`. +/// +/// Any `AnyDevice` is just like an `Arc<Device<A>>`, except that the `A` type +/// parameter is erased. To access the `Device`, you must downcast to a +/// particular backend with the \[`downcast_ref`\] or \[`downcast_clone`\] +/// methods. +pub struct AnyDevice { + data: NonNull<()>, + vtable: &'static AnyDeviceVtable, +} + +impl AnyDevice { + /// Return an `AnyDevice` that holds an owning `Arc` pointer to `device`. + pub fn new<A: HalApi>(device: Arc<Device<A>>) -> AnyDevice { + unsafe fn drop_glue<A: HalApi>(ptr: *mut ()) { + // Drop the arc this instance is holding. + unsafe { + _ = Arc::from_raw(ptr.cast::<A::Surface>()); + } + } + + // SAFETY: The pointer returned by Arc::into_raw is guaranteed to be + // non-null. + let data = unsafe { NonNull::new_unchecked(Arc::into_raw(device).cast_mut()) }; + + AnyDevice { + data: data.cast(), + vtable: &AnyDeviceVtable { + backend: A::VARIANT, + drop: drop_glue::<A>, + }, + } + } + + /// If `self` is an `Arc<Device<A>>`, return a reference to the + /// device. + pub fn downcast_ref<A: HalApi>(&self) -> Option<&Device<A>> { + if self.vtable.backend != A::VARIANT { + return None; + } + + // SAFETY: We just checked the instance above implicitly by the backend + // that it was statically constructed through. + Some(unsafe { &*(self.data.as_ptr().cast::<Device<A>>()) }) + } + + /// If `self` is an `Arc<Device<A>>`, return a clone of that. + pub fn downcast_clone<A: HalApi>(&self) -> Option<Arc<Device<A>>> { + if self.vtable.backend != A::VARIANT { + return None; + } + + // We need to prevent the destructor of the arc from running, since it + // refers to the instance held by this object. Dropping it would + // invalidate this object. + // + // SAFETY: We just checked the instance above implicitly by the backend + // that it was statically constructed through. + let this = + ManuallyDrop::new(unsafe { Arc::from_raw(self.data.as_ptr().cast::<Device<A>>()) }); + + // Cloning it increases the reference count, and we return a new arc + // instance. + Some((*this).clone()) + } +} + +impl Drop for AnyDevice { + fn drop(&mut self) { + unsafe { (self.vtable.drop)(self.data.as_ptr()) } + } +} + +impl fmt::Debug for AnyDevice { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "AnyDevice<{}>", self.vtable.backend) + } +} + +#[cfg(send_sync)] +unsafe impl Send for AnyDevice {} +#[cfg(send_sync)] +unsafe impl Sync for AnyDevice {} diff --git a/third_party/rust/wgpu-core/src/device/bgl.rs b/third_party/rust/wgpu-core/src/device/bgl.rs new file mode 100644 index 0000000000..d606f049a3 --- /dev/null +++ b/third_party/rust/wgpu-core/src/device/bgl.rs @@ -0,0 +1,129 @@ +use std::hash::{Hash, Hasher}; + +use crate::{ + binding_model::{self}, + FastIndexMap, +}; + +/// Where a given BGL came from. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum Origin { + /// The bind group layout was created by the user and is present in the BGL resource pool. + Pool, + /// The bind group layout was derived and is not present in the BGL resource pool. + Derived, +} + +/// A HashMap-like structure that stores a BindGroupLayouts [`wgt::BindGroupLayoutEntry`]s. +/// +/// It is hashable, so bind group layouts can be deduplicated. +#[derive(Debug, Default, Clone, Eq)] +pub struct EntryMap { + /// We use a IndexMap here so that we can sort the entries by their binding index, + /// guaranteeing that the hash of equivalent layouts will be the same. + inner: FastIndexMap<u32, wgt::BindGroupLayoutEntry>, + /// We keep track of whether the map is sorted or not, so that we can assert that + /// it is sorted, so that PartialEq and Hash will be stable. + /// + /// We only need sorted if it is used in a Hash or PartialEq, so we never need + /// to actively sort it. + sorted: bool, +} + +impl PartialEq for EntryMap { + fn eq(&self, other: &Self) -> bool { + self.assert_sorted(); + other.assert_sorted(); + + self.inner == other.inner + } +} + +impl Hash for EntryMap { + fn hash<H: Hasher>(&self, state: &mut H) { + self.assert_sorted(); + + // We don't need to hash the keys, since they are just extracted from the values. + // + // We know this is stable and will match the behavior of PartialEq as we ensure + // that the array is sorted. + for entry in self.inner.values() { + entry.hash(state); + } + } +} + +impl EntryMap { + fn assert_sorted(&self) { + assert!(self.sorted); + } + + /// Create a new [`BindGroupLayoutEntryMap`] from a slice of [`wgt::BindGroupLayoutEntry`]s. + /// + /// Errors if there are duplicate bindings or if any binding index is greater than + /// the device's limits. + pub fn from_entries( + device_limits: &wgt::Limits, + entries: &[wgt::BindGroupLayoutEntry], + ) -> Result<Self, binding_model::CreateBindGroupLayoutError> { + let mut inner = FastIndexMap::with_capacity_and_hasher(entries.len(), Default::default()); + for entry in entries { + if entry.binding >= device_limits.max_bindings_per_bind_group { + return Err( + binding_model::CreateBindGroupLayoutError::InvalidBindingIndex { + binding: entry.binding, + maximum: device_limits.max_bindings_per_bind_group, + }, + ); + } + if inner.insert(entry.binding, *entry).is_some() { + return Err(binding_model::CreateBindGroupLayoutError::ConflictBinding( + entry.binding, + )); + } + } + inner.sort_unstable_keys(); + + Ok(Self { + inner, + sorted: true, + }) + } + + /// Get the count of [`wgt::BindGroupLayoutEntry`]s in this map. + pub fn len(&self) -> usize { + self.inner.len() + } + + /// Get the [`wgt::BindGroupLayoutEntry`] for the given binding index. + pub fn get(&self, binding: u32) -> Option<&wgt::BindGroupLayoutEntry> { + self.inner.get(&binding) + } + + /// Iterator over all the binding indices in this map. + pub fn indices(&self) -> impl ExactSizeIterator<Item = u32> + '_ { + self.inner.keys().copied() + } + + /// Iterator over all the [`wgt::BindGroupLayoutEntry`]s in this map. + pub fn values(&self) -> impl ExactSizeIterator<Item = &wgt::BindGroupLayoutEntry> + '_ { + self.inner.values() + } + + pub fn iter(&self) -> impl ExactSizeIterator<Item = (&u32, &wgt::BindGroupLayoutEntry)> + '_ { + self.inner.iter() + } + + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub fn contains_key(&self, key: u32) -> bool { + self.inner.contains_key(&key) + } + + pub fn entry(&mut self, key: u32) -> indexmap::map::Entry<'_, u32, wgt::BindGroupLayoutEntry> { + self.sorted = false; + self.inner.entry(key) + } +} diff --git a/third_party/rust/wgpu-core/src/device/global.rs b/third_party/rust/wgpu-core/src/device/global.rs new file mode 100644 index 0000000000..64fd6d4de7 --- /dev/null +++ b/third_party/rust/wgpu-core/src/device/global.rs @@ -0,0 +1,2559 @@ +#[cfg(feature = "trace")] +use crate::device::trace; +use crate::{ + api_log, binding_model, command, conv, + device::{ + bgl, life::WaitIdleError, map_buffer, queue, DeviceError, DeviceLostClosure, + DeviceLostReason, HostMap, IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL, + }, + global::Global, + hal_api::HalApi, + id::{self, AdapterId, DeviceId, QueueId, SurfaceId}, + init_tracker::TextureInitTracker, + instance::{self, Adapter, Surface}, + pipeline, present, + resource::{self, BufferAccessResult}, + resource::{BufferAccessError, BufferMapOperation, CreateBufferError, Resource}, + validation::check_buffer_usage, + Label, LabelHelpers as _, +}; + +use arrayvec::ArrayVec; +use hal::Device as _; +use parking_lot::RwLock; + +use wgt::{BufferAddress, TextureFormat}; + +use std::{ + borrow::Cow, + iter, + ops::Range, + ptr, + sync::{atomic::Ordering, Arc}, +}; + +use super::{ImplicitPipelineIds, InvalidDevice, UserClosures}; + +impl Global { + pub fn adapter_is_surface_supported<A: HalApi>( + &self, + adapter_id: AdapterId, + surface_id: SurfaceId, + ) -> Result<bool, instance::IsSurfaceSupportedError> { + let hub = A::hub(self); + + let surface_guard = self.surfaces.read(); + let adapter_guard = hub.adapters.read(); + let adapter = adapter_guard + .get(adapter_id) + .map_err(|_| instance::IsSurfaceSupportedError::InvalidAdapter)?; + let surface = surface_guard + .get(surface_id) + .map_err(|_| instance::IsSurfaceSupportedError::InvalidSurface)?; + Ok(adapter.is_surface_supported(surface)) + } + + pub fn surface_get_capabilities<A: HalApi>( + &self, + surface_id: SurfaceId, + adapter_id: AdapterId, + ) -> Result<wgt::SurfaceCapabilities, instance::GetSurfaceSupportError> { + profiling::scope!("Surface::get_capabilities"); + self.fetch_adapter_and_surface::<A, _, _>(surface_id, adapter_id, |adapter, surface| { + let mut hal_caps = surface.get_capabilities(adapter)?; + + hal_caps.formats.sort_by_key(|f| !f.is_srgb()); + + let usages = conv::map_texture_usage_from_hal(hal_caps.usage); + + Ok(wgt::SurfaceCapabilities { + formats: hal_caps.formats, + present_modes: hal_caps.present_modes, + alpha_modes: hal_caps.composite_alpha_modes, + usages, + }) + }) + } + + fn fetch_adapter_and_surface< + A: HalApi, + F: FnOnce(&Adapter<A>, &Surface) -> Result<B, instance::GetSurfaceSupportError>, + B, + >( + &self, + surface_id: SurfaceId, + adapter_id: AdapterId, + get_supported_callback: F, + ) -> Result<B, instance::GetSurfaceSupportError> { + let hub = A::hub(self); + + let surface_guard = self.surfaces.read(); + let adapter_guard = hub.adapters.read(); + let adapter = adapter_guard + .get(adapter_id) + .map_err(|_| instance::GetSurfaceSupportError::InvalidAdapter)?; + let surface = surface_guard + .get(surface_id) + .map_err(|_| instance::GetSurfaceSupportError::InvalidSurface)?; + + get_supported_callback(adapter, surface) + } + + pub fn device_features<A: HalApi>( + &self, + device_id: DeviceId, + ) -> Result<wgt::Features, InvalidDevice> { + let hub = A::hub(self); + + let device = hub.devices.get(device_id).map_err(|_| InvalidDevice)?; + if !device.is_valid() { + return Err(InvalidDevice); + } + + Ok(device.features) + } + + pub fn device_limits<A: HalApi>( + &self, + device_id: DeviceId, + ) -> Result<wgt::Limits, InvalidDevice> { + let hub = A::hub(self); + + let device = hub.devices.get(device_id).map_err(|_| InvalidDevice)?; + if !device.is_valid() { + return Err(InvalidDevice); + } + + Ok(device.limits.clone()) + } + + pub fn device_downlevel_properties<A: HalApi>( + &self, + device_id: DeviceId, + ) -> Result<wgt::DownlevelCapabilities, InvalidDevice> { + let hub = A::hub(self); + + let device = hub.devices.get(device_id).map_err(|_| InvalidDevice)?; + if !device.is_valid() { + return Err(InvalidDevice); + } + + Ok(device.downlevel.clone()) + } + + pub fn device_create_buffer<A: HalApi>( + &self, + device_id: DeviceId, + desc: &resource::BufferDescriptor, + id_in: Option<id::BufferId>, + ) -> (id::BufferId, Option<CreateBufferError>) { + profiling::scope!("Device::create_buffer"); + + let hub = A::hub(self); + let fid = hub.buffers.prepare(id_in); + + let mut to_destroy: ArrayVec<resource::Buffer<A>, 2> = ArrayVec::new(); + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => { + break DeviceError::Invalid.into(); + } + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + if desc.usage.is_empty() { + // Per spec, `usage` must not be zero. + break CreateBufferError::InvalidUsage(desc.usage); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + let mut desc = desc.clone(); + let mapped_at_creation = std::mem::replace(&mut desc.mapped_at_creation, false); + if mapped_at_creation && !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) { + desc.usage |= wgt::BufferUsages::COPY_DST; + } + trace.add(trace::Action::CreateBuffer(fid.id(), desc)); + } + + let buffer = match device.create_buffer(desc, false) { + Ok(buffer) => buffer, + Err(e) => { + break e; + } + }; + + let buffer_use = if !desc.mapped_at_creation { + hal::BufferUses::empty() + } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) { + // buffer is mappable, so we are just doing that at start + let map_size = buffer.size; + let ptr = if map_size == 0 { + std::ptr::NonNull::dangling() + } else { + match map_buffer(device.raw(), &buffer, 0, map_size, HostMap::Write) { + Ok(ptr) => ptr, + Err(e) => { + to_destroy.push(buffer); + break e.into(); + } + } + }; + *buffer.map_state.lock() = resource::BufferMapState::Active { + ptr, + range: 0..map_size, + host: HostMap::Write, + }; + hal::BufferUses::MAP_WRITE + } else { + // buffer needs staging area for initialization only + let stage_desc = wgt::BufferDescriptor { + label: Some(Cow::Borrowed( + "(wgpu internal) initializing unmappable buffer", + )), + size: desc.size, + usage: wgt::BufferUsages::MAP_WRITE | wgt::BufferUsages::COPY_SRC, + mapped_at_creation: false, + }; + let stage = match device.create_buffer(&stage_desc, true) { + Ok(stage) => stage, + Err(e) => { + to_destroy.push(buffer); + break e; + } + }; + + let snatch_guard = device.snatchable_lock.read(); + let stage_raw = stage.raw(&snatch_guard).unwrap(); + let mapping = match unsafe { device.raw().map_buffer(stage_raw, 0..stage.size) } { + Ok(mapping) => mapping, + Err(e) => { + to_destroy.push(buffer); + to_destroy.push(stage); + break CreateBufferError::Device(e.into()); + } + }; + + let stage_fid = hub.buffers.request(); + let stage = stage_fid.init(stage); + + assert_eq!(buffer.size % wgt::COPY_BUFFER_ALIGNMENT, 0); + // Zero initialize memory and then mark both staging and buffer as initialized + // (it's guaranteed that this is the case by the time the buffer is usable) + unsafe { ptr::write_bytes(mapping.ptr.as_ptr(), 0, buffer.size as usize) }; + buffer.initialization_status.write().drain(0..buffer.size); + stage.initialization_status.write().drain(0..buffer.size); + + *buffer.map_state.lock() = resource::BufferMapState::Init { + ptr: mapping.ptr, + needs_flush: !mapping.is_coherent, + stage_buffer: stage, + }; + hal::BufferUses::COPY_DST + }; + + let (id, resource) = fid.assign(buffer); + api_log!("Device::create_buffer({desc:?}) -> {id:?}"); + + device + .trackers + .lock() + .buffers + .insert_single(id, resource, buffer_use); + + return (id, None); + }; + + // Error path + + for buffer in to_destroy { + let device = Arc::clone(&buffer.device); + device + .lock_life() + .schedule_resource_destruction(queue::TempResource::Buffer(Arc::new(buffer)), !0); + } + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + /// Assign `id_in` an error with the given `label`. + /// + /// Ensure that future attempts to use `id_in` as a buffer ID will propagate + /// the error, following the WebGPU ["contagious invalidity"] style. + /// + /// Firefox uses this function to comply strictly with the WebGPU spec, + /// which requires [`GPUBufferDescriptor`] validation to be generated on the + /// Device timeline and leave the newly created [`GPUBuffer`] invalid. + /// + /// Ideally, we would simply let [`device_create_buffer`] take care of all + /// of this, but some errors must be detected before we can even construct a + /// [`wgpu_types::BufferDescriptor`] to give it. For example, the WebGPU API + /// allows a `GPUBufferDescriptor`'s [`usage`] property to be any WebIDL + /// `unsigned long` value, but we can't construct a + /// [`wgpu_types::BufferUsages`] value from values with unassigned bits + /// set. This means we must validate `usage` before we can call + /// `device_create_buffer`. + /// + /// When that validation fails, we must arrange for the buffer id to be + /// considered invalid. This method provides the means to do so. + /// + /// ["contagious invalidity"]: https://www.w3.org/TR/webgpu/#invalidity + /// [`GPUBufferDescriptor`]: https://www.w3.org/TR/webgpu/#dictdef-gpubufferdescriptor + /// [`GPUBuffer`]: https://www.w3.org/TR/webgpu/#gpubuffer + /// [`wgpu_types::BufferDescriptor`]: wgt::BufferDescriptor + /// [`device_create_buffer`]: Global::device_create_buffer + /// [`usage`]: https://www.w3.org/TR/webgpu/#dom-gputexturedescriptor-usage + /// [`wgpu_types::BufferUsages`]: wgt::BufferUsages + pub fn create_buffer_error<A: HalApi>(&self, id_in: Option<id::BufferId>, label: Label) { + let hub = A::hub(self); + let fid = hub.buffers.prepare(id_in); + + fid.assign_error(label.borrow_or_default()); + } + + pub fn create_render_bundle_error<A: HalApi>( + &self, + id_in: Option<id::RenderBundleId>, + label: Label, + ) { + let hub = A::hub(self); + let fid = hub.render_bundles.prepare(id_in); + + fid.assign_error(label.borrow_or_default()); + } + + /// Assign `id_in` an error with the given `label`. + /// + /// See `create_buffer_error` for more context and explanation. + pub fn create_texture_error<A: HalApi>(&self, id_in: Option<id::TextureId>, label: Label) { + let hub = A::hub(self); + let fid = hub.textures.prepare(id_in); + + fid.assign_error(label.borrow_or_default()); + } + + #[cfg(feature = "replay")] + pub fn device_wait_for_buffer<A: HalApi>( + &self, + device_id: DeviceId, + buffer_id: id::BufferId, + ) -> Result<(), WaitIdleError> { + let hub = A::hub(self); + + let last_submission = { + let buffer_guard = hub.buffers.write(); + match buffer_guard.get(buffer_id) { + Ok(buffer) => buffer.info.submission_index(), + Err(_) => return Ok(()), + } + }; + + hub.devices + .get(device_id) + .map_err(|_| DeviceError::Invalid)? + .wait_for_submit(last_submission) + } + + #[doc(hidden)] + pub fn device_set_buffer_sub_data<A: HalApi>( + &self, + device_id: DeviceId, + buffer_id: id::BufferId, + offset: BufferAddress, + data: &[u8], + ) -> BufferAccessResult { + profiling::scope!("Device::set_buffer_sub_data"); + + let hub = A::hub(self); + + let device = hub + .devices + .get(device_id) + .map_err(|_| DeviceError::Invalid)?; + let snatch_guard = device.snatchable_lock.read(); + if !device.is_valid() { + return Err(DeviceError::Lost.into()); + } + + let buffer = hub + .buffers + .get(buffer_id) + .map_err(|_| BufferAccessError::Invalid)?; + check_buffer_usage(buffer.usage, wgt::BufferUsages::MAP_WRITE)?; + //assert!(buffer isn't used by the GPU); + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + let data_path = trace.make_binary("bin", data); + trace.add(trace::Action::WriteBuffer { + id: buffer_id, + data: data_path, + range: offset..offset + data.len() as BufferAddress, + queued: false, + }); + } + + let raw_buf = buffer + .raw(&snatch_guard) + .ok_or(BufferAccessError::Destroyed)?; + unsafe { + let mapping = device + .raw() + .map_buffer(raw_buf, offset..offset + data.len() as u64) + .map_err(DeviceError::from)?; + ptr::copy_nonoverlapping(data.as_ptr(), mapping.ptr.as_ptr(), data.len()); + if !mapping.is_coherent { + device + .raw() + .flush_mapped_ranges(raw_buf, iter::once(offset..offset + data.len() as u64)); + } + device + .raw() + .unmap_buffer(raw_buf) + .map_err(DeviceError::from)?; + } + + Ok(()) + } + + #[doc(hidden)] + pub fn device_get_buffer_sub_data<A: HalApi>( + &self, + device_id: DeviceId, + buffer_id: id::BufferId, + offset: BufferAddress, + data: &mut [u8], + ) -> BufferAccessResult { + profiling::scope!("Device::get_buffer_sub_data"); + + let hub = A::hub(self); + + let device = hub + .devices + .get(device_id) + .map_err(|_| DeviceError::Invalid)?; + if !device.is_valid() { + return Err(DeviceError::Lost.into()); + } + + let snatch_guard = device.snatchable_lock.read(); + + let buffer = hub + .buffers + .get(buffer_id) + .map_err(|_| BufferAccessError::Invalid)?; + check_buffer_usage(buffer.usage, wgt::BufferUsages::MAP_READ)?; + //assert!(buffer isn't used by the GPU); + + let raw_buf = buffer + .raw(&snatch_guard) + .ok_or(BufferAccessError::Destroyed)?; + unsafe { + let mapping = device + .raw() + .map_buffer(raw_buf, offset..offset + data.len() as u64) + .map_err(DeviceError::from)?; + if !mapping.is_coherent { + device.raw().invalidate_mapped_ranges( + raw_buf, + iter::once(offset..offset + data.len() as u64), + ); + } + ptr::copy_nonoverlapping(mapping.ptr.as_ptr(), data.as_mut_ptr(), data.len()); + device + .raw() + .unmap_buffer(raw_buf) + .map_err(DeviceError::from)?; + } + + Ok(()) + } + + pub fn buffer_label<A: HalApi>(&self, id: id::BufferId) -> String { + A::hub(self).buffers.label_for_resource(id) + } + + pub fn buffer_destroy<A: HalApi>( + &self, + buffer_id: id::BufferId, + ) -> Result<(), resource::DestroyError> { + profiling::scope!("Buffer::destroy"); + api_log!("Buffer::destroy {buffer_id:?}"); + + let hub = A::hub(self); + + let buffer = hub + .buffers + .get(buffer_id) + .map_err(|_| resource::DestroyError::Invalid)?; + + let _ = buffer.unmap(); + + buffer.destroy() + } + + pub fn buffer_drop<A: HalApi>(&self, buffer_id: id::BufferId, wait: bool) { + profiling::scope!("Buffer::drop"); + api_log!("Buffer::drop {buffer_id:?}"); + + let hub = A::hub(self); + + let buffer = match hub.buffers.unregister(buffer_id) { + Some(buffer) => buffer, + None => { + return; + } + }; + + let _ = buffer.unmap(); + + let last_submit_index = buffer.info.submission_index(); + + let device = buffer.device.clone(); + + if device + .pending_writes + .lock() + .as_ref() + .unwrap() + .dst_buffers + .contains_key(&buffer_id) + { + device.lock_life().future_suspected_buffers.push(buffer); + } else { + device + .lock_life() + .suspected_resources + .buffers + .insert(buffer_id, buffer); + } + + if wait { + match device.wait_for_submit(last_submit_index) { + Ok(()) => (), + Err(e) => log::error!("Failed to wait for buffer {:?}: {}", buffer_id, e), + } + } + } + + pub fn device_create_texture<A: HalApi>( + &self, + device_id: DeviceId, + desc: &resource::TextureDescriptor, + id_in: Option<id::TextureId>, + ) -> (id::TextureId, Option<resource::CreateTextureError>) { + profiling::scope!("Device::create_texture"); + + let hub = A::hub(self); + + let fid = hub.textures.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreateTexture(fid.id(), desc.clone())); + } + + let texture = match device.create_texture(&device.adapter, desc) { + Ok(texture) => texture, + Err(error) => break error, + }; + + let (id, resource) = fid.assign(texture); + api_log!("Device::create_texture({desc:?}) -> {id:?}"); + + device.trackers.lock().textures.insert_single( + id, + resource, + hal::TextureUses::UNINITIALIZED, + ); + + return (id, None); + }; + + log::error!("Device::create_texture error: {error}"); + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + /// # Safety + /// + /// - `hal_texture` must be created from `device_id` corresponding raw handle. + /// - `hal_texture` must be created respecting `desc` + /// - `hal_texture` must be initialized + pub unsafe fn create_texture_from_hal<A: HalApi>( + &self, + hal_texture: A::Texture, + device_id: DeviceId, + desc: &resource::TextureDescriptor, + id_in: Option<id::TextureId>, + ) -> (id::TextureId, Option<resource::CreateTextureError>) { + profiling::scope!("Device::create_texture_from_hal"); + + let hub = A::hub(self); + + let fid = hub.textures.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + // NB: Any change done through the raw texture handle will not be + // recorded in the replay + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreateTexture(fid.id(), desc.clone())); + } + + let format_features = match device + .describe_format_features(&device.adapter, desc.format) + .map_err(|error| resource::CreateTextureError::MissingFeatures(desc.format, error)) + { + Ok(features) => features, + Err(error) => break error, + }; + + let mut texture = device.create_texture_from_hal( + hal_texture, + conv::map_texture_usage(desc.usage, desc.format.into()), + desc, + format_features, + resource::TextureClearMode::None, + ); + if desc.usage.contains(wgt::TextureUsages::COPY_DST) { + texture.hal_usage |= hal::TextureUses::COPY_DST; + } + + texture.initialization_status = + RwLock::new(TextureInitTracker::new(desc.mip_level_count, 0)); + + let (id, resource) = fid.assign(texture); + api_log!("Device::create_texture({desc:?}) -> {id:?}"); + + device.trackers.lock().textures.insert_single( + id, + resource, + hal::TextureUses::UNINITIALIZED, + ); + + return (id, None); + }; + + log::error!("Device::create_texture error: {error}"); + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + /// # Safety + /// + /// - `hal_buffer` must be created from `device_id` corresponding raw handle. + /// - `hal_buffer` must be created respecting `desc` + /// - `hal_buffer` must be initialized + pub unsafe fn create_buffer_from_hal<A: HalApi>( + &self, + hal_buffer: A::Buffer, + device_id: DeviceId, + desc: &resource::BufferDescriptor, + id_in: Option<id::BufferId>, + ) -> (id::BufferId, Option<CreateBufferError>) { + profiling::scope!("Device::create_buffer"); + + let hub = A::hub(self); + let fid = hub.buffers.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + // NB: Any change done through the raw buffer handle will not be + // recorded in the replay + #[cfg(feature = "trace")] + if let Some(trace) = device.trace.lock().as_mut() { + trace.add(trace::Action::CreateBuffer(fid.id(), desc.clone())); + } + + let buffer = device.create_buffer_from_hal(hal_buffer, desc); + + let (id, buffer) = fid.assign(buffer); + api_log!("Device::create_buffer -> {id:?}"); + + device + .trackers + .lock() + .buffers + .insert_single(id, buffer, hal::BufferUses::empty()); + + return (id, None); + }; + + log::error!("Device::create_buffer error: {error}"); + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + pub fn texture_label<A: HalApi>(&self, id: id::TextureId) -> String { + A::hub(self).textures.label_for_resource(id) + } + + pub fn texture_destroy<A: HalApi>( + &self, + texture_id: id::TextureId, + ) -> Result<(), resource::DestroyError> { + profiling::scope!("Texture::destroy"); + api_log!("Texture::destroy {texture_id:?}"); + + let hub = A::hub(self); + + let texture = hub + .textures + .get(texture_id) + .map_err(|_| resource::DestroyError::Invalid)?; + + texture.destroy() + } + + pub fn texture_drop<A: HalApi>(&self, texture_id: id::TextureId, wait: bool) { + profiling::scope!("Texture::drop"); + api_log!("Texture::drop {texture_id:?}"); + + let hub = A::hub(self); + + if let Some(texture) = hub.textures.unregister(texture_id) { + let last_submit_index = texture.info.submission_index(); + + let device = &texture.device; + { + if device + .pending_writes + .lock() + .as_ref() + .unwrap() + .dst_textures + .contains_key(&texture_id) + { + device + .lock_life() + .future_suspected_textures + .push(texture.clone()); + } else { + device + .lock_life() + .suspected_resources + .textures + .insert(texture_id, texture.clone()); + } + } + + if wait { + match device.wait_for_submit(last_submit_index) { + Ok(()) => (), + Err(e) => log::error!("Failed to wait for texture {texture_id:?}: {e}"), + } + } + } + } + + #[allow(unused_unsafe)] + pub fn texture_create_view<A: HalApi>( + &self, + texture_id: id::TextureId, + desc: &resource::TextureViewDescriptor, + id_in: Option<id::TextureViewId>, + ) -> (id::TextureViewId, Option<resource::CreateTextureViewError>) { + profiling::scope!("Texture::create_view"); + + let hub = A::hub(self); + + let fid = hub.texture_views.prepare(id_in); + + let error = loop { + let texture = match hub.textures.get(texture_id) { + Ok(texture) => texture, + Err(_) => break resource::CreateTextureViewError::InvalidTexture, + }; + let device = &texture.device; + { + let snatch_guard = device.snatchable_lock.read(); + if texture.is_destroyed(&snatch_guard) { + break resource::CreateTextureViewError::InvalidTexture; + } + } + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreateTextureView { + id: fid.id(), + parent_id: texture_id, + desc: desc.clone(), + }); + } + + let view = match unsafe { device.create_texture_view(&texture, desc) } { + Ok(view) => view, + Err(e) => break e, + }; + + let (id, resource) = fid.assign(view); + + { + let mut views = texture.views.lock(); + views.push(Arc::downgrade(&resource)); + } + + api_log!("Texture::create_view({texture_id:?}) -> {id:?}"); + device.trackers.lock().views.insert_single(id, resource); + return (id, None); + }; + + log::error!("Texture::create_view({texture_id:?}) error: {error}"); + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + pub fn texture_view_label<A: HalApi>(&self, id: id::TextureViewId) -> String { + A::hub(self).texture_views.label_for_resource(id) + } + + pub fn texture_view_drop<A: HalApi>( + &self, + texture_view_id: id::TextureViewId, + wait: bool, + ) -> Result<(), resource::TextureViewDestroyError> { + profiling::scope!("TextureView::drop"); + api_log!("TextureView::drop {texture_view_id:?}"); + + let hub = A::hub(self); + + if let Some(view) = hub.texture_views.unregister(texture_view_id) { + let last_submit_index = view.info.submission_index(); + + view.device + .lock_life() + .suspected_resources + .texture_views + .insert(texture_view_id, view.clone()); + + if wait { + match view.device.wait_for_submit(last_submit_index) { + Ok(()) => (), + Err(e) => { + log::error!("Failed to wait for texture view {texture_view_id:?}: {e}") + } + } + } + } + Ok(()) + } + + pub fn device_create_sampler<A: HalApi>( + &self, + device_id: DeviceId, + desc: &resource::SamplerDescriptor, + id_in: Option<id::SamplerId>, + ) -> (id::SamplerId, Option<resource::CreateSamplerError>) { + profiling::scope!("Device::create_sampler"); + + let hub = A::hub(self); + let fid = hub.samplers.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreateSampler(fid.id(), desc.clone())); + } + + let sampler = match device.create_sampler(desc) { + Ok(sampler) => sampler, + Err(e) => break e, + }; + + let (id, resource) = fid.assign(sampler); + api_log!("Device::create_sampler -> {id:?}"); + device.trackers.lock().samplers.insert_single(id, resource); + + return (id, None); + }; + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + pub fn sampler_label<A: HalApi>(&self, id: id::SamplerId) -> String { + A::hub(self).samplers.label_for_resource(id) + } + + pub fn sampler_drop<A: HalApi>(&self, sampler_id: id::SamplerId) { + profiling::scope!("Sampler::drop"); + api_log!("Sampler::drop {sampler_id:?}"); + + let hub = A::hub(self); + + if let Some(sampler) = hub.samplers.unregister(sampler_id) { + sampler + .device + .lock_life() + .suspected_resources + .samplers + .insert(sampler_id, sampler.clone()); + } + } + + pub fn device_create_bind_group_layout<A: HalApi>( + &self, + device_id: DeviceId, + desc: &binding_model::BindGroupLayoutDescriptor, + id_in: Option<id::BindGroupLayoutId>, + ) -> ( + id::BindGroupLayoutId, + Option<binding_model::CreateBindGroupLayoutError>, + ) { + profiling::scope!("Device::create_bind_group_layout"); + + let hub = A::hub(self); + let fid = hub.bind_group_layouts.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreateBindGroupLayout(fid.id(), desc.clone())); + } + + let entry_map = match bgl::EntryMap::from_entries(&device.limits, &desc.entries) { + Ok(map) => map, + Err(e) => break e, + }; + + // Currently we make a distinction between fid.assign and fid.assign_existing. This distinction is incorrect, + // but see https://github.com/gfx-rs/wgpu/issues/4912. + // + // `assign` also registers the ID with the resource info, so it can be automatically reclaimed. This needs to + // happen with a mutable reference, which means it can only happen on creation. + // + // Because we need to call `assign` inside the closure (to get mut access), we need to "move" the future id into the closure. + // Rust cannot figure out at compile time that we only ever consume the ID once, so we need to move the check + // to runtime using an Option. + let mut fid = Some(fid); + + // The closure might get called, and it might give us an ID. Side channel it out of the closure. + let mut id = None; + + let bgl_result = device.bgl_pool.get_or_init(entry_map, |entry_map| { + let bgl = + device.create_bind_group_layout(&desc.label, entry_map, bgl::Origin::Pool)?; + + let (id_inner, arc) = fid.take().unwrap().assign(bgl); + id = Some(id_inner); + + Ok(arc) + }); + + let layout = match bgl_result { + Ok(layout) => layout, + Err(e) => break e, + }; + + // If the ID was not assigned, and we survived the above check, + // it means that the bind group layout already existed and we need to call `assign_existing`. + // + // Calling this function _will_ leak the ID. See https://github.com/gfx-rs/wgpu/issues/4912. + if id.is_none() { + id = Some(fid.take().unwrap().assign_existing(&layout)) + } + + api_log!("Device::create_bind_group_layout -> {id:?}"); + return (id.unwrap(), None); + }; + + let fid = hub.bind_group_layouts.prepare(id_in); + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + pub fn bind_group_layout_label<A: HalApi>(&self, id: id::BindGroupLayoutId) -> String { + A::hub(self).bind_group_layouts.label_for_resource(id) + } + + pub fn bind_group_layout_drop<A: HalApi>(&self, bind_group_layout_id: id::BindGroupLayoutId) { + profiling::scope!("BindGroupLayout::drop"); + api_log!("BindGroupLayout::drop {bind_group_layout_id:?}"); + + let hub = A::hub(self); + + if let Some(layout) = hub.bind_group_layouts.unregister(bind_group_layout_id) { + layout + .device + .lock_life() + .suspected_resources + .bind_group_layouts + .insert(bind_group_layout_id, layout.clone()); + } + } + + pub fn device_create_pipeline_layout<A: HalApi>( + &self, + device_id: DeviceId, + desc: &binding_model::PipelineLayoutDescriptor, + id_in: Option<id::PipelineLayoutId>, + ) -> ( + id::PipelineLayoutId, + Option<binding_model::CreatePipelineLayoutError>, + ) { + profiling::scope!("Device::create_pipeline_layout"); + + let hub = A::hub(self); + let fid = hub.pipeline_layouts.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreatePipelineLayout(fid.id(), desc.clone())); + } + + let layout = match device.create_pipeline_layout(desc, &hub.bind_group_layouts) { + Ok(layout) => layout, + Err(e) => break e, + }; + + let (id, _) = fid.assign(layout); + api_log!("Device::create_pipeline_layout -> {id:?}"); + return (id, None); + }; + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + pub fn pipeline_layout_label<A: HalApi>(&self, id: id::PipelineLayoutId) -> String { + A::hub(self).pipeline_layouts.label_for_resource(id) + } + + pub fn pipeline_layout_drop<A: HalApi>(&self, pipeline_layout_id: id::PipelineLayoutId) { + profiling::scope!("PipelineLayout::drop"); + api_log!("PipelineLayout::drop {pipeline_layout_id:?}"); + + let hub = A::hub(self); + if let Some(layout) = hub.pipeline_layouts.unregister(pipeline_layout_id) { + layout + .device + .lock_life() + .suspected_resources + .pipeline_layouts + .insert(pipeline_layout_id, layout.clone()); + } + } + + pub fn device_create_bind_group<A: HalApi>( + &self, + device_id: DeviceId, + desc: &binding_model::BindGroupDescriptor, + id_in: Option<id::BindGroupId>, + ) -> (id::BindGroupId, Option<binding_model::CreateBindGroupError>) { + profiling::scope!("Device::create_bind_group"); + + let hub = A::hub(self); + let fid = hub.bind_groups.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreateBindGroup(fid.id(), desc.clone())); + } + + let bind_group_layout = match hub.bind_group_layouts.get(desc.layout) { + Ok(layout) => layout, + Err(..) => break binding_model::CreateBindGroupError::InvalidLayout, + }; + + if bind_group_layout.device.as_info().id() != device.as_info().id() { + break DeviceError::WrongDevice.into(); + } + + let bind_group = match device.create_bind_group(&bind_group_layout, desc, hub) { + Ok(bind_group) => bind_group, + Err(e) => break e, + }; + + let (id, resource) = fid.assign(bind_group); + + let weak_ref = Arc::downgrade(&resource); + for range in &resource.used_texture_ranges { + range.texture.bind_groups.lock().push(weak_ref.clone()); + } + for range in &resource.used_buffer_ranges { + range.buffer.bind_groups.lock().push(weak_ref.clone()); + } + + api_log!("Device::create_bind_group -> {id:?}"); + + device + .trackers + .lock() + .bind_groups + .insert_single(id, resource); + return (id, None); + }; + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + pub fn bind_group_label<A: HalApi>(&self, id: id::BindGroupId) -> String { + A::hub(self).bind_groups.label_for_resource(id) + } + + pub fn bind_group_drop<A: HalApi>(&self, bind_group_id: id::BindGroupId) { + profiling::scope!("BindGroup::drop"); + api_log!("BindGroup::drop {bind_group_id:?}"); + + let hub = A::hub(self); + + if let Some(bind_group) = hub.bind_groups.unregister(bind_group_id) { + bind_group + .device + .lock_life() + .suspected_resources + .bind_groups + .insert(bind_group_id, bind_group.clone()); + } + } + + pub fn device_create_shader_module<A: HalApi>( + &self, + device_id: DeviceId, + desc: &pipeline::ShaderModuleDescriptor, + source: pipeline::ShaderModuleSource, + id_in: Option<id::ShaderModuleId>, + ) -> ( + id::ShaderModuleId, + Option<pipeline::CreateShaderModuleError>, + ) { + profiling::scope!("Device::create_shader_module"); + + let hub = A::hub(self); + let fid = hub.shader_modules.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + let data = match source { + #[cfg(feature = "wgsl")] + pipeline::ShaderModuleSource::Wgsl(ref code) => { + trace.make_binary("wgsl", code.as_bytes()) + } + #[cfg(feature = "glsl")] + pipeline::ShaderModuleSource::Glsl(ref code, _) => { + trace.make_binary("glsl", code.as_bytes()) + } + #[cfg(feature = "spirv")] + pipeline::ShaderModuleSource::SpirV(ref code, _) => { + trace.make_binary("spirv", bytemuck::cast_slice::<u32, u8>(code)) + } + pipeline::ShaderModuleSource::Naga(ref module) => { + let string = + ron::ser::to_string_pretty(module, ron::ser::PrettyConfig::default()) + .unwrap(); + trace.make_binary("ron", string.as_bytes()) + } + pipeline::ShaderModuleSource::Dummy(_) => { + panic!("found `ShaderModuleSource::Dummy`") + } + }; + trace.add(trace::Action::CreateShaderModule { + id: fid.id(), + desc: desc.clone(), + data, + }); + }; + + let shader = match device.create_shader_module(desc, source) { + Ok(shader) => shader, + Err(e) => break e, + }; + + let (id, _) = fid.assign(shader); + api_log!("Device::create_shader_module -> {id:?}"); + return (id, None); + }; + + log::error!("Device::create_shader_module error: {error}"); + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + // Unsafe-ness of internal calls has little to do with unsafe-ness of this. + #[allow(unused_unsafe)] + /// # Safety + /// + /// This function passes SPIR-V binary to the backend as-is and can potentially result in a + /// driver crash. + pub unsafe fn device_create_shader_module_spirv<A: HalApi>( + &self, + device_id: DeviceId, + desc: &pipeline::ShaderModuleDescriptor, + source: Cow<[u32]>, + id_in: Option<id::ShaderModuleId>, + ) -> ( + id::ShaderModuleId, + Option<pipeline::CreateShaderModuleError>, + ) { + profiling::scope!("Device::create_shader_module"); + + let hub = A::hub(self); + let fid = hub.shader_modules.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + let data = trace.make_binary("spv", unsafe { + std::slice::from_raw_parts(source.as_ptr() as *const u8, source.len() * 4) + }); + trace.add(trace::Action::CreateShaderModule { + id: fid.id(), + desc: desc.clone(), + data, + }); + }; + + let shader = match unsafe { device.create_shader_module_spirv(desc, &source) } { + Ok(shader) => shader, + Err(e) => break e, + }; + let (id, _) = fid.assign(shader); + api_log!("Device::create_shader_module_spirv -> {id:?}"); + return (id, None); + }; + + log::error!("Device::create_shader_module_spirv error: {error}"); + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + pub fn shader_module_label<A: HalApi>(&self, id: id::ShaderModuleId) -> String { + A::hub(self).shader_modules.label_for_resource(id) + } + + pub fn shader_module_drop<A: HalApi>(&self, shader_module_id: id::ShaderModuleId) { + profiling::scope!("ShaderModule::drop"); + api_log!("ShaderModule::drop {shader_module_id:?}"); + + let hub = A::hub(self); + hub.shader_modules.unregister(shader_module_id); + } + + pub fn device_create_command_encoder<A: HalApi>( + &self, + device_id: DeviceId, + desc: &wgt::CommandEncoderDescriptor<Label>, + id_in: Option<id::CommandEncoderId>, + ) -> (id::CommandEncoderId, Option<DeviceError>) { + profiling::scope!("Device::create_command_encoder"); + + let hub = A::hub(self); + let fid = hub.command_buffers.prepare(id_in.map(|id| id.transmute())); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid, + }; + if !device.is_valid() { + break DeviceError::Lost; + } + let queue = match hub.queues.get(device.queue_id.read().unwrap()) { + Ok(queue) => queue, + Err(_) => break DeviceError::InvalidQueueId, + }; + let encoder = match device + .command_allocator + .lock() + .as_mut() + .unwrap() + .acquire_encoder(device.raw(), queue.raw.as_ref().unwrap()) + { + Ok(raw) => raw, + Err(_) => break DeviceError::OutOfMemory, + }; + let command_buffer = command::CommandBuffer::new( + encoder, + &device, + #[cfg(feature = "trace")] + device.trace.lock().is_some(), + desc.label + .to_hal(device.instance_flags) + .map(|s| s.to_string()), + ); + + let (id, _) = fid.assign(command_buffer); + api_log!("Device::create_command_encoder -> {id:?}"); + return (id.transmute(), None); + }; + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id.transmute(), Some(error)) + } + + pub fn command_buffer_label<A: HalApi>(&self, id: id::CommandBufferId) -> String { + A::hub(self).command_buffers.label_for_resource(id) + } + + pub fn command_encoder_drop<A: HalApi>(&self, command_encoder_id: id::CommandEncoderId) { + profiling::scope!("CommandEncoder::drop"); + api_log!("CommandEncoder::drop {command_encoder_id:?}"); + + let hub = A::hub(self); + + if let Some(cmd_buf) = hub + .command_buffers + .unregister(command_encoder_id.transmute()) + { + cmd_buf + .device + .untrack(&cmd_buf.data.lock().as_ref().unwrap().trackers); + } + } + + pub fn command_buffer_drop<A: HalApi>(&self, command_buffer_id: id::CommandBufferId) { + profiling::scope!("CommandBuffer::drop"); + api_log!("CommandBuffer::drop {command_buffer_id:?}"); + self.command_encoder_drop::<A>(command_buffer_id.transmute()) + } + + pub fn device_create_render_bundle_encoder( + &self, + device_id: DeviceId, + desc: &command::RenderBundleEncoderDescriptor, + ) -> ( + *mut command::RenderBundleEncoder, + Option<command::CreateRenderBundleError>, + ) { + profiling::scope!("Device::create_render_bundle_encoder"); + api_log!("Device::device_create_render_bundle_encoder"); + let (encoder, error) = match command::RenderBundleEncoder::new(desc, device_id, None) { + Ok(encoder) => (encoder, None), + Err(e) => (command::RenderBundleEncoder::dummy(device_id), Some(e)), + }; + (Box::into_raw(Box::new(encoder)), error) + } + + pub fn render_bundle_encoder_finish<A: HalApi>( + &self, + bundle_encoder: command::RenderBundleEncoder, + desc: &command::RenderBundleDescriptor, + id_in: Option<id::RenderBundleId>, + ) -> (id::RenderBundleId, Option<command::RenderBundleError>) { + profiling::scope!("RenderBundleEncoder::finish"); + + let hub = A::hub(self); + + let fid = hub.render_bundles.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(bundle_encoder.parent()) { + Ok(device) => device, + Err(_) => break command::RenderBundleError::INVALID_DEVICE, + }; + if !device.is_valid() { + break command::RenderBundleError::INVALID_DEVICE; + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreateRenderBundle { + id: fid.id(), + desc: trace::new_render_bundle_encoder_descriptor( + desc.label.clone(), + &bundle_encoder.context, + bundle_encoder.is_depth_read_only, + bundle_encoder.is_stencil_read_only, + ), + base: bundle_encoder.to_base_pass(), + }); + } + + let render_bundle = match bundle_encoder.finish(desc, &device, hub) { + Ok(bundle) => bundle, + Err(e) => break e, + }; + + let (id, resource) = fid.assign(render_bundle); + api_log!("RenderBundleEncoder::finish -> {id:?}"); + device.trackers.lock().bundles.insert_single(id, resource); + return (id, None); + }; + + let id = fid.assign_error(desc.label.borrow_or_default()); + (id, Some(error)) + } + + pub fn render_bundle_label<A: HalApi>(&self, id: id::RenderBundleId) -> String { + A::hub(self).render_bundles.label_for_resource(id) + } + + pub fn render_bundle_drop<A: HalApi>(&self, render_bundle_id: id::RenderBundleId) { + profiling::scope!("RenderBundle::drop"); + api_log!("RenderBundle::drop {render_bundle_id:?}"); + + let hub = A::hub(self); + + if let Some(bundle) = hub.render_bundles.unregister(render_bundle_id) { + bundle + .device + .lock_life() + .suspected_resources + .render_bundles + .insert(render_bundle_id, bundle.clone()); + } + } + + pub fn device_create_query_set<A: HalApi>( + &self, + device_id: DeviceId, + desc: &resource::QuerySetDescriptor, + id_in: Option<id::QuerySetId>, + ) -> (id::QuerySetId, Option<resource::CreateQuerySetError>) { + profiling::scope!("Device::create_query_set"); + + let hub = A::hub(self); + let fid = hub.query_sets.prepare(id_in); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreateQuerySet { + id: fid.id(), + desc: desc.clone(), + }); + } + + let query_set = match device.create_query_set(desc) { + Ok(query_set) => query_set, + Err(err) => break err, + }; + + let (id, resource) = fid.assign(query_set); + api_log!("Device::create_query_set -> {id:?}"); + device + .trackers + .lock() + .query_sets + .insert_single(id, resource); + + return (id, None); + }; + + let id = fid.assign_error(""); + (id, Some(error)) + } + + pub fn query_set_drop<A: HalApi>(&self, query_set_id: id::QuerySetId) { + profiling::scope!("QuerySet::drop"); + api_log!("QuerySet::drop {query_set_id:?}"); + + let hub = A::hub(self); + + if let Some(query_set) = hub.query_sets.unregister(query_set_id) { + let device = &query_set.device; + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::DestroyQuerySet(query_set_id)); + } + + device + .lock_life() + .suspected_resources + .query_sets + .insert(query_set_id, query_set.clone()); + } + } + + pub fn query_set_label<A: HalApi>(&self, id: id::QuerySetId) -> String { + A::hub(self).query_sets.label_for_resource(id) + } + + pub fn device_create_render_pipeline<A: HalApi>( + &self, + device_id: DeviceId, + desc: &pipeline::RenderPipelineDescriptor, + id_in: Option<id::RenderPipelineId>, + implicit_pipeline_ids: Option<ImplicitPipelineIds<'_>>, + ) -> ( + id::RenderPipelineId, + Option<pipeline::CreateRenderPipelineError>, + ) { + profiling::scope!("Device::create_render_pipeline"); + + let hub = A::hub(self); + + let fid = hub.render_pipelines.prepare(id_in); + let implicit_context = implicit_pipeline_ids.map(|ipi| ipi.prepare(hub)); + let implicit_error_context = implicit_context.clone(); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreateRenderPipeline { + id: fid.id(), + desc: desc.clone(), + implicit_context: implicit_context.clone(), + }); + } + + let pipeline = + match device.create_render_pipeline(&device.adapter, desc, implicit_context, hub) { + Ok(pair) => pair, + Err(e) => break e, + }; + + let (id, resource) = fid.assign(pipeline); + api_log!("Device::create_render_pipeline -> {id:?}"); + + device + .trackers + .lock() + .render_pipelines + .insert_single(id, resource); + + return (id, None); + }; + + let id = fid.assign_error(desc.label.borrow_or_default()); + + // We also need to assign errors to the implicit pipeline layout and the + // implicit bind group layout. We have to remove any existing entries first. + let mut pipeline_layout_guard = hub.pipeline_layouts.write(); + let mut bgl_guard = hub.bind_group_layouts.write(); + if let Some(ref ids) = implicit_error_context { + if pipeline_layout_guard.contains(ids.root_id) { + pipeline_layout_guard.remove(ids.root_id); + } + pipeline_layout_guard.insert_error(ids.root_id, IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL); + for &bgl_id in ids.group_ids.iter() { + if bgl_guard.contains(bgl_id) { + bgl_guard.remove(bgl_id); + } + bgl_guard.insert_error(bgl_id, IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL); + } + } + + log::error!("Device::create_render_pipeline error: {error}"); + + (id, Some(error)) + } + + /// Get an ID of one of the bind group layouts. The ID adds a refcount, + /// which needs to be released by calling `bind_group_layout_drop`. + pub fn render_pipeline_get_bind_group_layout<A: HalApi>( + &self, + pipeline_id: id::RenderPipelineId, + index: u32, + id_in: Option<id::BindGroupLayoutId>, + ) -> ( + id::BindGroupLayoutId, + Option<binding_model::GetBindGroupLayoutError>, + ) { + let hub = A::hub(self); + + let error = loop { + let pipeline = match hub.render_pipelines.get(pipeline_id) { + Ok(pipeline) => pipeline, + Err(_) => break binding_model::GetBindGroupLayoutError::InvalidPipeline, + }; + let id = match pipeline.layout.bind_group_layouts.get(index as usize) { + Some(bg) => hub.bind_group_layouts.prepare(id_in).assign_existing(bg), + None => break binding_model::GetBindGroupLayoutError::InvalidGroupIndex(index), + }; + return (id, None); + }; + + let id = hub + .bind_group_layouts + .prepare(id_in) + .assign_error("<derived>"); + (id, Some(error)) + } + + pub fn render_pipeline_label<A: HalApi>(&self, id: id::RenderPipelineId) -> String { + A::hub(self).render_pipelines.label_for_resource(id) + } + + pub fn render_pipeline_drop<A: HalApi>(&self, render_pipeline_id: id::RenderPipelineId) { + profiling::scope!("RenderPipeline::drop"); + api_log!("RenderPipeline::drop {render_pipeline_id:?}"); + + let hub = A::hub(self); + + if let Some(pipeline) = hub.render_pipelines.unregister(render_pipeline_id) { + let layout_id = pipeline.layout.as_info().id(); + let device = &pipeline.device; + let mut life_lock = device.lock_life(); + life_lock + .suspected_resources + .render_pipelines + .insert(render_pipeline_id, pipeline.clone()); + + life_lock + .suspected_resources + .pipeline_layouts + .insert(layout_id, pipeline.layout.clone()); + } + } + + pub fn device_create_compute_pipeline<A: HalApi>( + &self, + device_id: DeviceId, + desc: &pipeline::ComputePipelineDescriptor, + id_in: Option<id::ComputePipelineId>, + implicit_pipeline_ids: Option<ImplicitPipelineIds<'_>>, + ) -> ( + id::ComputePipelineId, + Option<pipeline::CreateComputePipelineError>, + ) { + profiling::scope!("Device::create_compute_pipeline"); + + let hub = A::hub(self); + + let fid = hub.compute_pipelines.prepare(id_in); + let implicit_context = implicit_pipeline_ids.map(|ipi| ipi.prepare(hub)); + let implicit_error_context = implicit_context.clone(); + + let error = loop { + let device = match hub.devices.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::CreateComputePipeline { + id: fid.id(), + desc: desc.clone(), + implicit_context: implicit_context.clone(), + }); + } + let pipeline = match device.create_compute_pipeline(desc, implicit_context, hub) { + Ok(pair) => pair, + Err(e) => break e, + }; + + let (id, resource) = fid.assign(pipeline); + api_log!("Device::create_compute_pipeline -> {id:?}"); + + device + .trackers + .lock() + .compute_pipelines + .insert_single(id, resource); + return (id, None); + }; + + let id = fid.assign_error(desc.label.borrow_or_default()); + + // We also need to assign errors to the implicit pipeline layout and the + // implicit bind group layout. We have to remove any existing entries first. + let mut pipeline_layout_guard = hub.pipeline_layouts.write(); + let mut bgl_guard = hub.bind_group_layouts.write(); + if let Some(ref ids) = implicit_error_context { + if pipeline_layout_guard.contains(ids.root_id) { + pipeline_layout_guard.remove(ids.root_id); + } + pipeline_layout_guard.insert_error(ids.root_id, IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL); + for &bgl_id in ids.group_ids.iter() { + if bgl_guard.contains(bgl_id) { + bgl_guard.remove(bgl_id); + } + bgl_guard.insert_error(bgl_id, IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL); + } + } + (id, Some(error)) + } + + /// Get an ID of one of the bind group layouts. The ID adds a refcount, + /// which needs to be released by calling `bind_group_layout_drop`. + pub fn compute_pipeline_get_bind_group_layout<A: HalApi>( + &self, + pipeline_id: id::ComputePipelineId, + index: u32, + id_in: Option<id::BindGroupLayoutId>, + ) -> ( + id::BindGroupLayoutId, + Option<binding_model::GetBindGroupLayoutError>, + ) { + let hub = A::hub(self); + + let error = loop { + let pipeline = match hub.compute_pipelines.get(pipeline_id) { + Ok(pipeline) => pipeline, + Err(_) => break binding_model::GetBindGroupLayoutError::InvalidPipeline, + }; + + let id = match pipeline.layout.bind_group_layouts.get(index as usize) { + Some(bg) => hub.bind_group_layouts.prepare(id_in).assign_existing(bg), + None => break binding_model::GetBindGroupLayoutError::InvalidGroupIndex(index), + }; + + return (id, None); + }; + + let id = hub + .bind_group_layouts + .prepare(id_in) + .assign_error("<derived>"); + (id, Some(error)) + } + + pub fn compute_pipeline_label<A: HalApi>(&self, id: id::ComputePipelineId) -> String { + A::hub(self).compute_pipelines.label_for_resource(id) + } + + pub fn compute_pipeline_drop<A: HalApi>(&self, compute_pipeline_id: id::ComputePipelineId) { + profiling::scope!("ComputePipeline::drop"); + api_log!("ComputePipeline::drop {compute_pipeline_id:?}"); + + let hub = A::hub(self); + + if let Some(pipeline) = hub.compute_pipelines.unregister(compute_pipeline_id) { + let layout_id = pipeline.layout.as_info().id(); + let device = &pipeline.device; + let mut life_lock = device.lock_life(); + life_lock + .suspected_resources + .compute_pipelines + .insert(compute_pipeline_id, pipeline.clone()); + life_lock + .suspected_resources + .pipeline_layouts + .insert(layout_id, pipeline.layout.clone()); + } + } + + pub fn surface_configure<A: HalApi>( + &self, + surface_id: SurfaceId, + device_id: DeviceId, + config: &wgt::SurfaceConfiguration<Vec<TextureFormat>>, + ) -> Option<present::ConfigureSurfaceError> { + use hal::{Adapter as _, Surface as _}; + use present::ConfigureSurfaceError as E; + profiling::scope!("surface_configure"); + + fn validate_surface_configuration( + config: &mut hal::SurfaceConfiguration, + caps: &hal::SurfaceCapabilities, + max_texture_dimension_2d: u32, + ) -> Result<(), E> { + let width = config.extent.width; + let height = config.extent.height; + + if width > max_texture_dimension_2d || height > max_texture_dimension_2d { + return Err(E::TooLarge { + width, + height, + max_texture_dimension_2d, + }); + } + + if !caps.present_modes.contains(&config.present_mode) { + let new_mode = 'b: loop { + // Automatic present mode checks. + // + // The "Automatic" modes are never supported by the backends. + let fallbacks = match config.present_mode { + wgt::PresentMode::AutoVsync => { + &[wgt::PresentMode::FifoRelaxed, wgt::PresentMode::Fifo][..] + } + // Always end in FIFO to make sure it's always supported + wgt::PresentMode::AutoNoVsync => &[ + wgt::PresentMode::Immediate, + wgt::PresentMode::Mailbox, + wgt::PresentMode::Fifo, + ][..], + _ => { + return Err(E::UnsupportedPresentMode { + requested: config.present_mode, + available: caps.present_modes.clone(), + }); + } + }; + + for &fallback in fallbacks { + if caps.present_modes.contains(&fallback) { + break 'b fallback; + } + } + + unreachable!("Fallback system failed to choose present mode. This is a bug. Mode: {:?}, Options: {:?}", config.present_mode, &caps.present_modes); + }; + + api_log!( + "Automatically choosing presentation mode by rule {:?}. Chose {new_mode:?}", + config.present_mode + ); + config.present_mode = new_mode; + } + if !caps.formats.contains(&config.format) { + return Err(E::UnsupportedFormat { + requested: config.format, + available: caps.formats.clone(), + }); + } + if !caps + .composite_alpha_modes + .contains(&config.composite_alpha_mode) + { + let new_alpha_mode = 'alpha: loop { + // Automatic alpha mode checks. + let fallbacks = match config.composite_alpha_mode { + wgt::CompositeAlphaMode::Auto => &[ + wgt::CompositeAlphaMode::Opaque, + wgt::CompositeAlphaMode::Inherit, + ][..], + _ => { + return Err(E::UnsupportedAlphaMode { + requested: config.composite_alpha_mode, + available: caps.composite_alpha_modes.clone(), + }); + } + }; + + for &fallback in fallbacks { + if caps.composite_alpha_modes.contains(&fallback) { + break 'alpha fallback; + } + } + + unreachable!( + "Fallback system failed to choose alpha mode. This is a bug. \ + AlphaMode: {:?}, Options: {:?}", + config.composite_alpha_mode, &caps.composite_alpha_modes + ); + }; + + api_log!( + "Automatically choosing alpha mode by rule {:?}. Chose {new_alpha_mode:?}", + config.composite_alpha_mode + ); + config.composite_alpha_mode = new_alpha_mode; + } + if !caps.usage.contains(config.usage) { + return Err(E::UnsupportedUsage); + } + if width == 0 || height == 0 { + return Err(E::ZeroArea); + } + Ok(()) + } + + log::debug!("configuring surface with {:?}", config); + + let error = 'outer: loop { + // User callbacks must not be called while we are holding locks. + let user_callbacks; + { + let hub = A::hub(self); + let surface_guard = self.surfaces.read(); + let device_guard = hub.devices.read(); + + let device = match device_guard.get(device_id) { + Ok(device) => device, + Err(_) => break DeviceError::Invalid.into(), + }; + if !device.is_valid() { + break DeviceError::Lost.into(); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(trace::Action::ConfigureSurface(surface_id, config.clone())); + } + + let surface = match surface_guard.get(surface_id) { + Ok(surface) => surface, + Err(_) => break E::InvalidSurface, + }; + + let caps = unsafe { + let suf = A::get_surface(surface); + let adapter = &device.adapter; + match adapter.raw.adapter.surface_capabilities(suf.unwrap()) { + Some(caps) => caps, + None => break E::UnsupportedQueueFamily, + } + }; + + let mut hal_view_formats = vec![]; + for format in config.view_formats.iter() { + if *format == config.format { + continue; + } + if !caps.formats.contains(&config.format) { + break 'outer E::UnsupportedFormat { + requested: config.format, + available: caps.formats, + }; + } + if config.format.remove_srgb_suffix() != format.remove_srgb_suffix() { + break 'outer E::InvalidViewFormat(*format, config.format); + } + hal_view_formats.push(*format); + } + + if !hal_view_formats.is_empty() { + if let Err(missing_flag) = + device.require_downlevel_flags(wgt::DownlevelFlags::SURFACE_VIEW_FORMATS) + { + break 'outer E::MissingDownlevelFlags(missing_flag); + } + } + + let maximum_frame_latency = config.desired_maximum_frame_latency.clamp( + *caps.maximum_frame_latency.start(), + *caps.maximum_frame_latency.end(), + ); + let mut hal_config = hal::SurfaceConfiguration { + maximum_frame_latency, + present_mode: config.present_mode, + composite_alpha_mode: config.alpha_mode, + format: config.format, + extent: wgt::Extent3d { + width: config.width, + height: config.height, + depth_or_array_layers: 1, + }, + usage: conv::map_texture_usage(config.usage, hal::FormatAspects::COLOR), + view_formats: hal_view_formats, + }; + + if let Err(error) = validate_surface_configuration( + &mut hal_config, + &caps, + device.limits.max_texture_dimension_2d, + ) { + break error; + } + + // Wait for all work to finish before configuring the surface. + let fence = device.fence.read(); + let fence = fence.as_ref().unwrap(); + match device.maintain(fence, wgt::Maintain::Wait) { + Ok((closures, _)) => { + user_callbacks = closures; + } + Err(e) => { + break e.into(); + } + } + + // All textures must be destroyed before the surface can be re-configured. + if let Some(present) = surface.presentation.lock().take() { + if present.acquired_texture.is_some() { + break E::PreviousOutputExists; + } + } + + // TODO: Texture views may still be alive that point to the texture. + // this will allow the user to render to the surface texture, long after + // it has been removed. + // + // https://github.com/gfx-rs/wgpu/issues/4105 + + match unsafe { + A::get_surface(surface) + .unwrap() + .configure(device.raw(), &hal_config) + } { + Ok(()) => (), + Err(error) => { + break match error { + hal::SurfaceError::Outdated | hal::SurfaceError::Lost => { + E::InvalidSurface + } + hal::SurfaceError::Device(error) => E::Device(error.into()), + hal::SurfaceError::Other(message) => { + log::error!("surface configuration failed: {}", message); + E::InvalidSurface + } + } + } + } + + let mut presentation = surface.presentation.lock(); + *presentation = Some(present::Presentation { + device: super::any_device::AnyDevice::new(device.clone()), + config: config.clone(), + acquired_texture: None, + }); + } + + user_callbacks.fire(); + return None; + }; + + Some(error) + } + + #[cfg(feature = "replay")] + /// Only triangle suspected resource IDs. This helps us to avoid ID collisions + /// upon creating new resources when re-playing a trace. + pub fn device_maintain_ids<A: HalApi>(&self, device_id: DeviceId) -> Result<(), InvalidDevice> { + let hub = A::hub(self); + + let device = hub.devices.get(device_id).map_err(|_| InvalidDevice)?; + if !device.is_valid() { + return Err(InvalidDevice); + } + device.lock_life().triage_suspected(&device.trackers); + Ok(()) + } + + /// Check `device_id` for freeable resources and completed buffer mappings. + /// + /// Return `queue_empty` indicating whether there are more queue submissions still in flight. + pub fn device_poll<A: HalApi>( + &self, + device_id: DeviceId, + maintain: wgt::Maintain<queue::WrappedSubmissionIndex>, + ) -> Result<bool, WaitIdleError> { + api_log!("Device::poll"); + + let hub = A::hub(self); + let device = hub + .devices + .get(device_id) + .map_err(|_| DeviceError::Invalid)?; + + let (closures, queue_empty) = { + if let wgt::Maintain::WaitForSubmissionIndex(submission_index) = maintain { + if submission_index.queue_id != device_id.transmute() { + return Err(WaitIdleError::WrongSubmissionIndex( + submission_index.queue_id, + device_id, + )); + } + } + + let fence = device.fence.read(); + let fence = fence.as_ref().unwrap(); + device.maintain(fence, maintain)? + }; + + // Some deferred destroys are scheduled in maintain so run this right after + // to avoid holding on to them until the next device poll. + device.deferred_resource_destruction(); + + closures.fire(); + + Ok(queue_empty) + } + + /// Poll all devices belonging to the backend `A`. + /// + /// If `force_wait` is true, block until all buffer mappings are done. + /// + /// Return `all_queue_empty` indicating whether there are more queue + /// submissions still in flight. + fn poll_device<A: HalApi>( + &self, + force_wait: bool, + closures: &mut UserClosures, + ) -> Result<bool, WaitIdleError> { + profiling::scope!("poll_device"); + + let hub = A::hub(self); + let mut all_queue_empty = true; + { + let device_guard = hub.devices.read(); + + for (_id, device) in device_guard.iter(A::VARIANT) { + let maintain = if force_wait { + wgt::Maintain::Wait + } else { + wgt::Maintain::Poll + }; + let fence = device.fence.read(); + let fence = fence.as_ref().unwrap(); + let (cbs, queue_empty) = device.maintain(fence, maintain)?; + all_queue_empty = all_queue_empty && queue_empty; + + closures.extend(cbs); + } + } + + Ok(all_queue_empty) + } + + /// Poll all devices on all backends. + /// + /// This is the implementation of `wgpu::Instance::poll_all`. + /// + /// Return `all_queue_empty` indicating whether there are more queue + /// submissions still in flight. + pub fn poll_all_devices(&self, force_wait: bool) -> Result<bool, WaitIdleError> { + api_log!("poll_all_devices"); + let mut closures = UserClosures::default(); + let mut all_queue_empty = true; + + #[cfg(vulkan)] + { + all_queue_empty = + self.poll_device::<hal::api::Vulkan>(force_wait, &mut closures)? && all_queue_empty; + } + #[cfg(metal)] + { + all_queue_empty = + self.poll_device::<hal::api::Metal>(force_wait, &mut closures)? && all_queue_empty; + } + #[cfg(dx12)] + { + all_queue_empty = + self.poll_device::<hal::api::Dx12>(force_wait, &mut closures)? && all_queue_empty; + } + #[cfg(gles)] + { + all_queue_empty = + self.poll_device::<hal::api::Gles>(force_wait, &mut closures)? && all_queue_empty; + } + + closures.fire(); + + Ok(all_queue_empty) + } + + pub fn device_label<A: HalApi>(&self, id: DeviceId) -> String { + A::hub(self).devices.label_for_resource(id) + } + + pub fn device_start_capture<A: HalApi>(&self, id: DeviceId) { + api_log!("Device::start_capture"); + + let hub = A::hub(self); + + if let Ok(device) = hub.devices.get(id) { + if !device.is_valid() { + return; + } + unsafe { device.raw().start_capture() }; + } + } + + pub fn device_stop_capture<A: HalApi>(&self, id: DeviceId) { + api_log!("Device::stop_capture"); + + let hub = A::hub(self); + + if let Ok(device) = hub.devices.get(id) { + if !device.is_valid() { + return; + } + unsafe { device.raw().stop_capture() }; + } + } + + pub fn device_drop<A: HalApi>(&self, device_id: DeviceId) { + profiling::scope!("Device::drop"); + api_log!("Device::drop {device_id:?}"); + + let hub = A::hub(self); + if let Some(device) = hub.devices.unregister(device_id) { + let device_lost_closure = device.lock_life().device_lost_closure.take(); + if let Some(closure) = device_lost_closure { + closure.call(DeviceLostReason::Dropped, String::from("Device dropped.")); + } + + // The things `Device::prepare_to_die` takes care are mostly + // unnecessary here. We know our queue is empty, so we don't + // need to wait for submissions or triage them. We know we were + // just polled, so `life_tracker.free_resources` is empty. + debug_assert!(device.lock_life().queue_empty()); + { + let mut pending_writes = device.pending_writes.lock(); + let pending_writes = pending_writes.as_mut().unwrap(); + pending_writes.deactivate(); + } + + drop(device); + } + } + + // This closure will be called exactly once during "lose the device", + // or when it is replaced. + pub fn device_set_device_lost_closure<A: HalApi>( + &self, + device_id: DeviceId, + device_lost_closure: DeviceLostClosure, + ) { + let hub = A::hub(self); + + if let Ok(device) = hub.devices.get(device_id) { + let mut life_tracker = device.lock_life(); + if let Some(existing_closure) = life_tracker.device_lost_closure.take() { + // It's important to not hold the lock while calling the closure. + drop(life_tracker); + existing_closure.call(DeviceLostReason::ReplacedCallback, "".to_string()); + life_tracker = device.lock_life(); + } + life_tracker.device_lost_closure = Some(device_lost_closure); + } + } + + pub fn device_destroy<A: HalApi>(&self, device_id: DeviceId) { + api_log!("Device::destroy {device_id:?}"); + + let hub = A::hub(self); + + if let Ok(device) = hub.devices.get(device_id) { + // Follow the steps at + // https://gpuweb.github.io/gpuweb/#dom-gpudevice-destroy. + // It's legal to call destroy multiple times, but if the device + // is already invalid, there's nothing more to do. There's also + // no need to return an error. + if !device.is_valid() { + return; + } + + // The last part of destroy is to lose the device. The spec says + // delay that until all "currently-enqueued operations on any + // queue on this device are completed." This is accomplished by + // setting valid to false, and then relying upon maintain to + // check for empty queues and a DeviceLostClosure. At that time, + // the DeviceLostClosure will be called with "destroyed" as the + // reason. + device.valid.store(false, Ordering::Relaxed); + } + } + + pub fn device_mark_lost<A: HalApi>(&self, device_id: DeviceId, message: &str) { + api_log!("Device::mark_lost {device_id:?}"); + + let hub = A::hub(self); + + if let Ok(device) = hub.devices.get(device_id) { + device.lose(message); + } + } + + pub fn queue_drop<A: HalApi>(&self, queue_id: QueueId) { + profiling::scope!("Queue::drop"); + api_log!("Queue::drop {queue_id:?}"); + + let hub = A::hub(self); + if let Some(queue) = hub.queues.unregister(queue_id) { + drop(queue); + } + } + + pub fn buffer_map_async<A: HalApi>( + &self, + buffer_id: id::BufferId, + range: Range<BufferAddress>, + op: BufferMapOperation, + ) -> BufferAccessResult { + api_log!("Buffer::map_async {buffer_id:?} range {range:?} op: {op:?}"); + + // User callbacks must not be called while holding buffer_map_async_inner's locks, so we + // defer the error callback if it needs to be called immediately (typically when running + // into errors). + if let Err((mut operation, err)) = self.buffer_map_async_inner::<A>(buffer_id, range, op) { + if let Some(callback) = operation.callback.take() { + callback.call(Err(err.clone())); + } + log::error!("Buffer::map_async error: {err}"); + return Err(err); + } + + Ok(()) + } + + // Returns the mapping callback in case of error so that the callback can be fired outside + // of the locks that are held in this function. + fn buffer_map_async_inner<A: HalApi>( + &self, + buffer_id: id::BufferId, + range: Range<BufferAddress>, + op: BufferMapOperation, + ) -> Result<(), (BufferMapOperation, BufferAccessError)> { + profiling::scope!("Buffer::map_async"); + + let hub = A::hub(self); + + let (pub_usage, internal_use) = match op.host { + HostMap::Read => (wgt::BufferUsages::MAP_READ, hal::BufferUses::MAP_READ), + HostMap::Write => (wgt::BufferUsages::MAP_WRITE, hal::BufferUses::MAP_WRITE), + }; + + if range.start % wgt::MAP_ALIGNMENT != 0 || range.end % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err((op, BufferAccessError::UnalignedRange)); + } + + let buffer = { + let buffer = hub + .buffers + .get(buffer_id) + .map_err(|_| BufferAccessError::Invalid); + + let buffer = match buffer { + Ok(b) => b, + Err(e) => { + return Err((op, e)); + } + }; + + let device = &buffer.device; + if !device.is_valid() { + return Err((op, DeviceError::Lost.into())); + } + + if let Err(e) = check_buffer_usage(buffer.usage, pub_usage) { + return Err((op, e.into())); + } + + if range.start > range.end { + return Err(( + op, + BufferAccessError::NegativeRange { + start: range.start, + end: range.end, + }, + )); + } + if range.end > buffer.size { + return Err(( + op, + BufferAccessError::OutOfBoundsOverrun { + index: range.end, + max: buffer.size, + }, + )); + } + + let snatch_guard = device.snatchable_lock.read(); + if buffer.is_destroyed(&snatch_guard) { + return Err((op, BufferAccessError::Destroyed)); + } + + { + let map_state = &mut *buffer.map_state.lock(); + *map_state = match *map_state { + resource::BufferMapState::Init { .. } + | resource::BufferMapState::Active { .. } => { + return Err((op, BufferAccessError::AlreadyMapped)); + } + resource::BufferMapState::Waiting(_) => { + return Err((op, BufferAccessError::MapAlreadyPending)); + } + resource::BufferMapState::Idle => { + resource::BufferMapState::Waiting(resource::BufferPendingMapping { + range, + op, + _parent_buffer: buffer.clone(), + }) + } + }; + } + + { + let mut trackers = buffer.device.as_ref().trackers.lock(); + trackers.buffers.set_single(&buffer, internal_use); + //TODO: Check if draining ALL buffers is correct! + let _ = trackers.buffers.drain_transitions(&snatch_guard); + } + + drop(snatch_guard); + + buffer + }; + + buffer.device.lock_life().map(&buffer); + + Ok(()) + } + + pub fn buffer_get_mapped_range<A: HalApi>( + &self, + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option<BufferAddress>, + ) -> Result<(*mut u8, u64), BufferAccessError> { + profiling::scope!("Buffer::get_mapped_range"); + api_log!("Buffer::get_mapped_range {buffer_id:?} offset {offset:?} size {size:?}"); + + let hub = A::hub(self); + + let buffer = hub + .buffers + .get(buffer_id) + .map_err(|_| BufferAccessError::Invalid)?; + + { + let snatch_guard = buffer.device.snatchable_lock.read(); + if buffer.is_destroyed(&snatch_guard) { + return Err(BufferAccessError::Destroyed); + } + } + + let range_size = if let Some(size) = size { + size + } else if offset > buffer.size { + 0 + } else { + buffer.size - offset + }; + + if offset % wgt::MAP_ALIGNMENT != 0 { + return Err(BufferAccessError::UnalignedOffset { offset }); + } + if range_size % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err(BufferAccessError::UnalignedRangeSize { range_size }); + } + let map_state = &*buffer.map_state.lock(); + match *map_state { + resource::BufferMapState::Init { ref ptr, .. } => { + // offset (u64) can not be < 0, so no need to validate the lower bound + if offset + range_size > buffer.size { + return Err(BufferAccessError::OutOfBoundsOverrun { + index: offset + range_size - 1, + max: buffer.size, + }); + } + unsafe { Ok((ptr.as_ptr().offset(offset as isize), range_size)) } + } + resource::BufferMapState::Active { + ref ptr, ref range, .. + } => { + if offset < range.start { + return Err(BufferAccessError::OutOfBoundsUnderrun { + index: offset, + min: range.start, + }); + } + if offset + range_size > range.end { + return Err(BufferAccessError::OutOfBoundsOverrun { + index: offset + range_size - 1, + max: range.end, + }); + } + // ptr points to the beginning of the range we mapped in map_async + // rather than the beginning of the buffer. + let relative_offset = (offset - range.start) as isize; + unsafe { Ok((ptr.as_ptr().offset(relative_offset), range_size)) } + } + resource::BufferMapState::Idle | resource::BufferMapState::Waiting(_) => { + Err(BufferAccessError::NotMapped) + } + } + } + pub fn buffer_unmap<A: HalApi>(&self, buffer_id: id::BufferId) -> BufferAccessResult { + profiling::scope!("unmap", "Buffer"); + api_log!("Buffer::unmap {buffer_id:?}"); + + let hub = A::hub(self); + + let buffer = hub + .buffers + .get(buffer_id) + .map_err(|_| BufferAccessError::Invalid)?; + + let snatch_guard = buffer.device.snatchable_lock.read(); + if buffer.is_destroyed(&snatch_guard) { + return Err(BufferAccessError::Destroyed); + } + drop(snatch_guard); + + if !buffer.device.is_valid() { + return Err(DeviceError::Lost.into()); + } + + buffer.unmap() + } +} diff --git a/third_party/rust/wgpu-core/src/device/life.rs b/third_party/rust/wgpu-core/src/device/life.rs new file mode 100644 index 0000000000..86c5d027c7 --- /dev/null +++ b/third_party/rust/wgpu-core/src/device/life.rs @@ -0,0 +1,831 @@ +use crate::{ + binding_model::{BindGroup, BindGroupLayout, PipelineLayout}, + command::RenderBundle, + device::{ + queue::{EncoderInFlight, SubmittedWorkDoneClosure, TempResource}, + DeviceError, DeviceLostClosure, + }, + hal_api::HalApi, + id::{ + self, BindGroupId, BindGroupLayoutId, BufferId, ComputePipelineId, Id, PipelineLayoutId, + QuerySetId, RenderBundleId, RenderPipelineId, SamplerId, StagingBufferId, TextureId, + TextureViewId, + }, + pipeline::{ComputePipeline, RenderPipeline}, + resource::{ + self, Buffer, DestroyedBuffer, DestroyedTexture, QuerySet, Resource, Sampler, + StagingBuffer, Texture, TextureView, + }, + track::{ResourceTracker, Tracker}, + FastHashMap, SubmissionIndex, +}; +use smallvec::SmallVec; + +use parking_lot::Mutex; +use std::sync::Arc; +use thiserror::Error; + +/// A struct that keeps lists of resources that are no longer needed by the user. +#[derive(Default)] +pub(crate) struct ResourceMaps<A: HalApi> { + pub buffers: FastHashMap<BufferId, Arc<Buffer<A>>>, + pub staging_buffers: FastHashMap<StagingBufferId, Arc<StagingBuffer<A>>>, + pub textures: FastHashMap<TextureId, Arc<Texture<A>>>, + pub texture_views: FastHashMap<TextureViewId, Arc<TextureView<A>>>, + pub samplers: FastHashMap<SamplerId, Arc<Sampler<A>>>, + pub bind_groups: FastHashMap<BindGroupId, Arc<BindGroup<A>>>, + pub bind_group_layouts: FastHashMap<BindGroupLayoutId, Arc<BindGroupLayout<A>>>, + pub render_pipelines: FastHashMap<RenderPipelineId, Arc<RenderPipeline<A>>>, + pub compute_pipelines: FastHashMap<ComputePipelineId, Arc<ComputePipeline<A>>>, + pub pipeline_layouts: FastHashMap<PipelineLayoutId, Arc<PipelineLayout<A>>>, + pub render_bundles: FastHashMap<RenderBundleId, Arc<RenderBundle<A>>>, + pub query_sets: FastHashMap<QuerySetId, Arc<QuerySet<A>>>, + pub destroyed_buffers: FastHashMap<BufferId, Arc<DestroyedBuffer<A>>>, + pub destroyed_textures: FastHashMap<TextureId, Arc<DestroyedTexture<A>>>, +} + +impl<A: HalApi> ResourceMaps<A> { + pub(crate) fn new() -> Self { + ResourceMaps { + buffers: FastHashMap::default(), + staging_buffers: FastHashMap::default(), + textures: FastHashMap::default(), + texture_views: FastHashMap::default(), + samplers: FastHashMap::default(), + bind_groups: FastHashMap::default(), + bind_group_layouts: FastHashMap::default(), + render_pipelines: FastHashMap::default(), + compute_pipelines: FastHashMap::default(), + pipeline_layouts: FastHashMap::default(), + render_bundles: FastHashMap::default(), + query_sets: FastHashMap::default(), + destroyed_buffers: FastHashMap::default(), + destroyed_textures: FastHashMap::default(), + } + } + + pub(crate) fn clear(&mut self) { + let ResourceMaps { + buffers, + staging_buffers, + textures, + texture_views, + samplers, + bind_groups, + bind_group_layouts, + render_pipelines, + compute_pipelines, + pipeline_layouts, + render_bundles, + query_sets, + destroyed_buffers, + destroyed_textures, + } = self; + buffers.clear(); + staging_buffers.clear(); + textures.clear(); + texture_views.clear(); + samplers.clear(); + bind_groups.clear(); + bind_group_layouts.clear(); + render_pipelines.clear(); + compute_pipelines.clear(); + pipeline_layouts.clear(); + render_bundles.clear(); + query_sets.clear(); + destroyed_buffers.clear(); + destroyed_textures.clear(); + } + + pub(crate) fn extend(&mut self, mut other: Self) { + let ResourceMaps { + buffers, + staging_buffers, + textures, + texture_views, + samplers, + bind_groups, + bind_group_layouts, + render_pipelines, + compute_pipelines, + pipeline_layouts, + render_bundles, + query_sets, + destroyed_buffers, + destroyed_textures, + } = self; + buffers.extend(other.buffers.drain()); + staging_buffers.extend(other.staging_buffers.drain()); + textures.extend(other.textures.drain()); + texture_views.extend(other.texture_views.drain()); + samplers.extend(other.samplers.drain()); + bind_groups.extend(other.bind_groups.drain()); + bind_group_layouts.extend(other.bind_group_layouts.drain()); + render_pipelines.extend(other.render_pipelines.drain()); + compute_pipelines.extend(other.compute_pipelines.drain()); + pipeline_layouts.extend(other.pipeline_layouts.drain()); + render_bundles.extend(other.render_bundles.drain()); + query_sets.extend(other.query_sets.drain()); + destroyed_buffers.extend(other.destroyed_buffers.drain()); + destroyed_textures.extend(other.destroyed_textures.drain()); + } +} + +/// Resources used by a queue submission, and work to be done once it completes. +struct ActiveSubmission<A: HalApi> { + /// The index of the submission we track. + /// + /// When `Device::fence`'s value is greater than or equal to this, our queue + /// submission has completed. + index: SubmissionIndex, + + /// Resources to be freed once this queue submission has completed. + /// + /// When the device is polled, for completed submissions, + /// `triage_submissions` removes resources that don't need to be held alive any longer + /// from there. + /// + /// This includes things like temporary resources and resources that are + /// used by submitted commands but have been dropped by the user (meaning that + /// this submission is their last reference.) + last_resources: ResourceMaps<A>, + + /// Buffers to be mapped once this submission has completed. + mapped: Vec<Arc<Buffer<A>>>, + + encoders: Vec<EncoderInFlight<A>>, + + /// List of queue "on_submitted_work_done" closures to be called once this + /// submission has completed. + work_done_closures: SmallVec<[SubmittedWorkDoneClosure; 1]>, +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum WaitIdleError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Tried to wait using a submission index from the wrong device. Submission index is from device {0:?}. Called poll on device {1:?}.")] + WrongSubmissionIndex(id::QueueId, id::DeviceId), + #[error("GPU got stuck :(")] + StuckGpu, +} + +/// Resource tracking for a device. +/// +/// ## Host mapping buffers +/// +/// A buffer cannot be mapped until all active queue submissions that use it +/// have completed. To that end: +/// +/// - Each buffer's `ResourceInfo::submission_index` records the index of the +/// most recent queue submission that uses that buffer. +/// +/// - Calling `Global::buffer_map_async` adds the buffer to +/// `self.mapped`, and changes `Buffer::map_state` to prevent it +/// from being used in any new submissions. +/// +/// - When the device is polled, the following `LifetimeTracker` methods decide +/// what should happen next: +/// +/// 1) `triage_mapped` drains `self.mapped`, checking the submission index +/// of each buffer against the queue submissions that have finished +/// execution. Buffers used by submissions still in flight go in +/// `self.active[index].mapped`, and the rest go into +/// `self.ready_to_map`. +/// +/// 2) `triage_submissions` moves entries in `self.active[i]` for completed +/// submissions to `self.ready_to_map`. At this point, both +/// `self.active` and `self.ready_to_map` are up to date with the given +/// submission index. +/// +/// 3) `handle_mapping` drains `self.ready_to_map` and actually maps the +/// buffers, collecting a list of notification closures to call. But any +/// buffers that were dropped by the user get moved to +/// `self.free_resources`. +/// +/// Only calling `Global::buffer_map_async` clones a new `Arc` for the +/// buffer. This new `Arc` is only dropped by `handle_mapping`. +pub(crate) struct LifetimeTracker<A: HalApi> { + /// Resources that the user has requested be mapped, but which are used by + /// queue submissions still in flight. + mapped: Vec<Arc<Buffer<A>>>, + + /// Buffers can be used in a submission that is yet to be made, by the + /// means of `write_buffer()`, so we have a special place for them. + pub future_suspected_buffers: Vec<Arc<Buffer<A>>>, + + /// Textures can be used in the upcoming submission by `write_texture`. + pub future_suspected_textures: Vec<Arc<Texture<A>>>, + + /// Resources whose user handle has died (i.e. drop/destroy has been called) + /// and will likely be ready for destruction soon. + pub suspected_resources: ResourceMaps<A>, + + /// Resources used by queue submissions still in flight. One entry per + /// submission, with older submissions appearing before younger. + /// + /// Entries are added by `track_submission` and drained by + /// `LifetimeTracker::triage_submissions`. Lots of methods contribute data + /// to particular entries. + active: Vec<ActiveSubmission<A>>, + + /// Buffers the user has asked us to map, and which are not used by any + /// queue submission still in flight. + ready_to_map: Vec<Arc<Buffer<A>>>, + + /// Queue "on_submitted_work_done" closures that were initiated for while there is no + /// currently pending submissions. These cannot be immediately invoked as they + /// must happen _after_ all mapped buffer callbacks are mapped, so we defer them + /// here until the next time the device is maintained. + work_done_closures: SmallVec<[SubmittedWorkDoneClosure; 1]>, + + /// Closure to be called on "lose the device". This is invoked directly by + /// device.lose or by the UserCallbacks returned from maintain when the device + /// has been destroyed and its queues are empty. + pub device_lost_closure: Option<DeviceLostClosure>, +} + +impl<A: HalApi> LifetimeTracker<A> { + pub fn new() -> Self { + Self { + mapped: Vec::new(), + future_suspected_buffers: Vec::new(), + future_suspected_textures: Vec::new(), + suspected_resources: ResourceMaps::new(), + active: Vec::new(), + ready_to_map: Vec::new(), + work_done_closures: SmallVec::new(), + device_lost_closure: None, + } + } + + /// Return true if there are no queue submissions still in flight. + pub fn queue_empty(&self) -> bool { + self.active.is_empty() + } + + /// Start tracking resources associated with a new queue submission. + pub fn track_submission( + &mut self, + index: SubmissionIndex, + temp_resources: impl Iterator<Item = TempResource<A>>, + encoders: Vec<EncoderInFlight<A>>, + ) { + let mut last_resources = ResourceMaps::new(); + for res in temp_resources { + match res { + TempResource::Buffer(raw) => { + last_resources.buffers.insert(raw.as_info().id(), raw); + } + TempResource::StagingBuffer(raw) => { + last_resources + .staging_buffers + .insert(raw.as_info().id(), raw); + } + TempResource::DestroyedBuffer(destroyed) => { + last_resources + .destroyed_buffers + .insert(destroyed.id, destroyed); + } + TempResource::Texture(raw) => { + last_resources.textures.insert(raw.as_info().id(), raw); + } + TempResource::DestroyedTexture(destroyed) => { + last_resources + .destroyed_textures + .insert(destroyed.id, destroyed); + } + } + } + + self.active.push(ActiveSubmission { + index, + last_resources, + mapped: Vec::new(), + encoders, + work_done_closures: SmallVec::new(), + }); + } + + pub fn post_submit(&mut self) { + for v in self.future_suspected_buffers.drain(..).take(1) { + self.suspected_resources.buffers.insert(v.as_info().id(), v); + } + for v in self.future_suspected_textures.drain(..).take(1) { + self.suspected_resources + .textures + .insert(v.as_info().id(), v); + } + } + + pub(crate) fn map(&mut self, value: &Arc<Buffer<A>>) { + self.mapped.push(value.clone()); + } + + /// Sort out the consequences of completed submissions. + /// + /// Assume that all submissions up through `last_done` have completed. + /// + /// - Buffers used by those submissions are now ready to map, if + /// requested. Add any buffers in the submission's [`mapped`] list to + /// [`self.ready_to_map`], where [`LifetimeTracker::handle_mapping`] will find + /// them. + /// + /// - Resources whose final use was in those submissions are now ready to + /// free. Add any resources in the submission's [`last_resources`] table + /// to [`self.free_resources`], where [`LifetimeTracker::cleanup`] will find + /// them. + /// + /// Return a list of [`SubmittedWorkDoneClosure`]s to run. + /// + /// [`mapped`]: ActiveSubmission::mapped + /// [`self.ready_to_map`]: LifetimeTracker::ready_to_map + /// [`last_resources`]: ActiveSubmission::last_resources + /// [`self.free_resources`]: LifetimeTracker::free_resources + /// [`SubmittedWorkDoneClosure`]: crate::device::queue::SubmittedWorkDoneClosure + #[must_use] + pub fn triage_submissions( + &mut self, + last_done: SubmissionIndex, + command_allocator: &mut super::CommandAllocator<A>, + ) -> SmallVec<[SubmittedWorkDoneClosure; 1]> { + profiling::scope!("triage_submissions"); + + //TODO: enable when `is_sorted_by_key` is stable + //debug_assert!(self.active.is_sorted_by_key(|a| a.index)); + let done_count = self + .active + .iter() + .position(|a| a.index > last_done) + .unwrap_or(self.active.len()); + + let mut work_done_closures: SmallVec<_> = self.work_done_closures.drain(..).collect(); + for a in self.active.drain(..done_count) { + log::debug!("Active submission {} is done", a.index); + self.ready_to_map.extend(a.mapped); + for encoder in a.encoders { + let raw = unsafe { encoder.land() }; + command_allocator.release_encoder(raw); + } + work_done_closures.extend(a.work_done_closures); + } + work_done_closures + } + + pub fn schedule_resource_destruction( + &mut self, + temp_resource: TempResource<A>, + last_submit_index: SubmissionIndex, + ) { + let resources = self + .active + .iter_mut() + .find(|a| a.index == last_submit_index) + .map(|a| &mut a.last_resources); + if let Some(resources) = resources { + match temp_resource { + TempResource::Buffer(raw) => { + resources.buffers.insert(raw.as_info().id(), raw); + } + TempResource::StagingBuffer(raw) => { + resources.staging_buffers.insert(raw.as_info().id(), raw); + } + TempResource::DestroyedBuffer(destroyed) => { + resources.destroyed_buffers.insert(destroyed.id, destroyed); + } + TempResource::Texture(raw) => { + resources.textures.insert(raw.as_info().id(), raw); + } + TempResource::DestroyedTexture(destroyed) => { + resources.destroyed_textures.insert(destroyed.id, destroyed); + } + } + } + } + + pub fn add_work_done_closure(&mut self, closure: SubmittedWorkDoneClosure) { + match self.active.last_mut() { + Some(active) => { + active.work_done_closures.push(closure); + } + // We must defer the closure until all previously occurring map_async closures + // have fired. This is required by the spec. + None => { + self.work_done_closures.push(closure); + } + } + } +} + +impl<A: HalApi> LifetimeTracker<A> { + fn triage_resources<R>( + resources_map: &mut FastHashMap<Id<R::Marker>, Arc<R>>, + active: &mut [ActiveSubmission<A>], + trackers: &mut impl ResourceTracker<R>, + get_resource_map: impl Fn(&mut ResourceMaps<A>) -> &mut FastHashMap<Id<R::Marker>, Arc<R>>, + ) -> Vec<Arc<R>> + where + R: Resource, + { + let mut removed_resources = Vec::new(); + resources_map.retain(|&id, resource| { + let submit_index = resource.as_info().submission_index(); + let non_referenced_resources = active + .iter_mut() + .find(|a| a.index == submit_index) + .map(|a| &mut a.last_resources); + + let is_removed = trackers.remove_abandoned(id); + if is_removed { + removed_resources.push(resource.clone()); + if let Some(resources) = non_referenced_resources { + get_resource_map(resources).insert(id, resource.clone()); + } + } + !is_removed + }); + removed_resources + } + + fn triage_suspected_render_bundles(&mut self, trackers: &Mutex<Tracker<A>>) -> &mut Self { + let mut trackers = trackers.lock(); + let resource_map = &mut self.suspected_resources.render_bundles; + let mut removed_resources = Self::triage_resources( + resource_map, + self.active.as_mut_slice(), + &mut trackers.bundles, + |maps| &mut maps.render_bundles, + ); + removed_resources.drain(..).for_each(|bundle| { + for v in bundle.used.buffers.write().drain_resources() { + self.suspected_resources.buffers.insert(v.as_info().id(), v); + } + for v in bundle.used.textures.write().drain_resources() { + self.suspected_resources + .textures + .insert(v.as_info().id(), v); + } + for v in bundle.used.bind_groups.write().drain_resources() { + self.suspected_resources + .bind_groups + .insert(v.as_info().id(), v); + } + for v in bundle.used.render_pipelines.write().drain_resources() { + self.suspected_resources + .render_pipelines + .insert(v.as_info().id(), v); + } + for v in bundle.used.query_sets.write().drain_resources() { + self.suspected_resources + .query_sets + .insert(v.as_info().id(), v); + } + }); + self + } + + fn triage_suspected_bind_groups(&mut self, trackers: &Mutex<Tracker<A>>) -> &mut Self { + let mut trackers = trackers.lock(); + let resource_map = &mut self.suspected_resources.bind_groups; + let mut removed_resource = Self::triage_resources( + resource_map, + self.active.as_mut_slice(), + &mut trackers.bind_groups, + |maps| &mut maps.bind_groups, + ); + removed_resource.drain(..).for_each(|bind_group| { + for v in bind_group.used.buffers.drain_resources() { + self.suspected_resources.buffers.insert(v.as_info().id(), v); + } + for v in bind_group.used.textures.drain_resources() { + self.suspected_resources + .textures + .insert(v.as_info().id(), v); + } + for v in bind_group.used.views.drain_resources() { + self.suspected_resources + .texture_views + .insert(v.as_info().id(), v); + } + for v in bind_group.used.samplers.drain_resources() { + self.suspected_resources + .samplers + .insert(v.as_info().id(), v); + } + + self.suspected_resources + .bind_group_layouts + .insert(bind_group.layout.as_info().id(), bind_group.layout.clone()); + }); + self + } + + fn triage_suspected_texture_views(&mut self, trackers: &Mutex<Tracker<A>>) -> &mut Self { + let mut trackers = trackers.lock(); + let resource_map = &mut self.suspected_resources.texture_views; + Self::triage_resources( + resource_map, + self.active.as_mut_slice(), + &mut trackers.views, + |maps| &mut maps.texture_views, + ); + self + } + + fn triage_suspected_textures(&mut self, trackers: &Mutex<Tracker<A>>) -> &mut Self { + let mut trackers = trackers.lock(); + let resource_map = &mut self.suspected_resources.textures; + Self::triage_resources( + resource_map, + self.active.as_mut_slice(), + &mut trackers.textures, + |maps| &mut maps.textures, + ); + self + } + + fn triage_suspected_samplers(&mut self, trackers: &Mutex<Tracker<A>>) -> &mut Self { + let mut trackers = trackers.lock(); + let resource_map = &mut self.suspected_resources.samplers; + Self::triage_resources( + resource_map, + self.active.as_mut_slice(), + &mut trackers.samplers, + |maps| &mut maps.samplers, + ); + self + } + + fn triage_suspected_buffers(&mut self, trackers: &Mutex<Tracker<A>>) -> &mut Self { + let mut trackers = trackers.lock(); + let resource_map = &mut self.suspected_resources.buffers; + Self::triage_resources( + resource_map, + self.active.as_mut_slice(), + &mut trackers.buffers, + |maps| &mut maps.buffers, + ); + + self + } + + fn triage_suspected_destroyed_buffers(&mut self) { + for (id, buffer) in self.suspected_resources.destroyed_buffers.drain() { + let submit_index = buffer.submission_index; + if let Some(resources) = self.active.iter_mut().find(|a| a.index == submit_index) { + resources + .last_resources + .destroyed_buffers + .insert(id, buffer); + } + } + } + + fn triage_suspected_destroyed_textures(&mut self) { + for (id, texture) in self.suspected_resources.destroyed_textures.drain() { + let submit_index = texture.submission_index; + if let Some(resources) = self.active.iter_mut().find(|a| a.index == submit_index) { + resources + .last_resources + .destroyed_textures + .insert(id, texture); + } + } + } + + fn triage_suspected_compute_pipelines(&mut self, trackers: &Mutex<Tracker<A>>) -> &mut Self { + let mut trackers = trackers.lock(); + let resource_map = &mut self.suspected_resources.compute_pipelines; + let mut removed_resources = Self::triage_resources( + resource_map, + self.active.as_mut_slice(), + &mut trackers.compute_pipelines, + |maps| &mut maps.compute_pipelines, + ); + removed_resources.drain(..).for_each(|compute_pipeline| { + self.suspected_resources.pipeline_layouts.insert( + compute_pipeline.layout.as_info().id(), + compute_pipeline.layout.clone(), + ); + }); + self + } + + fn triage_suspected_render_pipelines(&mut self, trackers: &Mutex<Tracker<A>>) -> &mut Self { + let mut trackers = trackers.lock(); + let resource_map = &mut self.suspected_resources.render_pipelines; + let mut removed_resources = Self::triage_resources( + resource_map, + self.active.as_mut_slice(), + &mut trackers.render_pipelines, + |maps| &mut maps.render_pipelines, + ); + removed_resources.drain(..).for_each(|render_pipeline| { + self.suspected_resources.pipeline_layouts.insert( + render_pipeline.layout.as_info().id(), + render_pipeline.layout.clone(), + ); + }); + self + } + + fn triage_suspected_pipeline_layouts(&mut self) -> &mut Self { + let mut removed_resources = Vec::new(); + self.suspected_resources + .pipeline_layouts + .retain(|_pipeline_layout_id, pipeline_layout| { + removed_resources.push(pipeline_layout.clone()); + false + }); + removed_resources.drain(..).for_each(|pipeline_layout| { + for bgl in &pipeline_layout.bind_group_layouts { + self.suspected_resources + .bind_group_layouts + .insert(bgl.as_info().id(), bgl.clone()); + } + }); + self + } + + fn triage_suspected_bind_group_layouts(&mut self) -> &mut Self { + //Note: this has to happen after all the suspected pipelines are destroyed + //Note: nothing else can bump the refcount since the guard is locked exclusively + //Note: same BGL can appear multiple times in the list, but only the last + self.suspected_resources.bind_group_layouts.clear(); + + self + } + + fn triage_suspected_query_sets(&mut self, trackers: &Mutex<Tracker<A>>) -> &mut Self { + let mut trackers = trackers.lock(); + let resource_map = &mut self.suspected_resources.query_sets; + Self::triage_resources( + resource_map, + self.active.as_mut_slice(), + &mut trackers.query_sets, + |maps| &mut maps.query_sets, + ); + self + } + + fn triage_suspected_staging_buffers(&mut self) -> &mut Self { + self.suspected_resources.staging_buffers.clear(); + + self + } + + /// Identify resources to free, according to `trackers` and `self.suspected_resources`. + /// + /// Given `trackers`, the [`Tracker`] belonging to same [`Device`] as + /// `self`, and `hub`, the [`Hub`] to which that `Device` belongs: + /// + /// Remove from `trackers` each resource mentioned in + /// [`self.suspected_resources`]. If `trackers` held the final reference to + /// that resource, add it to the appropriate free list, to be destroyed by + /// the hal: + /// + /// - Add resources used by queue submissions still in flight to the + /// [`last_resources`] table of the last such submission's entry in + /// [`self.active`]. When that submission has finished execution. the + /// [`triage_submissions`] method will remove from the tracker and the + /// resource reference count will be responsible carrying out deallocation. + /// + /// ## Entrained resources + /// + /// This function finds resources that are used only by other resources + /// ready to be freed, and adds those to the free lists as well. For + /// example, if there's some texture `T` used only by some texture view + /// `TV`, then if `TV` can be freed, `T` gets added to the free lists too. + /// + /// Since `wgpu-core` resource ownership patterns are acyclic, we can visit + /// each type that can be owned after all types that could possibly own + /// it. This way, we can detect all free-able objects in a single pass, + /// simply by starting with types that are roots of the ownership DAG (like + /// render bundles) and working our way towards leaf types (like buffers). + /// + /// [`Device`]: super::Device + /// [`self.suspected_resources`]: LifetimeTracker::suspected_resources + /// [`last_resources`]: ActiveSubmission::last_resources + /// [`self.active`]: LifetimeTracker::active + /// [`triage_submissions`]: LifetimeTracker::triage_submissions + pub(crate) fn triage_suspected(&mut self, trackers: &Mutex<Tracker<A>>) { + profiling::scope!("triage_suspected"); + + //NOTE: the order is important to release resources that depends between each other! + self.triage_suspected_render_bundles(trackers); + self.triage_suspected_compute_pipelines(trackers); + self.triage_suspected_render_pipelines(trackers); + self.triage_suspected_bind_groups(trackers); + self.triage_suspected_pipeline_layouts(); + self.triage_suspected_bind_group_layouts(); + self.triage_suspected_query_sets(trackers); + self.triage_suspected_samplers(trackers); + self.triage_suspected_staging_buffers(); + self.triage_suspected_texture_views(trackers); + self.triage_suspected_textures(trackers); + self.triage_suspected_buffers(trackers); + self.triage_suspected_destroyed_buffers(); + self.triage_suspected_destroyed_textures(); + } + + /// Determine which buffers are ready to map, and which must wait for the + /// GPU. + /// + /// See the documentation for [`LifetimeTracker`] for details. + pub(crate) fn triage_mapped(&mut self) { + if self.mapped.is_empty() { + return; + } + + for buffer in self.mapped.drain(..) { + let submit_index = buffer.info.submission_index(); + log::trace!( + "Mapping of {:?} at submission {:?} gets assigned to active {:?}", + buffer.info.id(), + submit_index, + self.active.iter().position(|a| a.index == submit_index) + ); + + self.active + .iter_mut() + .find(|a| a.index == submit_index) + .map_or(&mut self.ready_to_map, |a| &mut a.mapped) + .push(buffer); + } + } + + /// Map the buffers in `self.ready_to_map`. + /// + /// Return a list of mapping notifications to send. + /// + /// See the documentation for [`LifetimeTracker`] for details. + #[must_use] + pub(crate) fn handle_mapping( + &mut self, + raw: &A::Device, + trackers: &Mutex<Tracker<A>>, + ) -> Vec<super::BufferMapPendingClosure> { + if self.ready_to_map.is_empty() { + return Vec::new(); + } + let mut pending_callbacks: Vec<super::BufferMapPendingClosure> = + Vec::with_capacity(self.ready_to_map.len()); + + for buffer in self.ready_to_map.drain(..) { + let buffer_id = buffer.info.id(); + let is_removed = { + let mut trackers = trackers.lock(); + trackers.buffers.remove_abandoned(buffer_id) + }; + if is_removed { + *buffer.map_state.lock() = resource::BufferMapState::Idle; + log::trace!("Buffer ready to map {:?} is not tracked anymore", buffer_id); + } else { + let mapping = match std::mem::replace( + &mut *buffer.map_state.lock(), + resource::BufferMapState::Idle, + ) { + resource::BufferMapState::Waiting(pending_mapping) => pending_mapping, + // Mapping cancelled + resource::BufferMapState::Idle => continue, + // Mapping queued at least twice by map -> unmap -> map + // and was already successfully mapped below + active @ resource::BufferMapState::Active { .. } => { + *buffer.map_state.lock() = active; + continue; + } + _ => panic!("No pending mapping."), + }; + let status = if mapping.range.start != mapping.range.end { + log::debug!("Buffer {:?} map state -> Active", buffer_id); + let host = mapping.op.host; + let size = mapping.range.end - mapping.range.start; + match super::map_buffer(raw, &buffer, mapping.range.start, size, host) { + Ok(ptr) => { + *buffer.map_state.lock() = resource::BufferMapState::Active { + ptr, + range: mapping.range.start..mapping.range.start + size, + host, + }; + Ok(()) + } + Err(e) => { + log::error!("Mapping failed: {e}"); + Err(e) + } + } + } else { + *buffer.map_state.lock() = resource::BufferMapState::Active { + ptr: std::ptr::NonNull::dangling(), + range: mapping.range, + host: mapping.op.host, + }; + Ok(()) + }; + pending_callbacks.push((mapping.op, status)); + } + } + pending_callbacks + } +} diff --git a/third_party/rust/wgpu-core/src/device/mod.rs b/third_party/rust/wgpu-core/src/device/mod.rs new file mode 100644 index 0000000000..7ecda830a3 --- /dev/null +++ b/third_party/rust/wgpu-core/src/device/mod.rs @@ -0,0 +1,480 @@ +use crate::{ + binding_model, + hal_api::HalApi, + hub::Hub, + id::{BindGroupLayoutId, PipelineLayoutId}, + resource::{Buffer, BufferAccessResult}, + resource::{BufferAccessError, BufferMapOperation}, + resource_log, Label, DOWNLEVEL_ERROR_MESSAGE, +}; + +use arrayvec::ArrayVec; +use hal::Device as _; +use smallvec::SmallVec; +use std::os::raw::c_char; +use thiserror::Error; +use wgt::{BufferAddress, DeviceLostReason, TextureFormat}; + +use std::{iter, num::NonZeroU32, ptr}; + +pub mod any_device; +pub(crate) mod bgl; +pub mod global; +mod life; +pub mod queue; +pub mod resource; +#[cfg(any(feature = "trace", feature = "replay"))] +pub mod trace; +pub use {life::WaitIdleError, resource::Device}; + +pub const SHADER_STAGE_COUNT: usize = hal::MAX_CONCURRENT_SHADER_STAGES; +// Should be large enough for the largest possible texture row. This +// value is enough for a 16k texture with float4 format. +pub(crate) const ZERO_BUFFER_SIZE: BufferAddress = 512 << 10; + +const CLEANUP_WAIT_MS: u32 = 5000; + +const IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL: &str = "Implicit BindGroupLayout in the Error State"; +const ENTRYPOINT_FAILURE_ERROR: &str = "The given EntryPoint is Invalid"; + +pub type DeviceDescriptor<'a> = wgt::DeviceDescriptor<Label<'a>>; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum HostMap { + Read, + Write, +} + +#[derive(Clone, Debug, Hash, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub(crate) struct AttachmentData<T> { + pub colors: ArrayVec<Option<T>, { hal::MAX_COLOR_ATTACHMENTS }>, + pub resolves: ArrayVec<T, { hal::MAX_COLOR_ATTACHMENTS }>, + pub depth_stencil: Option<T>, +} +impl<T: PartialEq> Eq for AttachmentData<T> {} +impl<T> AttachmentData<T> { + pub(crate) fn map<U, F: Fn(&T) -> U>(&self, fun: F) -> AttachmentData<U> { + AttachmentData { + colors: self.colors.iter().map(|c| c.as_ref().map(&fun)).collect(), + resolves: self.resolves.iter().map(&fun).collect(), + depth_stencil: self.depth_stencil.as_ref().map(&fun), + } + } +} + +#[derive(Debug, Copy, Clone)] +pub enum RenderPassCompatibilityCheckType { + RenderPipeline, + RenderBundle, +} + +#[derive(Clone, Debug, Hash, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub(crate) struct RenderPassContext { + pub attachments: AttachmentData<TextureFormat>, + pub sample_count: u32, + pub multiview: Option<NonZeroU32>, +} +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum RenderPassCompatibilityError { + #[error( + "Incompatible color attachments at indices {indices:?}: the RenderPass uses textures with formats {expected:?} but the {ty:?} uses attachments with formats {actual:?}", + )] + IncompatibleColorAttachment { + indices: Vec<usize>, + expected: Vec<Option<TextureFormat>>, + actual: Vec<Option<TextureFormat>>, + ty: RenderPassCompatibilityCheckType, + }, + #[error( + "Incompatible depth-stencil attachment format: the RenderPass uses a texture with format {expected:?} but the {ty:?} uses an attachment with format {actual:?}", + )] + IncompatibleDepthStencilAttachment { + expected: Option<TextureFormat>, + actual: Option<TextureFormat>, + ty: RenderPassCompatibilityCheckType, + }, + #[error( + "Incompatible sample count: the RenderPass uses textures with sample count {expected:?} but the {ty:?} uses attachments with format {actual:?}", + )] + IncompatibleSampleCount { + expected: u32, + actual: u32, + ty: RenderPassCompatibilityCheckType, + }, + #[error("Incompatible multiview setting: the RenderPass uses setting {expected:?} but the {ty:?} uses setting {actual:?}")] + IncompatibleMultiview { + expected: Option<NonZeroU32>, + actual: Option<NonZeroU32>, + ty: RenderPassCompatibilityCheckType, + }, +} + +impl RenderPassContext { + // Assumes the renderpass only contains one subpass + pub(crate) fn check_compatible( + &self, + other: &Self, + ty: RenderPassCompatibilityCheckType, + ) -> Result<(), RenderPassCompatibilityError> { + if self.attachments.colors != other.attachments.colors { + let indices = self + .attachments + .colors + .iter() + .zip(&other.attachments.colors) + .enumerate() + .filter_map(|(idx, (left, right))| (left != right).then_some(idx)) + .collect(); + return Err(RenderPassCompatibilityError::IncompatibleColorAttachment { + indices, + expected: self.attachments.colors.iter().cloned().collect(), + actual: other.attachments.colors.iter().cloned().collect(), + ty, + }); + } + if self.attachments.depth_stencil != other.attachments.depth_stencil { + return Err( + RenderPassCompatibilityError::IncompatibleDepthStencilAttachment { + expected: self.attachments.depth_stencil, + actual: other.attachments.depth_stencil, + ty, + }, + ); + } + if self.sample_count != other.sample_count { + return Err(RenderPassCompatibilityError::IncompatibleSampleCount { + expected: self.sample_count, + actual: other.sample_count, + ty, + }); + } + if self.multiview != other.multiview { + return Err(RenderPassCompatibilityError::IncompatibleMultiview { + expected: self.multiview, + actual: other.multiview, + ty, + }); + } + Ok(()) + } +} + +pub type BufferMapPendingClosure = (BufferMapOperation, BufferAccessResult); + +#[derive(Default)] +pub struct UserClosures { + pub mappings: Vec<BufferMapPendingClosure>, + pub submissions: SmallVec<[queue::SubmittedWorkDoneClosure; 1]>, + pub device_lost_invocations: SmallVec<[DeviceLostInvocation; 1]>, +} + +impl UserClosures { + fn extend(&mut self, other: Self) { + self.mappings.extend(other.mappings); + self.submissions.extend(other.submissions); + self.device_lost_invocations + .extend(other.device_lost_invocations); + } + + fn fire(self) { + // Note: this logic is specifically moved out of `handle_mapping()` in order to + // have nothing locked by the time we execute users callback code. + + // Mappings _must_ be fired before submissions, as the spec requires all mapping callbacks that are registered before + // a on_submitted_work_done callback to be fired before the on_submitted_work_done callback. + for (mut operation, status) in self.mappings { + if let Some(callback) = operation.callback.take() { + callback.call(status); + } + } + for closure in self.submissions { + closure.call(); + } + for invocation in self.device_lost_invocations { + invocation + .closure + .call(invocation.reason, invocation.message); + } + } +} + +#[cfg(send_sync)] +pub type DeviceLostCallback = Box<dyn Fn(DeviceLostReason, String) + Send + 'static>; +#[cfg(not(send_sync))] +pub type DeviceLostCallback = Box<dyn Fn(DeviceLostReason, String) + 'static>; + +pub struct DeviceLostClosureRust { + pub callback: DeviceLostCallback, + consumed: bool, +} + +impl Drop for DeviceLostClosureRust { + fn drop(&mut self) { + if !self.consumed { + panic!("DeviceLostClosureRust must be consumed before it is dropped."); + } + } +} + +#[repr(C)] +pub struct DeviceLostClosureC { + pub callback: unsafe extern "C" fn(user_data: *mut u8, reason: u8, message: *const c_char), + pub user_data: *mut u8, + consumed: bool, +} + +#[cfg(send_sync)] +unsafe impl Send for DeviceLostClosureC {} + +impl Drop for DeviceLostClosureC { + fn drop(&mut self) { + if !self.consumed { + panic!("DeviceLostClosureC must be consumed before it is dropped."); + } + } +} + +pub struct DeviceLostClosure { + // We wrap this so creating the enum in the C variant can be unsafe, + // allowing our call function to be safe. + inner: DeviceLostClosureInner, +} + +pub struct DeviceLostInvocation { + closure: DeviceLostClosure, + reason: DeviceLostReason, + message: String, +} + +enum DeviceLostClosureInner { + Rust { inner: DeviceLostClosureRust }, + C { inner: DeviceLostClosureC }, +} + +impl DeviceLostClosure { + pub fn from_rust(callback: DeviceLostCallback) -> Self { + let inner = DeviceLostClosureRust { + callback, + consumed: false, + }; + Self { + inner: DeviceLostClosureInner::Rust { inner }, + } + } + + /// # Safety + /// + /// - The callback pointer must be valid to call with the provided `user_data` + /// pointer. + /// + /// - Both pointers must point to `'static` data, as the callback may happen at + /// an unspecified time. + pub unsafe fn from_c(mut closure: DeviceLostClosureC) -> Self { + // Build an inner with the values from closure, ensuring that + // inner.consumed is false. + let inner = DeviceLostClosureC { + callback: closure.callback, + user_data: closure.user_data, + consumed: false, + }; + + // Mark the original closure as consumed, so we can safely drop it. + closure.consumed = true; + + Self { + inner: DeviceLostClosureInner::C { inner }, + } + } + + pub(crate) fn call(self, reason: DeviceLostReason, message: String) { + match self.inner { + DeviceLostClosureInner::Rust { mut inner } => { + inner.consumed = true; + + (inner.callback)(reason, message) + } + // SAFETY: the contract of the call to from_c says that this unsafe is sound. + DeviceLostClosureInner::C { mut inner } => unsafe { + inner.consumed = true; + + // Ensure message is structured as a null-terminated C string. It only + // needs to live as long as the callback invocation. + let message = std::ffi::CString::new(message).unwrap(); + (inner.callback)(inner.user_data, reason as u8, message.as_ptr()) + }, + } + } +} + +fn map_buffer<A: HalApi>( + raw: &A::Device, + buffer: &Buffer<A>, + offset: BufferAddress, + size: BufferAddress, + kind: HostMap, +) -> Result<ptr::NonNull<u8>, BufferAccessError> { + let snatch_guard = buffer.device.snatchable_lock.read(); + let raw_buffer = buffer + .raw(&snatch_guard) + .ok_or(BufferAccessError::Destroyed)?; + let mapping = unsafe { + raw.map_buffer(raw_buffer, offset..offset + size) + .map_err(DeviceError::from)? + }; + + *buffer.sync_mapped_writes.lock() = match kind { + HostMap::Read if !mapping.is_coherent => unsafe { + raw.invalidate_mapped_ranges(raw_buffer, iter::once(offset..offset + size)); + None + }, + HostMap::Write if !mapping.is_coherent => Some(offset..offset + size), + _ => None, + }; + + assert_eq!(offset % wgt::COPY_BUFFER_ALIGNMENT, 0); + assert_eq!(size % wgt::COPY_BUFFER_ALIGNMENT, 0); + // Zero out uninitialized parts of the mapping. (Spec dictates all resources + // behave as if they were initialized with zero) + // + // If this is a read mapping, ideally we would use a `clear_buffer` command + // before reading the data from GPU (i.e. `invalidate_range`). However, this + // would require us to kick off and wait for a command buffer or piggy back + // on an existing one (the later is likely the only worthwhile option). As + // reading uninitialized memory isn't a particular important path to + // support, we instead just initialize the memory here and make sure it is + // GPU visible, so this happens at max only once for every buffer region. + // + // If this is a write mapping zeroing out the memory here is the only + // reasonable way as all data is pushed to GPU anyways. + + // No need to flush if it is flushed later anyways. + let zero_init_needs_flush_now = + mapping.is_coherent && buffer.sync_mapped_writes.lock().is_none(); + let mapped = unsafe { std::slice::from_raw_parts_mut(mapping.ptr.as_ptr(), size as usize) }; + + for uninitialized in buffer + .initialization_status + .write() + .drain(offset..(size + offset)) + { + // The mapping's pointer is already offset, however we track the + // uninitialized range relative to the buffer's start. + let fill_range = + (uninitialized.start - offset) as usize..(uninitialized.end - offset) as usize; + mapped[fill_range].fill(0); + + if zero_init_needs_flush_now { + unsafe { raw.flush_mapped_ranges(raw_buffer, iter::once(uninitialized)) }; + } + } + + Ok(mapping.ptr) +} + +pub(crate) struct CommandAllocator<A: HalApi> { + free_encoders: Vec<A::CommandEncoder>, +} + +impl<A: HalApi> CommandAllocator<A> { + fn acquire_encoder( + &mut self, + device: &A::Device, + queue: &A::Queue, + ) -> Result<A::CommandEncoder, hal::DeviceError> { + match self.free_encoders.pop() { + Some(encoder) => Ok(encoder), + None => unsafe { + let hal_desc = hal::CommandEncoderDescriptor { label: None, queue }; + device.create_command_encoder(&hal_desc) + }, + } + } + + fn release_encoder(&mut self, encoder: A::CommandEncoder) { + self.free_encoders.push(encoder); + } + + fn dispose(self, device: &A::Device) { + resource_log!( + "CommandAllocator::dispose encoders {}", + self.free_encoders.len() + ); + for cmd_encoder in self.free_encoders { + unsafe { + device.destroy_command_encoder(cmd_encoder); + } + } + } +} + +#[derive(Clone, Debug, Error)] +#[error("Device is invalid")] +pub struct InvalidDevice; + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum DeviceError { + #[error("Parent device is invalid.")] + Invalid, + #[error("Parent device is lost")] + Lost, + #[error("Not enough memory left.")] + OutOfMemory, + #[error("Creation of a resource failed for a reason other than running out of memory.")] + ResourceCreationFailed, + #[error("QueueId is invalid")] + InvalidQueueId, + #[error("Attempt to use a resource with a different device from the one that created it")] + WrongDevice, +} + +impl From<hal::DeviceError> for DeviceError { + fn from(error: hal::DeviceError) -> Self { + match error { + hal::DeviceError::Lost => DeviceError::Lost, + hal::DeviceError::OutOfMemory => DeviceError::OutOfMemory, + hal::DeviceError::ResourceCreationFailed => DeviceError::ResourceCreationFailed, + } + } +} + +#[derive(Clone, Debug, Error)] +#[error("Features {0:?} are required but not enabled on the device")] +pub struct MissingFeatures(pub wgt::Features); + +#[derive(Clone, Debug, Error)] +#[error( + "Downlevel flags {0:?} are required but not supported on the device.\n{}", + DOWNLEVEL_ERROR_MESSAGE +)] +pub struct MissingDownlevelFlags(pub wgt::DownlevelFlags); + +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ImplicitPipelineContext { + pub root_id: PipelineLayoutId, + pub group_ids: ArrayVec<BindGroupLayoutId, { hal::MAX_BIND_GROUPS }>, +} + +pub struct ImplicitPipelineIds<'a> { + pub root_id: Option<PipelineLayoutId>, + pub group_ids: &'a [Option<BindGroupLayoutId>], +} + +impl ImplicitPipelineIds<'_> { + fn prepare<A: HalApi>(self, hub: &Hub<A>) -> ImplicitPipelineContext { + ImplicitPipelineContext { + root_id: hub.pipeline_layouts.prepare(self.root_id).into_id(), + group_ids: self + .group_ids + .iter() + .map(|id_in| hub.bind_group_layouts.prepare(*id_in).into_id()) + .collect(), + } + } +} diff --git a/third_party/rust/wgpu-core/src/device/queue.rs b/third_party/rust/wgpu-core/src/device/queue.rs new file mode 100644 index 0000000000..08c5b767b6 --- /dev/null +++ b/third_party/rust/wgpu-core/src/device/queue.rs @@ -0,0 +1,1569 @@ +#[cfg(feature = "trace")] +use crate::device::trace::Action; +use crate::{ + api_log, + command::{ + extract_texture_selector, validate_linear_texture_data, validate_texture_copy_range, + ClearError, CommandBuffer, CopySide, ImageCopyTexture, TransferError, + }, + conv, + device::{life::ResourceMaps, DeviceError, WaitIdleError}, + get_lowest_common_denom, + global::Global, + hal_api::HalApi, + hal_label, + id::{self, QueueId}, + init_tracker::{has_copy_partial_init_tracker_coverage, TextureInitRange}, + resource::{ + Buffer, BufferAccessError, BufferMapState, DestroyedBuffer, DestroyedTexture, Resource, + ResourceInfo, ResourceType, StagingBuffer, Texture, TextureInner, + }, + resource_log, track, FastHashMap, SubmissionIndex, +}; + +use hal::{CommandEncoder as _, Device as _, Queue as _}; +use parking_lot::Mutex; +use smallvec::SmallVec; + +use std::{ + iter, mem, ptr, + sync::{atomic::Ordering, Arc}, +}; +use thiserror::Error; + +use super::Device; + +pub struct Queue<A: HalApi> { + pub device: Option<Arc<Device<A>>>, + pub raw: Option<A::Queue>, + pub info: ResourceInfo<Queue<A>>, +} + +impl<A: HalApi> Resource for Queue<A> { + const TYPE: ResourceType = "Queue"; + + type Marker = crate::id::markers::Queue; + + fn as_info(&self) -> &ResourceInfo<Self> { + &self.info + } + + fn as_info_mut(&mut self) -> &mut ResourceInfo<Self> { + &mut self.info + } +} + +impl<A: HalApi> Drop for Queue<A> { + fn drop(&mut self) { + let queue = self.raw.take().unwrap(); + self.device.as_ref().unwrap().release_queue(queue); + } +} + +/// Number of command buffers that we generate from the same pool +/// for the write_xxx commands, before the pool is recycled. +/// +/// If we don't stop at some point, the pool will grow forever, +/// without a concrete moment of when it can be cleared. +const WRITE_COMMAND_BUFFERS_PER_POOL: usize = 64; + +#[repr(C)] +pub struct SubmittedWorkDoneClosureC { + pub callback: unsafe extern "C" fn(user_data: *mut u8), + pub user_data: *mut u8, +} + +#[cfg(send_sync)] +unsafe impl Send for SubmittedWorkDoneClosureC {} + +pub struct SubmittedWorkDoneClosure { + // We wrap this so creating the enum in the C variant can be unsafe, + // allowing our call function to be safe. + inner: SubmittedWorkDoneClosureInner, +} + +#[cfg(send_sync)] +type SubmittedWorkDoneCallback = Box<dyn FnOnce() + Send + 'static>; +#[cfg(not(send_sync))] +type SubmittedWorkDoneCallback = Box<dyn FnOnce() + 'static>; + +enum SubmittedWorkDoneClosureInner { + Rust { callback: SubmittedWorkDoneCallback }, + C { inner: SubmittedWorkDoneClosureC }, +} + +impl SubmittedWorkDoneClosure { + pub fn from_rust(callback: SubmittedWorkDoneCallback) -> Self { + Self { + inner: SubmittedWorkDoneClosureInner::Rust { callback }, + } + } + + /// # Safety + /// + /// - The callback pointer must be valid to call with the provided `user_data` + /// pointer. + /// + /// - Both pointers must point to `'static` data, as the callback may happen at + /// an unspecified time. + pub unsafe fn from_c(inner: SubmittedWorkDoneClosureC) -> Self { + Self { + inner: SubmittedWorkDoneClosureInner::C { inner }, + } + } + + pub(crate) fn call(self) { + match self.inner { + SubmittedWorkDoneClosureInner::Rust { callback } => callback(), + // SAFETY: the contract of the call to from_c says that this unsafe is sound. + SubmittedWorkDoneClosureInner::C { inner } => unsafe { + (inner.callback)(inner.user_data) + }, + } + } +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct WrappedSubmissionIndex { + pub queue_id: QueueId, + pub index: SubmissionIndex, +} + +/// A texture or buffer to be freed soon. +/// +/// This is just a tagged raw texture or buffer, generally about to be added to +/// some other more specific container like: +/// +/// - `PendingWrites::temp_resources`: resources used by queue writes and +/// unmaps, waiting to be folded in with the next queue submission +/// +/// - `ActiveSubmission::last_resources`: temporary resources used by a queue +/// submission, to be freed when it completes +/// +/// - `LifetimeTracker::free_resources`: resources to be freed in the next +/// `maintain` call, no longer used anywhere +#[derive(Debug)] +pub enum TempResource<A: HalApi> { + Buffer(Arc<Buffer<A>>), + StagingBuffer(Arc<StagingBuffer<A>>), + DestroyedBuffer(Arc<DestroyedBuffer<A>>), + DestroyedTexture(Arc<DestroyedTexture<A>>), + Texture(Arc<Texture<A>>), +} + +/// A queue execution for a particular command encoder. +pub(crate) struct EncoderInFlight<A: HalApi> { + raw: A::CommandEncoder, + cmd_buffers: Vec<A::CommandBuffer>, +} + +impl<A: HalApi> EncoderInFlight<A> { + pub(crate) unsafe fn land(mut self) -> A::CommandEncoder { + unsafe { self.raw.reset_all(self.cmd_buffers.into_iter()) }; + self.raw + } +} + +/// A private command encoder for writes made directly on the device +/// or queue. +/// +/// Operations like `buffer_unmap`, `queue_write_buffer`, and +/// `queue_write_texture` need to copy data to the GPU. At the hal +/// level, this must be done by encoding and submitting commands, but +/// these operations are not associated with any specific wgpu command +/// buffer. +/// +/// Instead, `Device::pending_writes` owns one of these values, which +/// has its own hal command encoder and resource lists. The commands +/// accumulated here are automatically submitted to the queue the next +/// time the user submits a wgpu command buffer, ahead of the user's +/// commands. +/// +/// Important: +/// When locking pending_writes be sure that tracker is not locked +/// and try to lock trackers for the minimum timespan possible +/// +/// All uses of [`StagingBuffer`]s end up here. +#[derive(Debug)] +pub(crate) struct PendingWrites<A: HalApi> { + pub command_encoder: A::CommandEncoder, + pub is_active: bool, + pub temp_resources: Vec<TempResource<A>>, + pub dst_buffers: FastHashMap<id::BufferId, Arc<Buffer<A>>>, + pub dst_textures: FastHashMap<id::TextureId, Arc<Texture<A>>>, + pub executing_command_buffers: Vec<A::CommandBuffer>, +} + +impl<A: HalApi> PendingWrites<A> { + pub fn new(command_encoder: A::CommandEncoder) -> Self { + Self { + command_encoder, + is_active: false, + temp_resources: Vec::new(), + dst_buffers: FastHashMap::default(), + dst_textures: FastHashMap::default(), + executing_command_buffers: Vec::new(), + } + } + + pub fn dispose(mut self, device: &A::Device) { + unsafe { + if self.is_active { + self.command_encoder.discard_encoding(); + } + self.command_encoder + .reset_all(self.executing_command_buffers.into_iter()); + device.destroy_command_encoder(self.command_encoder); + } + + self.temp_resources.clear(); + } + + pub fn consume_temp(&mut self, resource: TempResource<A>) { + self.temp_resources.push(resource); + } + + fn consume(&mut self, buffer: Arc<StagingBuffer<A>>) { + self.temp_resources + .push(TempResource::StagingBuffer(buffer)); + } + + fn pre_submit(&mut self) -> Result<Option<&A::CommandBuffer>, DeviceError> { + self.dst_buffers.clear(); + self.dst_textures.clear(); + if self.is_active { + let cmd_buf = unsafe { self.command_encoder.end_encoding()? }; + self.is_active = false; + self.executing_command_buffers.push(cmd_buf); + + return Ok(self.executing_command_buffers.last()); + } + + Ok(None) + } + + #[must_use] + fn post_submit( + &mut self, + command_allocator: &mut super::CommandAllocator<A>, + device: &A::Device, + queue: &A::Queue, + ) -> Option<EncoderInFlight<A>> { + if self.executing_command_buffers.len() >= WRITE_COMMAND_BUFFERS_PER_POOL { + let new_encoder = command_allocator.acquire_encoder(device, queue).unwrap(); + Some(EncoderInFlight { + raw: mem::replace(&mut self.command_encoder, new_encoder), + cmd_buffers: mem::take(&mut self.executing_command_buffers), + }) + } else { + None + } + } + + pub fn activate(&mut self) -> &mut A::CommandEncoder { + if !self.is_active { + unsafe { + self.command_encoder + .begin_encoding(Some("(wgpu internal) PendingWrites")) + .unwrap(); + } + self.is_active = true; + } + &mut self.command_encoder + } + + pub fn deactivate(&mut self) { + if self.is_active { + unsafe { + self.command_encoder.discard_encoding(); + } + self.is_active = false; + } + } +} + +fn prepare_staging_buffer<A: HalApi>( + device: &Arc<Device<A>>, + size: wgt::BufferAddress, + instance_flags: wgt::InstanceFlags, +) -> Result<(StagingBuffer<A>, *mut u8), DeviceError> { + profiling::scope!("prepare_staging_buffer"); + let stage_desc = hal::BufferDescriptor { + label: hal_label(Some("(wgpu internal) Staging"), instance_flags), + size, + usage: hal::BufferUses::MAP_WRITE | hal::BufferUses::COPY_SRC, + memory_flags: hal::MemoryFlags::TRANSIENT, + }; + + let buffer = unsafe { device.raw().create_buffer(&stage_desc)? }; + let mapping = unsafe { device.raw().map_buffer(&buffer, 0..size) }?; + + let staging_buffer = StagingBuffer { + raw: Mutex::new(Some(buffer)), + device: device.clone(), + size, + info: ResourceInfo::new("<StagingBuffer>"), + is_coherent: mapping.is_coherent, + }; + + Ok((staging_buffer, mapping.ptr.as_ptr())) +} + +impl<A: HalApi> StagingBuffer<A> { + unsafe fn flush(&self, device: &A::Device) -> Result<(), DeviceError> { + if !self.is_coherent { + unsafe { + device.flush_mapped_ranges( + self.raw.lock().as_ref().unwrap(), + iter::once(0..self.size), + ) + }; + } + unsafe { device.unmap_buffer(self.raw.lock().as_ref().unwrap())? }; + Ok(()) + } +} + +#[derive(Clone, Debug, Error)] +#[error("Queue is invalid")] +pub struct InvalidQueue; + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum QueueWriteError { + #[error(transparent)] + Queue(#[from] DeviceError), + #[error(transparent)] + Transfer(#[from] TransferError), + #[error(transparent)] + MemoryInitFailure(#[from] ClearError), +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum QueueSubmitError { + #[error(transparent)] + Queue(#[from] DeviceError), + #[error("Buffer {0:?} is destroyed")] + DestroyedBuffer(id::BufferId), + #[error("Texture {0:?} is destroyed")] + DestroyedTexture(id::TextureId), + #[error(transparent)] + Unmap(#[from] BufferAccessError), + #[error("Buffer {0:?} is still mapped")] + BufferStillMapped(id::BufferId), + #[error("Surface output was dropped before the command buffer got submitted")] + SurfaceOutputDropped, + #[error("Surface was unconfigured before the command buffer got submitted")] + SurfaceUnconfigured, + #[error("GPU got stuck :(")] + StuckGpu, +} + +//TODO: move out common parts of write_xxx. + +impl Global { + pub fn queue_write_buffer<A: HalApi>( + &self, + queue_id: QueueId, + buffer_id: id::BufferId, + buffer_offset: wgt::BufferAddress, + data: &[u8], + ) -> Result<(), QueueWriteError> { + profiling::scope!("Queue::write_buffer"); + api_log!("Queue::write_buffer {buffer_id:?} {}bytes", data.len()); + + let hub = A::hub(self); + + let queue = hub + .queues + .get(queue_id) + .map_err(|_| DeviceError::InvalidQueueId)?; + + let device = queue.device.as_ref().unwrap(); + + let data_size = data.len() as wgt::BufferAddress; + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + let data_path = trace.make_binary("bin", data); + trace.add(Action::WriteBuffer { + id: buffer_id, + data: data_path, + range: buffer_offset..buffer_offset + data_size, + queued: true, + }); + } + + if data_size == 0 { + log::trace!("Ignoring write_buffer of size 0"); + return Ok(()); + } + + // Platform validation requires that the staging buffer always be + // freed, even if an error occurs. All paths from here must call + // `device.pending_writes.consume`. + let (staging_buffer, staging_buffer_ptr) = + prepare_staging_buffer(device, data_size, device.instance_flags)?; + let mut pending_writes = device.pending_writes.lock(); + let pending_writes = pending_writes.as_mut().unwrap(); + + let stage_fid = hub.staging_buffers.request(); + let staging_buffer = stage_fid.init(staging_buffer); + + if let Err(flush_error) = unsafe { + profiling::scope!("copy"); + ptr::copy_nonoverlapping(data.as_ptr(), staging_buffer_ptr, data.len()); + staging_buffer.flush(device.raw()) + } { + pending_writes.consume(staging_buffer); + return Err(flush_error.into()); + } + + let result = self.queue_write_staging_buffer_impl( + device, + pending_writes, + &staging_buffer, + buffer_id, + buffer_offset, + ); + + pending_writes.consume(staging_buffer); + result + } + + pub fn queue_create_staging_buffer<A: HalApi>( + &self, + queue_id: QueueId, + buffer_size: wgt::BufferSize, + id_in: Option<id::StagingBufferId>, + ) -> Result<(id::StagingBufferId, *mut u8), QueueWriteError> { + profiling::scope!("Queue::create_staging_buffer"); + let hub = A::hub(self); + + let queue = hub + .queues + .get(queue_id) + .map_err(|_| DeviceError::InvalidQueueId)?; + + let device = queue.device.as_ref().unwrap(); + + let (staging_buffer, staging_buffer_ptr) = + prepare_staging_buffer(device, buffer_size.get(), device.instance_flags)?; + + let fid = hub.staging_buffers.prepare(id_in); + let (id, _) = fid.assign(staging_buffer); + resource_log!("Queue::create_staging_buffer {id:?}"); + + Ok((id, staging_buffer_ptr)) + } + + pub fn queue_write_staging_buffer<A: HalApi>( + &self, + queue_id: QueueId, + buffer_id: id::BufferId, + buffer_offset: wgt::BufferAddress, + staging_buffer_id: id::StagingBufferId, + ) -> Result<(), QueueWriteError> { + profiling::scope!("Queue::write_staging_buffer"); + let hub = A::hub(self); + + let queue = hub + .queues + .get(queue_id) + .map_err(|_| DeviceError::InvalidQueueId)?; + + let device = queue.device.as_ref().unwrap(); + + let staging_buffer = hub.staging_buffers.unregister(staging_buffer_id); + if staging_buffer.is_none() { + return Err(QueueWriteError::Transfer(TransferError::InvalidBuffer( + buffer_id, + ))); + } + let staging_buffer = staging_buffer.unwrap(); + let mut pending_writes = device.pending_writes.lock(); + let pending_writes = pending_writes.as_mut().unwrap(); + + // At this point, we have taken ownership of the staging_buffer from the + // user. Platform validation requires that the staging buffer always + // be freed, even if an error occurs. All paths from here must call + // `device.pending_writes.consume`. + if let Err(flush_error) = unsafe { staging_buffer.flush(device.raw()) } { + pending_writes.consume(staging_buffer); + return Err(flush_error.into()); + } + + let result = self.queue_write_staging_buffer_impl( + device, + pending_writes, + &staging_buffer, + buffer_id, + buffer_offset, + ); + + pending_writes.consume(staging_buffer); + result + } + + pub fn queue_validate_write_buffer<A: HalApi>( + &self, + _queue_id: QueueId, + buffer_id: id::BufferId, + buffer_offset: u64, + buffer_size: u64, + ) -> Result<(), QueueWriteError> { + profiling::scope!("Queue::validate_write_buffer"); + let hub = A::hub(self); + + let buffer = hub + .buffers + .get(buffer_id) + .map_err(|_| TransferError::InvalidBuffer(buffer_id))?; + + self.queue_validate_write_buffer_impl(&buffer, buffer_id, buffer_offset, buffer_size)?; + + Ok(()) + } + + fn queue_validate_write_buffer_impl<A: HalApi>( + &self, + buffer: &Buffer<A>, + buffer_id: id::BufferId, + buffer_offset: u64, + buffer_size: u64, + ) -> Result<(), TransferError> { + if !buffer.usage.contains(wgt::BufferUsages::COPY_DST) { + return Err(TransferError::MissingCopyDstUsageFlag( + Some(buffer_id), + None, + )); + } + if buffer_size % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err(TransferError::UnalignedCopySize(buffer_size)); + } + if buffer_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err(TransferError::UnalignedBufferOffset(buffer_offset)); + } + if buffer_offset + buffer_size > buffer.size { + return Err(TransferError::BufferOverrun { + start_offset: buffer_offset, + end_offset: buffer_offset + buffer_size, + buffer_size: buffer.size, + side: CopySide::Destination, + }); + } + + Ok(()) + } + + fn queue_write_staging_buffer_impl<A: HalApi>( + &self, + device: &Device<A>, + pending_writes: &mut PendingWrites<A>, + staging_buffer: &StagingBuffer<A>, + buffer_id: id::BufferId, + buffer_offset: u64, + ) -> Result<(), QueueWriteError> { + let hub = A::hub(self); + + let (dst, transition) = { + let buffer_guard = hub.buffers.read(); + let dst = buffer_guard + .get(buffer_id) + .map_err(|_| TransferError::InvalidBuffer(buffer_id))?; + let mut trackers = device.trackers.lock(); + trackers + .buffers + .set_single(dst, hal::BufferUses::COPY_DST) + .ok_or(TransferError::InvalidBuffer(buffer_id))? + }; + let snatch_guard = device.snatchable_lock.read(); + let dst_raw = dst + .raw + .get(&snatch_guard) + .ok_or(TransferError::InvalidBuffer(buffer_id))?; + + if dst.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + + let src_buffer_size = staging_buffer.size; + self.queue_validate_write_buffer_impl(&dst, buffer_id, buffer_offset, src_buffer_size)?; + + dst.info + .use_at(device.active_submission_index.load(Ordering::Relaxed) + 1); + + let region = wgt::BufferSize::new(src_buffer_size).map(|size| hal::BufferCopy { + src_offset: 0, + dst_offset: buffer_offset, + size, + }); + let inner_buffer = staging_buffer.raw.lock(); + let barriers = iter::once(hal::BufferBarrier { + buffer: inner_buffer.as_ref().unwrap(), + usage: hal::BufferUses::MAP_WRITE..hal::BufferUses::COPY_SRC, + }) + .chain(transition.map(|pending| pending.into_hal(&dst, &snatch_guard))); + let encoder = pending_writes.activate(); + unsafe { + encoder.transition_buffers(barriers); + encoder.copy_buffer_to_buffer( + inner_buffer.as_ref().unwrap(), + dst_raw, + region.into_iter(), + ); + } + let dst = hub.buffers.get(buffer_id).unwrap(); + pending_writes.dst_buffers.insert(buffer_id, dst.clone()); + + // Ensure the overwritten bytes are marked as initialized so + // they don't need to be nulled prior to mapping or binding. + { + dst.initialization_status + .write() + .drain(buffer_offset..(buffer_offset + src_buffer_size)); + } + + Ok(()) + } + + pub fn queue_write_texture<A: HalApi>( + &self, + queue_id: QueueId, + destination: &ImageCopyTexture, + data: &[u8], + data_layout: &wgt::ImageDataLayout, + size: &wgt::Extent3d, + ) -> Result<(), QueueWriteError> { + profiling::scope!("Queue::write_texture"); + api_log!("Queue::write_texture {:?} {size:?}", destination.texture); + + let hub = A::hub(self); + + let queue = hub + .queues + .get(queue_id) + .map_err(|_| DeviceError::InvalidQueueId)?; + + let device = queue.device.as_ref().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + let data_path = trace.make_binary("bin", data); + trace.add(Action::WriteTexture { + to: *destination, + data: data_path, + layout: *data_layout, + size: *size, + }); + } + + if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 { + log::trace!("Ignoring write_texture of size 0"); + return Ok(()); + } + + let dst = hub + .textures + .get(destination.texture) + .map_err(|_| TransferError::InvalidTexture(destination.texture))?; + + if dst.device.as_info().id() != queue_id.transmute() { + return Err(DeviceError::WrongDevice.into()); + } + + if !dst.desc.usage.contains(wgt::TextureUsages::COPY_DST) { + return Err( + TransferError::MissingCopyDstUsageFlag(None, Some(destination.texture)).into(), + ); + } + + // Note: Doing the copy range validation early is important because ensures that the + // dimensions are not going to cause overflow in other parts of the validation. + let (hal_copy_size, array_layer_count) = + validate_texture_copy_range(destination, &dst.desc, CopySide::Destination, size)?; + + let (selector, dst_base) = extract_texture_selector(destination, size, &dst)?; + + if !dst_base.aspect.is_one() { + return Err(TransferError::CopyAspectNotOne.into()); + } + + if !conv::is_valid_copy_dst_texture_format(dst.desc.format, destination.aspect) { + return Err(TransferError::CopyToForbiddenTextureFormat { + format: dst.desc.format, + aspect: destination.aspect, + } + .into()); + } + + // Note: `_source_bytes_per_array_layer` is ignored since we + // have a staging copy, and it can have a different value. + let (_, _source_bytes_per_array_layer) = validate_linear_texture_data( + data_layout, + dst.desc.format, + destination.aspect, + data.len() as wgt::BufferAddress, + CopySide::Source, + size, + false, + )?; + + if dst.desc.format.is_depth_stencil_format() { + device + .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES) + .map_err(TransferError::from)?; + } + + let (block_width, block_height) = dst.desc.format.block_dimensions(); + let width_blocks = size.width / block_width; + let height_blocks = size.height / block_height; + + let block_rows_per_image = data_layout.rows_per_image.unwrap_or( + // doesn't really matter because we need this only if we copy + // more than one layer, and then we validate for this being not + // None + height_blocks, + ); + + let block_size = dst + .desc + .format + .block_copy_size(Some(destination.aspect)) + .unwrap(); + let bytes_per_row_alignment = + get_lowest_common_denom(device.alignments.buffer_copy_pitch.get() as u32, block_size); + let stage_bytes_per_row = + wgt::math::align_to(block_size * width_blocks, bytes_per_row_alignment); + + let block_rows_in_copy = + (size.depth_or_array_layers - 1) * block_rows_per_image + height_blocks; + let stage_size = stage_bytes_per_row as u64 * block_rows_in_copy as u64; + + let mut pending_writes = device.pending_writes.lock(); + let pending_writes = pending_writes.as_mut().unwrap(); + let encoder = pending_writes.activate(); + + // If the copy does not fully cover the layers, we need to initialize to + // zero *first* as we don't keep track of partial texture layer inits. + // + // Strictly speaking we only need to clear the areas of a layer + // untouched, but this would get increasingly messy. + let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 { + // volume textures don't have a layer range as array volumes aren't supported + 0..1 + } else { + destination.origin.z..destination.origin.z + size.depth_or_array_layers + }; + let mut dst_initialization_status = dst.initialization_status.write(); + if dst_initialization_status.mips[destination.mip_level as usize] + .check(init_layer_range.clone()) + .is_some() + { + if has_copy_partial_init_tracker_coverage(size, destination.mip_level, &dst.desc) { + for layer_range in dst_initialization_status.mips[destination.mip_level as usize] + .drain(init_layer_range) + .collect::<Vec<std::ops::Range<u32>>>() + { + let mut trackers = device.trackers.lock(); + crate::command::clear_texture( + &dst, + TextureInitRange { + mip_range: destination.mip_level..(destination.mip_level + 1), + layer_range, + }, + encoder, + &mut trackers.textures, + &device.alignments, + device.zero_buffer.as_ref().unwrap(), + ) + .map_err(QueueWriteError::from)?; + } + } else { + dst_initialization_status.mips[destination.mip_level as usize] + .drain(init_layer_range); + } + } + + let snatch_guard = device.snatchable_lock.read(); + + // Re-get `dst` immutably here, so that the mutable borrow of the + // `texture_guard.get` above ends in time for the `clear_texture` + // call above. Since we've held `texture_guard` the whole time, we know + // the texture hasn't gone away in the mean time, so we can unwrap. + let dst = hub.textures.get(destination.texture).unwrap(); + dst.info + .use_at(device.active_submission_index.load(Ordering::Relaxed) + 1); + + let dst_raw = dst + .raw(&snatch_guard) + .ok_or(TransferError::InvalidTexture(destination.texture))?; + + let bytes_per_row = data_layout + .bytes_per_row + .unwrap_or(width_blocks * block_size); + + // Platform validation requires that the staging buffer always be + // freed, even if an error occurs. All paths from here must call + // `device.pending_writes.consume`. + let (staging_buffer, staging_buffer_ptr) = + prepare_staging_buffer(device, stage_size, device.instance_flags)?; + + let stage_fid = hub.staging_buffers.request(); + let staging_buffer = stage_fid.init(staging_buffer); + + if stage_bytes_per_row == bytes_per_row { + profiling::scope!("copy aligned"); + // Fast path if the data is already being aligned optimally. + unsafe { + ptr::copy_nonoverlapping( + data.as_ptr().offset(data_layout.offset as isize), + staging_buffer_ptr, + stage_size as usize, + ); + } + } else { + profiling::scope!("copy chunked"); + // Copy row by row into the optimal alignment. + let copy_bytes_per_row = stage_bytes_per_row.min(bytes_per_row) as usize; + for layer in 0..size.depth_or_array_layers { + let rows_offset = layer * block_rows_per_image; + for row in 0..height_blocks { + unsafe { + ptr::copy_nonoverlapping( + data.as_ptr().offset( + data_layout.offset as isize + + (rows_offset + row) as isize * bytes_per_row as isize, + ), + staging_buffer_ptr.offset( + (rows_offset + row) as isize * stage_bytes_per_row as isize, + ), + copy_bytes_per_row, + ); + } + } + } + } + + if let Err(e) = unsafe { staging_buffer.flush(device.raw()) } { + pending_writes.consume(staging_buffer); + return Err(e.into()); + } + + let regions = (0..array_layer_count).map(|rel_array_layer| { + let mut texture_base = dst_base.clone(); + texture_base.array_layer += rel_array_layer; + hal::BufferTextureCopy { + buffer_layout: wgt::ImageDataLayout { + offset: rel_array_layer as u64 + * block_rows_per_image as u64 + * stage_bytes_per_row as u64, + bytes_per_row: Some(stage_bytes_per_row), + rows_per_image: Some(block_rows_per_image), + }, + texture_base, + size: hal_copy_size, + } + }); + + { + let inner_buffer = staging_buffer.raw.lock(); + let barrier = hal::BufferBarrier { + buffer: inner_buffer.as_ref().unwrap(), + usage: hal::BufferUses::MAP_WRITE..hal::BufferUses::COPY_SRC, + }; + + let mut trackers = device.trackers.lock(); + let transition = trackers + .textures + .set_single(&dst, selector, hal::TextureUses::COPY_DST) + .ok_or(TransferError::InvalidTexture(destination.texture))?; + unsafe { + encoder.transition_textures(transition.map(|pending| pending.into_hal(dst_raw))); + encoder.transition_buffers(iter::once(barrier)); + encoder.copy_buffer_to_texture(inner_buffer.as_ref().unwrap(), dst_raw, regions); + } + } + + pending_writes.consume(staging_buffer); + pending_writes + .dst_textures + .insert(destination.texture, dst.clone()); + + Ok(()) + } + + #[cfg(webgl)] + pub fn queue_copy_external_image_to_texture<A: HalApi>( + &self, + queue_id: QueueId, + source: &wgt::ImageCopyExternalImage, + destination: crate::command::ImageCopyTextureTagged, + size: wgt::Extent3d, + ) -> Result<(), QueueWriteError> { + profiling::scope!("Queue::copy_external_image_to_texture"); + + let hub = A::hub(self); + + let queue = hub + .queues + .get(queue_id) + .map_err(|_| DeviceError::InvalidQueueId)?; + + let device = queue.device.as_ref().unwrap(); + + if size.width == 0 || size.height == 0 || size.depth_or_array_layers == 0 { + log::trace!("Ignoring write_texture of size 0"); + return Ok(()); + } + + let mut needs_flag = false; + needs_flag |= matches!(source.source, wgt::ExternalImageSource::OffscreenCanvas(_)); + needs_flag |= source.origin != wgt::Origin2d::ZERO; + needs_flag |= destination.color_space != wgt::PredefinedColorSpace::Srgb; + #[allow(clippy::bool_comparison)] + if matches!(source.source, wgt::ExternalImageSource::ImageBitmap(_)) { + needs_flag |= source.flip_y != false; + needs_flag |= destination.premultiplied_alpha != false; + } + + if needs_flag { + device + .require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES) + .map_err(TransferError::from)?; + } + + let src_width = source.source.width(); + let src_height = source.source.height(); + + let dst = hub.textures.get(destination.texture).unwrap(); + + if !conv::is_valid_external_image_copy_dst_texture_format(dst.desc.format) { + return Err( + TransferError::ExternalCopyToForbiddenTextureFormat(dst.desc.format).into(), + ); + } + if dst.desc.dimension != wgt::TextureDimension::D2 { + return Err(TransferError::InvalidDimensionExternal(destination.texture).into()); + } + if !dst.desc.usage.contains(wgt::TextureUsages::COPY_DST) { + return Err( + TransferError::MissingCopyDstUsageFlag(None, Some(destination.texture)).into(), + ); + } + if !dst + .desc + .usage + .contains(wgt::TextureUsages::RENDER_ATTACHMENT) + { + return Err( + TransferError::MissingRenderAttachmentUsageFlag(destination.texture).into(), + ); + } + if dst.desc.sample_count != 1 { + return Err(TransferError::InvalidSampleCount { + sample_count: dst.desc.sample_count, + } + .into()); + } + + if source.origin.x + size.width > src_width { + return Err(TransferError::TextureOverrun { + start_offset: source.origin.x, + end_offset: source.origin.x + size.width, + texture_size: src_width, + dimension: crate::resource::TextureErrorDimension::X, + side: CopySide::Source, + } + .into()); + } + if source.origin.y + size.height > src_height { + return Err(TransferError::TextureOverrun { + start_offset: source.origin.y, + end_offset: source.origin.y + size.height, + texture_size: src_height, + dimension: crate::resource::TextureErrorDimension::Y, + side: CopySide::Source, + } + .into()); + } + if size.depth_or_array_layers != 1 { + return Err(TransferError::TextureOverrun { + start_offset: 0, + end_offset: size.depth_or_array_layers, + texture_size: 1, + dimension: crate::resource::TextureErrorDimension::Z, + side: CopySide::Source, + } + .into()); + } + + // Note: Doing the copy range validation early is important because ensures that the + // dimensions are not going to cause overflow in other parts of the validation. + let (hal_copy_size, _) = validate_texture_copy_range( + &destination.to_untagged(), + &dst.desc, + CopySide::Destination, + &size, + )?; + + let (selector, dst_base) = + extract_texture_selector(&destination.to_untagged(), &size, &dst)?; + + let mut pending_writes = device.pending_writes.lock(); + let encoder = pending_writes.as_mut().unwrap().activate(); + + // If the copy does not fully cover the layers, we need to initialize to + // zero *first* as we don't keep track of partial texture layer inits. + // + // Strictly speaking we only need to clear the areas of a layer + // untouched, but this would get increasingly messy. + let init_layer_range = if dst.desc.dimension == wgt::TextureDimension::D3 { + // volume textures don't have a layer range as array volumes aren't supported + 0..1 + } else { + destination.origin.z..destination.origin.z + size.depth_or_array_layers + }; + let mut dst_initialization_status = dst.initialization_status.write(); + if dst_initialization_status.mips[destination.mip_level as usize] + .check(init_layer_range.clone()) + .is_some() + { + if has_copy_partial_init_tracker_coverage(&size, destination.mip_level, &dst.desc) { + for layer_range in dst_initialization_status.mips[destination.mip_level as usize] + .drain(init_layer_range) + .collect::<Vec<std::ops::Range<u32>>>() + { + let mut trackers = device.trackers.lock(); + crate::command::clear_texture( + &dst, + TextureInitRange { + mip_range: destination.mip_level..(destination.mip_level + 1), + layer_range, + }, + encoder, + &mut trackers.textures, + &device.alignments, + device.zero_buffer.as_ref().unwrap(), + ) + .map_err(QueueWriteError::from)?; + } + } else { + dst_initialization_status.mips[destination.mip_level as usize] + .drain(init_layer_range); + } + } + dst.info + .use_at(device.active_submission_index.load(Ordering::Relaxed) + 1); + + let snatch_guard = device.snatchable_lock.read(); + let dst_raw = dst + .raw(&snatch_guard) + .ok_or(TransferError::InvalidTexture(destination.texture))?; + + let regions = hal::TextureCopy { + src_base: hal::TextureCopyBase { + mip_level: 0, + array_layer: 0, + origin: source.origin.to_3d(0), + aspect: hal::FormatAspects::COLOR, + }, + dst_base, + size: hal_copy_size, + }; + + unsafe { + let mut trackers = device.trackers.lock(); + let transitions = trackers + .textures + .set_single(&dst, selector, hal::TextureUses::COPY_DST) + .ok_or(TransferError::InvalidTexture(destination.texture))?; + encoder.transition_textures(transitions.map(|pending| pending.into_hal(dst_raw))); + encoder.copy_external_image_to_texture( + source, + dst_raw, + destination.premultiplied_alpha, + iter::once(regions), + ); + } + + Ok(()) + } + + pub fn queue_submit<A: HalApi>( + &self, + queue_id: QueueId, + command_buffer_ids: &[id::CommandBufferId], + ) -> Result<WrappedSubmissionIndex, QueueSubmitError> { + profiling::scope!("Queue::submit"); + api_log!("Queue::submit {queue_id:?}"); + + let (submit_index, callbacks) = { + let hub = A::hub(self); + + let queue = hub + .queues + .get(queue_id) + .map_err(|_| DeviceError::InvalidQueueId)?; + + let device = queue.device.as_ref().unwrap(); + + let mut fence = device.fence.write(); + let fence = fence.as_mut().unwrap(); + let submit_index = device + .active_submission_index + .fetch_add(1, Ordering::Relaxed) + + 1; + let mut active_executions = Vec::new(); + + let mut used_surface_textures = track::TextureUsageScope::new(); + + let snatch_guard = device.snatchable_lock.read(); + + let mut submit_surface_textures_owned = SmallVec::<[_; 2]>::new(); + + { + let mut command_buffer_guard = hub.command_buffers.write(); + + if !command_buffer_ids.is_empty() { + profiling::scope!("prepare"); + + //TODO: if multiple command buffers are submitted, we can re-use the last + // native command buffer of the previous chain instead of always creating + // a temporary one, since the chains are not finished. + let mut temp_suspected = device.temp_suspected.lock(); + { + let mut suspected = temp_suspected.replace(ResourceMaps::new()).unwrap(); + suspected.clear(); + } + + // finish all the command buffers first + for &cmb_id in command_buffer_ids { + // we reset the used surface textures every time we use + // it, so make sure to set_size on it. + used_surface_textures.set_size(hub.textures.read().len()); + + #[allow(unused_mut)] + let mut cmdbuf = match command_buffer_guard.replace_with_error(cmb_id) { + Ok(cmdbuf) => cmdbuf, + Err(_) => continue, + }; + + if cmdbuf.device.as_info().id() != queue_id.transmute() { + return Err(DeviceError::WrongDevice.into()); + } + + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + trace.add(Action::Submit( + submit_index, + cmdbuf + .data + .lock() + .as_mut() + .unwrap() + .commands + .take() + .unwrap(), + )); + } + if !cmdbuf.is_finished() { + if let Some(cmdbuf) = Arc::into_inner(cmdbuf) { + device.destroy_command_buffer(cmdbuf); + } else { + panic!( + "Command buffer cannot be destroyed because is still in use" + ); + } + continue; + } + + // optimize the tracked states + // cmdbuf.trackers.optimize(); + { + let cmd_buf_data = cmdbuf.data.lock(); + let cmd_buf_trackers = &cmd_buf_data.as_ref().unwrap().trackers; + + // update submission IDs + for buffer in cmd_buf_trackers.buffers.used_resources() { + let id = buffer.info.id(); + let raw_buf = match buffer.raw.get(&snatch_guard) { + Some(raw) => raw, + None => { + return Err(QueueSubmitError::DestroyedBuffer(id)); + } + }; + buffer.info.use_at(submit_index); + if buffer.is_unique() { + if let BufferMapState::Active { .. } = *buffer.map_state.lock() + { + log::warn!("Dropped buffer has a pending mapping."); + unsafe { device.raw().unmap_buffer(raw_buf) } + .map_err(DeviceError::from)?; + } + temp_suspected + .as_mut() + .unwrap() + .buffers + .insert(id, buffer.clone()); + } else { + match *buffer.map_state.lock() { + BufferMapState::Idle => (), + _ => return Err(QueueSubmitError::BufferStillMapped(id)), + } + } + } + for texture in cmd_buf_trackers.textures.used_resources() { + let id = texture.info.id(); + let should_extend = match texture.inner.get(&snatch_guard) { + None => { + return Err(QueueSubmitError::DestroyedTexture(id)); + } + Some(TextureInner::Native { .. }) => false, + Some(TextureInner::Surface { + ref has_work, + ref raw, + .. + }) => { + has_work.store(true, Ordering::Relaxed); + + if raw.is_some() { + submit_surface_textures_owned.push(texture.clone()); + } + + true + } + }; + texture.info.use_at(submit_index); + if texture.is_unique() { + temp_suspected + .as_mut() + .unwrap() + .textures + .insert(id, texture.clone()); + } + if should_extend { + unsafe { + used_surface_textures + .merge_single(&texture, None, hal::TextureUses::PRESENT) + .unwrap(); + }; + } + } + for texture_view in cmd_buf_trackers.views.used_resources() { + texture_view.info.use_at(submit_index); + if texture_view.is_unique() { + temp_suspected + .as_mut() + .unwrap() + .texture_views + .insert(texture_view.as_info().id(), texture_view.clone()); + } + } + { + for bg in cmd_buf_trackers.bind_groups.used_resources() { + bg.info.use_at(submit_index); + // We need to update the submission indices for the contained + // state-less (!) resources as well, so that they don't get + // deleted too early if the parent bind group goes out of scope. + for view in bg.used.views.used_resources() { + view.info.use_at(submit_index); + } + for sampler in bg.used.samplers.used_resources() { + sampler.info.use_at(submit_index); + } + if bg.is_unique() { + temp_suspected + .as_mut() + .unwrap() + .bind_groups + .insert(bg.as_info().id(), bg.clone()); + } + } + } + // assert!(cmd_buf_trackers.samplers.is_empty()); + for compute_pipeline in + cmd_buf_trackers.compute_pipelines.used_resources() + { + compute_pipeline.info.use_at(submit_index); + if compute_pipeline.is_unique() { + temp_suspected.as_mut().unwrap().compute_pipelines.insert( + compute_pipeline.as_info().id(), + compute_pipeline.clone(), + ); + } + } + for render_pipeline in + cmd_buf_trackers.render_pipelines.used_resources() + { + render_pipeline.info.use_at(submit_index); + if render_pipeline.is_unique() { + temp_suspected.as_mut().unwrap().render_pipelines.insert( + render_pipeline.as_info().id(), + render_pipeline.clone(), + ); + } + } + for query_set in cmd_buf_trackers.query_sets.used_resources() { + query_set.info.use_at(submit_index); + if query_set.is_unique() { + temp_suspected + .as_mut() + .unwrap() + .query_sets + .insert(query_set.as_info().id(), query_set.clone()); + } + } + for bundle in cmd_buf_trackers.bundles.used_resources() { + bundle.info.use_at(submit_index); + // We need to update the submission indices for the contained + // state-less (!) resources as well, excluding the bind groups. + // They don't get deleted too early if the bundle goes out of scope. + for render_pipeline in + bundle.used.render_pipelines.read().used_resources() + { + render_pipeline.info.use_at(submit_index); + } + for query_set in bundle.used.query_sets.read().used_resources() { + query_set.info.use_at(submit_index); + } + if bundle.is_unique() { + temp_suspected + .as_mut() + .unwrap() + .render_bundles + .insert(bundle.as_info().id(), bundle.clone()); + } + } + } + let mut baked = cmdbuf.from_arc_into_baked(); + // execute resource transitions + unsafe { + baked + .encoder + .begin_encoding(hal_label( + Some("(wgpu internal) Transit"), + device.instance_flags, + )) + .map_err(DeviceError::from)? + }; + log::trace!("Stitching command buffer {:?} before submission", cmb_id); + + //Note: locking the trackers has to be done after the storages + let mut trackers = device.trackers.lock(); + baked + .initialize_buffer_memory(&mut *trackers) + .map_err(|err| QueueSubmitError::DestroyedBuffer(err.0))?; + baked + .initialize_texture_memory(&mut *trackers, device) + .map_err(|err| QueueSubmitError::DestroyedTexture(err.0))?; + //Note: stateless trackers are not merged: + // device already knows these resources exist. + CommandBuffer::insert_barriers_from_tracker( + &mut baked.encoder, + &mut *trackers, + &baked.trackers, + &snatch_guard, + ); + + let transit = unsafe { baked.encoder.end_encoding().unwrap() }; + baked.list.insert(0, transit); + + // Transition surface textures into `Present` state. + // Note: we could technically do it after all of the command buffers, + // but here we have a command encoder by hand, so it's easier to use it. + if !used_surface_textures.is_empty() { + unsafe { + baked + .encoder + .begin_encoding(hal_label( + Some("(wgpu internal) Present"), + device.instance_flags, + )) + .map_err(DeviceError::from)? + }; + trackers + .textures + .set_from_usage_scope(&used_surface_textures); + let (transitions, textures) = + trackers.textures.drain_transitions(&snatch_guard); + let texture_barriers = transitions + .into_iter() + .enumerate() + .map(|(i, p)| p.into_hal(textures[i].unwrap().raw().unwrap())); + let present = unsafe { + baked.encoder.transition_textures(texture_barriers); + baked.encoder.end_encoding().unwrap() + }; + baked.list.push(present); + used_surface_textures = track::TextureUsageScope::new(); + } + + // done + active_executions.push(EncoderInFlight { + raw: baked.encoder, + cmd_buffers: baked.list, + }); + } + + log::trace!("Device after submission {}", submit_index); + } + } + + let mut pending_writes = device.pending_writes.lock(); + let pending_writes = pending_writes.as_mut().unwrap(); + + { + used_surface_textures.set_size(hub.textures.read().len()); + for (&id, texture) in pending_writes.dst_textures.iter() { + match texture.inner.get(&snatch_guard) { + None => { + return Err(QueueSubmitError::DestroyedTexture(id)); + } + Some(TextureInner::Native { .. }) => {} + Some(TextureInner::Surface { + ref has_work, + ref raw, + .. + }) => { + has_work.store(true, Ordering::Relaxed); + + if raw.is_some() { + submit_surface_textures_owned.push(texture.clone()); + } + + unsafe { + used_surface_textures + .merge_single(texture, None, hal::TextureUses::PRESENT) + .unwrap() + }; + } + } + } + + if !used_surface_textures.is_empty() { + let mut trackers = device.trackers.lock(); + + trackers + .textures + .set_from_usage_scope(&used_surface_textures); + let (transitions, textures) = + trackers.textures.drain_transitions(&snatch_guard); + let texture_barriers = transitions + .into_iter() + .enumerate() + .map(|(i, p)| p.into_hal(textures[i].unwrap().raw().unwrap())); + unsafe { + pending_writes + .command_encoder + .transition_textures(texture_barriers); + }; + } + } + + let refs = pending_writes + .pre_submit()? + .into_iter() + .chain( + active_executions + .iter() + .flat_map(|pool_execution| pool_execution.cmd_buffers.iter()), + ) + .collect::<Vec<_>>(); + + let mut submit_surface_textures = + SmallVec::<[_; 2]>::with_capacity(submit_surface_textures_owned.len()); + + for texture in &submit_surface_textures_owned { + submit_surface_textures.extend(match texture.inner.get(&snatch_guard) { + Some(TextureInner::Surface { raw, .. }) => raw.as_ref(), + _ => None, + }); + } + + unsafe { + queue + .raw + .as_ref() + .unwrap() + .submit(&refs, &submit_surface_textures, Some((fence, submit_index))) + .map_err(DeviceError::from)?; + } + + profiling::scope!("cleanup"); + if let Some(pending_execution) = pending_writes.post_submit( + device.command_allocator.lock().as_mut().unwrap(), + device.raw(), + queue.raw.as_ref().unwrap(), + ) { + active_executions.push(pending_execution); + } + + // this will register the new submission to the life time tracker + let mut pending_write_resources = mem::take(&mut pending_writes.temp_resources); + device.lock_life().track_submission( + submit_index, + pending_write_resources.drain(..), + active_executions, + ); + + // This will schedule destruction of all resources that are no longer needed + // by the user but used in the command stream, among other things. + let (closures, _) = match device.maintain(fence, wgt::Maintain::Poll) { + Ok(closures) => closures, + Err(WaitIdleError::Device(err)) => return Err(QueueSubmitError::Queue(err)), + Err(WaitIdleError::StuckGpu) => return Err(QueueSubmitError::StuckGpu), + Err(WaitIdleError::WrongSubmissionIndex(..)) => unreachable!(), + }; + + // pending_write_resources has been drained, so it's empty, but we + // want to retain its heap allocation. + pending_writes.temp_resources = pending_write_resources; + device.lock_life().post_submit(); + + (submit_index, closures) + }; + + // the closures should execute with nothing locked! + callbacks.fire(); + + Ok(WrappedSubmissionIndex { + queue_id, + index: submit_index, + }) + } + + pub fn queue_get_timestamp_period<A: HalApi>( + &self, + queue_id: QueueId, + ) -> Result<f32, InvalidQueue> { + let hub = A::hub(self); + match hub.queues.get(queue_id) { + Ok(queue) => Ok(unsafe { queue.raw.as_ref().unwrap().get_timestamp_period() }), + Err(_) => Err(InvalidQueue), + } + } + + pub fn queue_on_submitted_work_done<A: HalApi>( + &self, + queue_id: QueueId, + closure: SubmittedWorkDoneClosure, + ) -> Result<(), InvalidQueue> { + api_log!("Queue::on_submitted_work_done {queue_id:?}"); + + //TODO: flush pending writes + let hub = A::hub(self); + match hub.queues.get(queue_id) { + Ok(queue) => queue + .device + .as_ref() + .unwrap() + .lock_life() + .add_work_done_closure(closure), + Err(_) => return Err(InvalidQueue), + } + Ok(()) + } +} diff --git a/third_party/rust/wgpu-core/src/device/resource.rs b/third_party/rust/wgpu-core/src/device/resource.rs new file mode 100644 index 0000000000..b2c85a056a --- /dev/null +++ b/third_party/rust/wgpu-core/src/device/resource.rs @@ -0,0 +1,3530 @@ +#[cfg(feature = "trace")] +use crate::device::trace; +use crate::{ + binding_model::{self, BindGroup, BindGroupLayout, BindGroupLayoutEntryError}, + command, conv, + device::{ + bgl, + life::{LifetimeTracker, WaitIdleError}, + queue::PendingWrites, + AttachmentData, CommandAllocator, DeviceLostInvocation, MissingDownlevelFlags, + MissingFeatures, RenderPassContext, CLEANUP_WAIT_MS, + }, + hal_api::HalApi, + hal_label, + hub::Hub, + id::QueueId, + init_tracker::{ + BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange, + TextureInitTracker, TextureInitTrackerAction, + }, + instance::Adapter, + pipeline, + pool::ResourcePool, + registry::Registry, + resource::{ + self, Buffer, QuerySet, Resource, ResourceInfo, ResourceType, Sampler, Texture, + TextureView, TextureViewNotRenderableReason, + }, + resource_log, + snatch::{SnatchGuard, SnatchLock, Snatchable}, + storage::Storage, + track::{BindGroupStates, TextureSelector, Tracker}, + validation::{self, check_buffer_usage, check_texture_usage}, + FastHashMap, LabelHelpers as _, SubmissionIndex, +}; + +use arrayvec::ArrayVec; +use hal::{CommandEncoder as _, Device as _}; +use parking_lot::{Mutex, MutexGuard, RwLock}; + +use smallvec::SmallVec; +use thiserror::Error; +use wgt::{DeviceLostReason, TextureFormat, TextureSampleType, TextureViewDimension}; + +use std::{ + borrow::Cow, + iter, + num::NonZeroU32, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Weak, + }, +}; + +use super::{ + life::{self, ResourceMaps}, + queue::{self}, + DeviceDescriptor, DeviceError, ImplicitPipelineContext, UserClosures, ENTRYPOINT_FAILURE_ERROR, + IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL, ZERO_BUFFER_SIZE, +}; + +/// Structure describing a logical device. Some members are internally mutable, +/// stored behind mutexes. +/// +/// TODO: establish clear order of locking for these: +/// `life_tracker`, `trackers`, `render_passes`, `pending_writes`, `trace`. +/// +/// Currently, the rules are: +/// 1. `life_tracker` is locked after `hub.devices`, enforced by the type system +/// 1. `self.trackers` is locked last (unenforced) +/// 1. `self.trace` is locked last (unenforced) +/// +/// Right now avoid locking twice same resource or registry in a call execution +/// and minimize the locking to the minimum scope possible +/// Unless otherwise specified, no lock may be acquired while holding another lock. +/// This means that you must inspect function calls made while a lock is held +/// to see what locks the callee may try to acquire. +/// +/// As far as this point: +/// device_maintain_ids locks Device::lifetime_tracker, and calls... +/// triage_suspected locks Device::trackers, and calls... +/// Registry::unregister locks Registry::storage +/// +/// Important: +/// When locking pending_writes please check that trackers is not locked +/// trackers should be locked only when needed for the shortest time possible +pub struct Device<A: HalApi> { + raw: Option<A::Device>, + pub(crate) adapter: Arc<Adapter<A>>, + pub(crate) queue_id: RwLock<Option<QueueId>>, + queue_to_drop: RwLock<Option<A::Queue>>, + pub(crate) zero_buffer: Option<A::Buffer>, + pub(crate) info: ResourceInfo<Device<A>>, + + pub(crate) command_allocator: Mutex<Option<CommandAllocator<A>>>, + //Note: The submission index here corresponds to the last submission that is done. + pub(crate) active_submission_index: AtomicU64, //SubmissionIndex, + pub(crate) fence: RwLock<Option<A::Fence>>, + pub(crate) snatchable_lock: SnatchLock, + + /// Is this device valid? Valid is closely associated with "lose the device", + /// which can be triggered by various methods, including at the end of device + /// destroy, and by any GPU errors that cause us to no longer trust the state + /// of the device. Ideally we would like to fold valid into the storage of + /// the device itself (for example as an Error enum), but unfortunately we + /// need to continue to be able to retrieve the device in poll_devices to + /// determine if it can be dropped. If our internal accesses of devices were + /// done through ref-counted references and external accesses checked for + /// Error enums, we wouldn't need this. For now, we need it. All the call + /// sites where we check it are areas that should be revisited if we start + /// using ref-counted references for internal access. + pub(crate) valid: AtomicBool, + + /// All live resources allocated with this [`Device`]. + /// + /// Has to be locked temporarily only (locked last) + /// and never before pending_writes + pub(crate) trackers: Mutex<Tracker<A>>, + // Life tracker should be locked right after the device and before anything else. + life_tracker: Mutex<LifetimeTracker<A>>, + /// Temporary storage for resource management functions. Cleared at the end + /// of every call (unless an error occurs). + pub(crate) temp_suspected: Mutex<Option<ResourceMaps<A>>>, + /// Pool of bind group layouts, allowing deduplication. + pub(crate) bgl_pool: ResourcePool<bgl::EntryMap, BindGroupLayout<A>>, + pub(crate) alignments: hal::Alignments, + pub(crate) limits: wgt::Limits, + pub(crate) features: wgt::Features, + pub(crate) downlevel: wgt::DownlevelCapabilities, + pub(crate) instance_flags: wgt::InstanceFlags, + pub(crate) pending_writes: Mutex<Option<PendingWrites<A>>>, + pub(crate) deferred_destroy: Mutex<Vec<DeferredDestroy<A>>>, + #[cfg(feature = "trace")] + pub(crate) trace: Mutex<Option<trace::Trace>>, +} + +pub(crate) enum DeferredDestroy<A: HalApi> { + TextureView(Weak<TextureView<A>>), + BindGroup(Weak<BindGroup<A>>), +} + +impl<A: HalApi> std::fmt::Debug for Device<A> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Device") + .field("adapter", &self.adapter.info.label()) + .field("limits", &self.limits) + .field("features", &self.features) + .field("downlevel", &self.downlevel) + .finish() + } +} + +impl<A: HalApi> Drop for Device<A> { + fn drop(&mut self) { + resource_log!("Destroy raw Device {:?}", self.info.label()); + let raw = self.raw.take().unwrap(); + let pending_writes = self.pending_writes.lock().take().unwrap(); + pending_writes.dispose(&raw); + self.command_allocator.lock().take().unwrap().dispose(&raw); + unsafe { + raw.destroy_buffer(self.zero_buffer.take().unwrap()); + raw.destroy_fence(self.fence.write().take().unwrap()); + let queue = self.queue_to_drop.write().take().unwrap(); + raw.exit(queue); + } + } +} + +#[derive(Clone, Debug, Error)] +pub enum CreateDeviceError { + #[error("Not enough memory left to create device")] + OutOfMemory, + #[error("Failed to create internal buffer for initializing textures")] + FailedToCreateZeroBuffer(#[from] DeviceError), +} + +impl<A: HalApi> Device<A> { + pub(crate) fn raw(&self) -> &A::Device { + self.raw.as_ref().unwrap() + } + pub(crate) fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> { + if self.features.contains(feature) { + Ok(()) + } else { + Err(MissingFeatures(feature)) + } + } + + pub(crate) fn require_downlevel_flags( + &self, + flags: wgt::DownlevelFlags, + ) -> Result<(), MissingDownlevelFlags> { + if self.downlevel.flags.contains(flags) { + Ok(()) + } else { + Err(MissingDownlevelFlags(flags)) + } + } +} + +impl<A: HalApi> Device<A> { + pub(crate) fn new( + raw_device: A::Device, + raw_queue: &A::Queue, + adapter: &Arc<Adapter<A>>, + desc: &DeviceDescriptor, + trace_path: Option<&std::path::Path>, + instance_flags: wgt::InstanceFlags, + ) -> Result<Self, CreateDeviceError> { + #[cfg(not(feature = "trace"))] + if let Some(_) = trace_path { + log::error!("Feature 'trace' is not enabled"); + } + let fence = + unsafe { raw_device.create_fence() }.map_err(|_| CreateDeviceError::OutOfMemory)?; + + let mut com_alloc = CommandAllocator { + free_encoders: Vec::new(), + }; + let pending_encoder = com_alloc + .acquire_encoder(&raw_device, raw_queue) + .map_err(|_| CreateDeviceError::OutOfMemory)?; + let mut pending_writes = queue::PendingWrites::<A>::new(pending_encoder); + + // Create zeroed buffer used for texture clears. + let zero_buffer = unsafe { + raw_device + .create_buffer(&hal::BufferDescriptor { + label: hal_label(Some("(wgpu internal) zero init buffer"), instance_flags), + size: ZERO_BUFFER_SIZE, + usage: hal::BufferUses::COPY_SRC | hal::BufferUses::COPY_DST, + memory_flags: hal::MemoryFlags::empty(), + }) + .map_err(DeviceError::from)? + }; + pending_writes.activate(); + unsafe { + pending_writes + .command_encoder + .transition_buffers(iter::once(hal::BufferBarrier { + buffer: &zero_buffer, + usage: hal::BufferUses::empty()..hal::BufferUses::COPY_DST, + })); + pending_writes + .command_encoder + .clear_buffer(&zero_buffer, 0..ZERO_BUFFER_SIZE); + pending_writes + .command_encoder + .transition_buffers(iter::once(hal::BufferBarrier { + buffer: &zero_buffer, + usage: hal::BufferUses::COPY_DST..hal::BufferUses::COPY_SRC, + })); + } + + let alignments = adapter.raw.capabilities.alignments.clone(); + let downlevel = adapter.raw.capabilities.downlevel.clone(); + + Ok(Self { + raw: Some(raw_device), + adapter: adapter.clone(), + queue_id: RwLock::new(None), + queue_to_drop: RwLock::new(None), + zero_buffer: Some(zero_buffer), + info: ResourceInfo::new("<device>"), + command_allocator: Mutex::new(Some(com_alloc)), + active_submission_index: AtomicU64::new(0), + fence: RwLock::new(Some(fence)), + snatchable_lock: unsafe { SnatchLock::new() }, + valid: AtomicBool::new(true), + trackers: Mutex::new(Tracker::new()), + life_tracker: Mutex::new(life::LifetimeTracker::new()), + temp_suspected: Mutex::new(Some(life::ResourceMaps::new())), + bgl_pool: ResourcePool::new(), + #[cfg(feature = "trace")] + trace: Mutex::new(trace_path.and_then(|path| match trace::Trace::new(path) { + Ok(mut trace) => { + trace.add(trace::Action::Init { + desc: desc.clone(), + backend: A::VARIANT, + }); + Some(trace) + } + Err(e) => { + log::error!("Unable to start a trace in '{path:?}': {e}"); + None + } + })), + alignments, + limits: desc.required_limits.clone(), + features: desc.required_features, + downlevel, + instance_flags, + pending_writes: Mutex::new(Some(pending_writes)), + deferred_destroy: Mutex::new(Vec::new()), + }) + } + + pub fn is_valid(&self) -> bool { + self.valid.load(Ordering::Acquire) + } + + pub(crate) fn release_queue(&self, queue: A::Queue) { + self.queue_to_drop.write().replace(queue); + } + + pub(crate) fn lock_life<'a>(&'a self) -> MutexGuard<'a, LifetimeTracker<A>> { + self.life_tracker.lock() + } + + /// Run some destroy operations that were deferred. + /// + /// Destroying the resources requires taking a write lock on the device's snatch lock, + /// so a good reason for deferring resource destruction is when we don't know for sure + /// how risky it is to take the lock (typically, it shouldn't be taken from the drop + /// implementation of a reference-counted structure). + /// The snatch lock must not be held while this function is called. + pub(crate) fn deferred_resource_destruction(&self) { + while let Some(item) = self.deferred_destroy.lock().pop() { + match item { + DeferredDestroy::TextureView(view) => { + let Some(view) = view.upgrade() else { + continue; + }; + let Some(raw_view) = view.raw.snatch(self.snatchable_lock.write()) else { + continue; + }; + + resource_log!("Destroy raw TextureView (destroyed) {:?}", view.label()); + #[cfg(feature = "trace")] + if let Some(t) = self.trace.lock().as_mut() { + t.add(trace::Action::DestroyTextureView(view.info.id())); + } + unsafe { + use hal::Device; + self.raw().destroy_texture_view(raw_view); + } + } + DeferredDestroy::BindGroup(bind_group) => { + let Some(bind_group) = bind_group.upgrade() else { + continue; + }; + let Some(raw_bind_group) = bind_group.raw.snatch(self.snatchable_lock.write()) else { + continue; + }; + + resource_log!("Destroy raw BindGroup (destroyed) {:?}", bind_group.label()); + #[cfg(feature = "trace")] + if let Some(t) = self.trace.lock().as_mut() { + t.add(trace::Action::DestroyBindGroup(bind_group.info.id())); + } + unsafe { + use hal::Device; + self.raw().destroy_bind_group(raw_bind_group); + } + } + } + } + } + + /// Check this device for completed commands. + /// + /// The `maintain` argument tells how the maintence function should behave, either + /// blocking or just polling the current state of the gpu. + /// + /// Return a pair `(closures, queue_empty)`, where: + /// + /// - `closures` is a list of actions to take: mapping buffers, notifying the user + /// + /// - `queue_empty` is a boolean indicating whether there are more queue + /// submissions still in flight. (We have to take the locks needed to + /// produce this information for other reasons, so we might as well just + /// return it to our callers.) + pub(crate) fn maintain<'this>( + &'this self, + fence: &A::Fence, + maintain: wgt::Maintain<queue::WrappedSubmissionIndex>, + ) -> Result<(UserClosures, bool), WaitIdleError> { + profiling::scope!("Device::maintain"); + let last_done_index = if maintain.is_wait() { + let index_to_wait_for = match maintain { + wgt::Maintain::WaitForSubmissionIndex(submission_index) => { + // We don't need to check to see if the queue id matches + // as we already checked this from inside the poll call. + submission_index.index + } + _ => self.active_submission_index.load(Ordering::Relaxed), + }; + unsafe { + self.raw + .as_ref() + .unwrap() + .wait(fence, index_to_wait_for, CLEANUP_WAIT_MS) + .map_err(DeviceError::from)? + }; + index_to_wait_for + } else { + unsafe { + self.raw + .as_ref() + .unwrap() + .get_fence_value(fence) + .map_err(DeviceError::from)? + } + }; + + let mut life_tracker = self.lock_life(); + let submission_closures = life_tracker.triage_submissions( + last_done_index, + self.command_allocator.lock().as_mut().unwrap(), + ); + + { + // Normally, `temp_suspected` exists only to save heap + // allocations: it's cleared at the start of the function + // call, and cleared by the end. But `Global::queue_submit` is + // fallible; if it exits early, it may leave some resources in + // `temp_suspected`. + let temp_suspected = self + .temp_suspected + .lock() + .replace(ResourceMaps::new()) + .unwrap(); + + life_tracker.suspected_resources.extend(temp_suspected); + + life_tracker.triage_suspected(&self.trackers); + life_tracker.triage_mapped(); + } + + let mapping_closures = life_tracker.handle_mapping(self.raw(), &self.trackers); + + let queue_empty = life_tracker.queue_empty(); + + // Detect if we have been destroyed and now need to lose the device. + // If we are invalid (set at start of destroy) and our queue is empty, + // and we have a DeviceLostClosure, return the closure to be called by + // our caller. This will complete the steps for both destroy and for + // "lose the device". + let mut device_lost_invocations = SmallVec::new(); + let mut should_release_gpu_resource = false; + if !self.is_valid() && queue_empty { + // We can release gpu resources associated with this device (but not + // while holding the life_tracker lock). + should_release_gpu_resource = true; + + // If we have a DeviceLostClosure, build an invocation with the + // reason DeviceLostReason::Destroyed and no message. + if life_tracker.device_lost_closure.is_some() { + device_lost_invocations.push(DeviceLostInvocation { + closure: life_tracker.device_lost_closure.take().unwrap(), + reason: DeviceLostReason::Destroyed, + message: String::new(), + }); + } + } + + // Don't hold the lock while calling release_gpu_resources. + drop(life_tracker); + + if should_release_gpu_resource { + self.release_gpu_resources(); + } + + let closures = UserClosures { + mappings: mapping_closures, + submissions: submission_closures, + device_lost_invocations, + }; + Ok((closures, queue_empty)) + } + + pub(crate) fn untrack(&self, trackers: &Tracker<A>) { + let mut temp_suspected = self + .temp_suspected + .lock() + .replace(ResourceMaps::new()) + .unwrap(); + temp_suspected.clear(); + // As the tracker is cleared/dropped, we need to consider all the resources + // that it references for destruction in the next GC pass. + { + for resource in trackers.buffers.used_resources() { + if resource.is_unique() { + temp_suspected + .buffers + .insert(resource.as_info().id(), resource.clone()); + } + } + for resource in trackers.textures.used_resources() { + if resource.is_unique() { + temp_suspected + .textures + .insert(resource.as_info().id(), resource.clone()); + } + } + for resource in trackers.views.used_resources() { + if resource.is_unique() { + temp_suspected + .texture_views + .insert(resource.as_info().id(), resource.clone()); + } + } + for resource in trackers.bind_groups.used_resources() { + if resource.is_unique() { + temp_suspected + .bind_groups + .insert(resource.as_info().id(), resource.clone()); + } + } + for resource in trackers.samplers.used_resources() { + if resource.is_unique() { + temp_suspected + .samplers + .insert(resource.as_info().id(), resource.clone()); + } + } + for resource in trackers.compute_pipelines.used_resources() { + if resource.is_unique() { + temp_suspected + .compute_pipelines + .insert(resource.as_info().id(), resource.clone()); + } + } + for resource in trackers.render_pipelines.used_resources() { + if resource.is_unique() { + temp_suspected + .render_pipelines + .insert(resource.as_info().id(), resource.clone()); + } + } + for resource in trackers.query_sets.used_resources() { + if resource.is_unique() { + temp_suspected + .query_sets + .insert(resource.as_info().id(), resource.clone()); + } + } + } + self.lock_life().suspected_resources.extend(temp_suspected); + } + + pub(crate) fn create_buffer( + self: &Arc<Self>, + desc: &resource::BufferDescriptor, + transient: bool, + ) -> Result<Buffer<A>, resource::CreateBufferError> { + debug_assert_eq!(self.as_info().id().backend(), A::VARIANT); + + if desc.size > self.limits.max_buffer_size { + return Err(resource::CreateBufferError::MaxBufferSize { + requested: desc.size, + maximum: self.limits.max_buffer_size, + }); + } + + if desc.usage.contains(wgt::BufferUsages::INDEX) + && desc.usage.contains( + wgt::BufferUsages::VERTEX + | wgt::BufferUsages::UNIFORM + | wgt::BufferUsages::INDIRECT + | wgt::BufferUsages::STORAGE, + ) + { + self.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)?; + } + + let mut usage = conv::map_buffer_usage(desc.usage); + + if desc.usage.is_empty() || desc.usage.contains_invalid_bits() { + return Err(resource::CreateBufferError::InvalidUsage(desc.usage)); + } + + if !self + .features + .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS) + { + use wgt::BufferUsages as Bu; + let write_mismatch = desc.usage.contains(Bu::MAP_WRITE) + && !(Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage); + let read_mismatch = desc.usage.contains(Bu::MAP_READ) + && !(Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage); + if write_mismatch || read_mismatch { + return Err(resource::CreateBufferError::UsageMismatch(desc.usage)); + } + } + + if desc.mapped_at_creation { + if desc.size % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err(resource::CreateBufferError::UnalignedSize); + } + if !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) { + // we are going to be copying into it, internally + usage |= hal::BufferUses::COPY_DST; + } + } else { + // We are required to zero out (initialize) all memory. This is done + // on demand using clear_buffer which requires write transfer usage! + usage |= hal::BufferUses::COPY_DST; + } + + let actual_size = if desc.size == 0 { + wgt::COPY_BUFFER_ALIGNMENT + } else if desc.usage.contains(wgt::BufferUsages::VERTEX) { + // Bumping the size by 1 so that we can bind an empty range at the + // end of the buffer. + desc.size + 1 + } else { + desc.size + }; + let clear_remainder = actual_size % wgt::COPY_BUFFER_ALIGNMENT; + let aligned_size = if clear_remainder != 0 { + actual_size + wgt::COPY_BUFFER_ALIGNMENT - clear_remainder + } else { + actual_size + }; + + let mut memory_flags = hal::MemoryFlags::empty(); + memory_flags.set(hal::MemoryFlags::TRANSIENT, transient); + + let hal_desc = hal::BufferDescriptor { + label: desc.label.to_hal(self.instance_flags), + size: aligned_size, + usage, + memory_flags, + }; + let buffer = unsafe { self.raw().create_buffer(&hal_desc) }.map_err(DeviceError::from)?; + + Ok(Buffer { + raw: Snatchable::new(buffer), + device: self.clone(), + usage: desc.usage, + size: desc.size, + initialization_status: RwLock::new(BufferInitTracker::new(aligned_size)), + sync_mapped_writes: Mutex::new(None), + map_state: Mutex::new(resource::BufferMapState::Idle), + info: ResourceInfo::new(desc.label.borrow_or_default()), + bind_groups: Mutex::new(Vec::new()), + }) + } + + pub(crate) fn create_texture_from_hal( + self: &Arc<Self>, + hal_texture: A::Texture, + hal_usage: hal::TextureUses, + desc: &resource::TextureDescriptor, + format_features: wgt::TextureFormatFeatures, + clear_mode: resource::TextureClearMode<A>, + ) -> Texture<A> { + debug_assert_eq!(self.as_info().id().backend(), A::VARIANT); + + Texture { + inner: Snatchable::new(resource::TextureInner::Native { raw: hal_texture }), + device: self.clone(), + desc: desc.map_label(|_| ()), + hal_usage, + format_features, + initialization_status: RwLock::new(TextureInitTracker::new( + desc.mip_level_count, + desc.array_layer_count(), + )), + full_range: TextureSelector { + mips: 0..desc.mip_level_count, + layers: 0..desc.array_layer_count(), + }, + info: ResourceInfo::new(desc.label.borrow_or_default()), + clear_mode: RwLock::new(clear_mode), + views: Mutex::new(Vec::new()), + bind_groups: Mutex::new(Vec::new()), + } + } + + pub fn create_buffer_from_hal( + self: &Arc<Self>, + hal_buffer: A::Buffer, + desc: &resource::BufferDescriptor, + ) -> Buffer<A> { + debug_assert_eq!(self.as_info().id().backend(), A::VARIANT); + + Buffer { + raw: Snatchable::new(hal_buffer), + device: self.clone(), + usage: desc.usage, + size: desc.size, + initialization_status: RwLock::new(BufferInitTracker::new(0)), + sync_mapped_writes: Mutex::new(None), + map_state: Mutex::new(resource::BufferMapState::Idle), + info: ResourceInfo::new(desc.label.borrow_or_default()), + bind_groups: Mutex::new(Vec::new()), + } + } + + pub(crate) fn create_texture( + self: &Arc<Self>, + adapter: &Adapter<A>, + desc: &resource::TextureDescriptor, + ) -> Result<Texture<A>, resource::CreateTextureError> { + use resource::{CreateTextureError, TextureDimensionError}; + + if desc.usage.is_empty() || desc.usage.contains_invalid_bits() { + return Err(CreateTextureError::InvalidUsage(desc.usage)); + } + + conv::check_texture_dimension_size( + desc.dimension, + desc.size, + desc.sample_count, + &self.limits, + )?; + + if desc.dimension != wgt::TextureDimension::D2 { + // Depth textures can only be 2D + if desc.format.is_depth_stencil_format() { + return Err(CreateTextureError::InvalidDepthDimension( + desc.dimension, + desc.format, + )); + } + // Renderable textures can only be 2D + if desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) { + return Err(CreateTextureError::InvalidDimensionUsages( + wgt::TextureUsages::RENDER_ATTACHMENT, + desc.dimension, + )); + } + + // Compressed textures can only be 2D + if desc.format.is_compressed() { + return Err(CreateTextureError::InvalidCompressedDimension( + desc.dimension, + desc.format, + )); + } + } + + if desc.format.is_compressed() { + let (block_width, block_height) = desc.format.block_dimensions(); + + if desc.size.width % block_width != 0 { + return Err(CreateTextureError::InvalidDimension( + TextureDimensionError::NotMultipleOfBlockWidth { + width: desc.size.width, + block_width, + format: desc.format, + }, + )); + } + + if desc.size.height % block_height != 0 { + return Err(CreateTextureError::InvalidDimension( + TextureDimensionError::NotMultipleOfBlockHeight { + height: desc.size.height, + block_height, + format: desc.format, + }, + )); + } + } + + { + let (width_multiple, height_multiple) = desc.format.size_multiple_requirement(); + + if desc.size.width % width_multiple != 0 { + return Err(CreateTextureError::InvalidDimension( + TextureDimensionError::WidthNotMultipleOf { + width: desc.size.width, + multiple: width_multiple, + format: desc.format, + }, + )); + } + + if desc.size.height % height_multiple != 0 { + return Err(CreateTextureError::InvalidDimension( + TextureDimensionError::HeightNotMultipleOf { + height: desc.size.height, + multiple: height_multiple, + format: desc.format, + }, + )); + } + } + + let format_features = self + .describe_format_features(adapter, desc.format) + .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?; + + if desc.sample_count > 1 { + if desc.mip_level_count != 1 { + return Err(CreateTextureError::InvalidMipLevelCount { + requested: desc.mip_level_count, + maximum: 1, + }); + } + + if desc.size.depth_or_array_layers != 1 { + return Err(CreateTextureError::InvalidDimension( + TextureDimensionError::MultisampledDepthOrArrayLayer( + desc.size.depth_or_array_layers, + ), + )); + } + + if desc.usage.contains(wgt::TextureUsages::STORAGE_BINDING) { + return Err(CreateTextureError::InvalidMultisampledStorageBinding); + } + + if !desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) { + return Err(CreateTextureError::MultisampledNotRenderAttachment); + } + + if !format_features.flags.intersects( + wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4 + | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X2 + | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X8 + | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X16, + ) { + return Err(CreateTextureError::InvalidMultisampledFormat(desc.format)); + } + + if !format_features + .flags + .sample_count_supported(desc.sample_count) + { + return Err(CreateTextureError::InvalidSampleCount( + desc.sample_count, + desc.format, + desc.format + .guaranteed_format_features(self.features) + .flags + .supported_sample_counts(), + adapter + .get_texture_format_features(desc.format) + .flags + .supported_sample_counts(), + )); + }; + } + + let mips = desc.mip_level_count; + let max_levels_allowed = desc.size.max_mips(desc.dimension).min(hal::MAX_MIP_LEVELS); + if mips == 0 || mips > max_levels_allowed { + return Err(CreateTextureError::InvalidMipLevelCount { + requested: mips, + maximum: max_levels_allowed, + }); + } + + let missing_allowed_usages = desc.usage - format_features.allowed_usages; + if !missing_allowed_usages.is_empty() { + // detect downlevel incompatibilities + let wgpu_allowed_usages = desc + .format + .guaranteed_format_features(self.features) + .allowed_usages; + let wgpu_missing_usages = desc.usage - wgpu_allowed_usages; + return Err(CreateTextureError::InvalidFormatUsages( + missing_allowed_usages, + desc.format, + wgpu_missing_usages.is_empty(), + )); + } + + let mut hal_view_formats = vec![]; + for format in desc.view_formats.iter() { + if desc.format == *format { + continue; + } + if desc.format.remove_srgb_suffix() != format.remove_srgb_suffix() { + return Err(CreateTextureError::InvalidViewFormat(*format, desc.format)); + } + hal_view_formats.push(*format); + } + if !hal_view_formats.is_empty() { + self.require_downlevel_flags(wgt::DownlevelFlags::VIEW_FORMATS)?; + } + + let hal_usage = conv::map_texture_usage_for_texture(desc, &format_features); + + let hal_desc = hal::TextureDescriptor { + label: desc.label.to_hal(self.instance_flags), + size: desc.size, + mip_level_count: desc.mip_level_count, + sample_count: desc.sample_count, + dimension: desc.dimension, + format: desc.format, + usage: hal_usage, + memory_flags: hal::MemoryFlags::empty(), + view_formats: hal_view_formats, + }; + + let raw_texture = unsafe { + self.raw + .as_ref() + .unwrap() + .create_texture(&hal_desc) + .map_err(DeviceError::from)? + }; + + let clear_mode = if hal_usage + .intersects(hal::TextureUses::DEPTH_STENCIL_WRITE | hal::TextureUses::COLOR_TARGET) + { + let (is_color, usage) = if desc.format.is_depth_stencil_format() { + (false, hal::TextureUses::DEPTH_STENCIL_WRITE) + } else { + (true, hal::TextureUses::COLOR_TARGET) + }; + let dimension = match desc.dimension { + wgt::TextureDimension::D1 => wgt::TextureViewDimension::D1, + wgt::TextureDimension::D2 => wgt::TextureViewDimension::D2, + wgt::TextureDimension::D3 => unreachable!(), + }; + + let clear_label = hal_label( + Some("(wgpu internal) clear texture view"), + self.instance_flags, + ); + + let mut clear_views = SmallVec::new(); + for mip_level in 0..desc.mip_level_count { + for array_layer in 0..desc.size.depth_or_array_layers { + macro_rules! push_clear_view { + ($format:expr, $aspect:expr) => { + let desc = hal::TextureViewDescriptor { + label: clear_label, + format: $format, + dimension, + usage, + range: wgt::ImageSubresourceRange { + aspect: $aspect, + base_mip_level: mip_level, + mip_level_count: Some(1), + base_array_layer: array_layer, + array_layer_count: Some(1), + }, + }; + clear_views.push(Some( + unsafe { self.raw().create_texture_view(&raw_texture, &desc) } + .map_err(DeviceError::from)?, + )); + }; + } + + if let Some(planes) = desc.format.planes() { + for plane in 0..planes { + let aspect = wgt::TextureAspect::from_plane(plane).unwrap(); + let format = desc.format.aspect_specific_format(aspect).unwrap(); + push_clear_view!(format, aspect); + } + } else { + push_clear_view!(desc.format, wgt::TextureAspect::All); + } + } + } + resource::TextureClearMode::RenderPass { + clear_views, + is_color, + } + } else { + resource::TextureClearMode::BufferCopy + }; + + let mut texture = + self.create_texture_from_hal(raw_texture, hal_usage, desc, format_features, clear_mode); + texture.hal_usage = hal_usage; + Ok(texture) + } + + pub(crate) fn create_texture_view( + self: &Arc<Self>, + texture: &Arc<Texture<A>>, + desc: &resource::TextureViewDescriptor, + ) -> Result<TextureView<A>, resource::CreateTextureViewError> { + let snatch_guard = texture.device.snatchable_lock.read(); + + let texture_raw = texture + .raw(&snatch_guard) + .ok_or(resource::CreateTextureViewError::InvalidTexture)?; + + // resolve TextureViewDescriptor defaults + // https://gpuweb.github.io/gpuweb/#abstract-opdef-resolving-gputextureviewdescriptor-defaults + let resolved_format = desc.format.unwrap_or_else(|| { + texture + .desc + .format + .aspect_specific_format(desc.range.aspect) + .unwrap_or(texture.desc.format) + }); + + let resolved_dimension = desc + .dimension + .unwrap_or_else(|| match texture.desc.dimension { + wgt::TextureDimension::D1 => wgt::TextureViewDimension::D1, + wgt::TextureDimension::D2 => { + if texture.desc.array_layer_count() == 1 { + wgt::TextureViewDimension::D2 + } else { + wgt::TextureViewDimension::D2Array + } + } + wgt::TextureDimension::D3 => wgt::TextureViewDimension::D3, + }); + + let resolved_mip_level_count = desc.range.mip_level_count.unwrap_or_else(|| { + texture + .desc + .mip_level_count + .saturating_sub(desc.range.base_mip_level) + }); + + let resolved_array_layer_count = + desc.range + .array_layer_count + .unwrap_or_else(|| match resolved_dimension { + wgt::TextureViewDimension::D1 + | wgt::TextureViewDimension::D2 + | wgt::TextureViewDimension::D3 => 1, + wgt::TextureViewDimension::Cube => 6, + wgt::TextureViewDimension::D2Array | wgt::TextureViewDimension::CubeArray => { + texture + .desc + .array_layer_count() + .saturating_sub(desc.range.base_array_layer) + } + }); + + // validate TextureViewDescriptor + + let aspects = hal::FormatAspects::new(texture.desc.format, desc.range.aspect); + if aspects.is_empty() { + return Err(resource::CreateTextureViewError::InvalidAspect { + texture_format: texture.desc.format, + requested_aspect: desc.range.aspect, + }); + } + + let format_is_good = if desc.range.aspect == wgt::TextureAspect::All { + resolved_format == texture.desc.format + || texture.desc.view_formats.contains(&resolved_format) + } else { + Some(resolved_format) + == texture + .desc + .format + .aspect_specific_format(desc.range.aspect) + }; + if !format_is_good { + return Err(resource::CreateTextureViewError::FormatReinterpretation { + texture: texture.desc.format, + view: resolved_format, + }); + } + + // check if multisampled texture is seen as anything but 2D + if texture.desc.sample_count > 1 && resolved_dimension != wgt::TextureViewDimension::D2 { + return Err( + resource::CreateTextureViewError::InvalidMultisampledTextureViewDimension( + resolved_dimension, + ), + ); + } + + // check if the dimension is compatible with the texture + if texture.desc.dimension != resolved_dimension.compatible_texture_dimension() { + return Err( + resource::CreateTextureViewError::InvalidTextureViewDimension { + view: resolved_dimension, + texture: texture.desc.dimension, + }, + ); + } + + match resolved_dimension { + TextureViewDimension::D1 | TextureViewDimension::D2 | TextureViewDimension::D3 => { + if resolved_array_layer_count != 1 { + return Err(resource::CreateTextureViewError::InvalidArrayLayerCount { + requested: resolved_array_layer_count, + dim: resolved_dimension, + }); + } + } + TextureViewDimension::Cube => { + if resolved_array_layer_count != 6 { + return Err( + resource::CreateTextureViewError::InvalidCubemapTextureDepth { + depth: resolved_array_layer_count, + }, + ); + } + } + TextureViewDimension::CubeArray => { + if resolved_array_layer_count % 6 != 0 { + return Err( + resource::CreateTextureViewError::InvalidCubemapArrayTextureDepth { + depth: resolved_array_layer_count, + }, + ); + } + } + _ => {} + } + + match resolved_dimension { + TextureViewDimension::Cube | TextureViewDimension::CubeArray => { + if texture.desc.size.width != texture.desc.size.height { + return Err(resource::CreateTextureViewError::InvalidCubeTextureViewSize); + } + } + _ => {} + } + + if resolved_mip_level_count == 0 { + return Err(resource::CreateTextureViewError::ZeroMipLevelCount); + } + + let mip_level_end = desc + .range + .base_mip_level + .saturating_add(resolved_mip_level_count); + + let level_end = texture.desc.mip_level_count; + if mip_level_end > level_end { + return Err(resource::CreateTextureViewError::TooManyMipLevels { + requested: mip_level_end, + total: level_end, + }); + } + + if resolved_array_layer_count == 0 { + return Err(resource::CreateTextureViewError::ZeroArrayLayerCount); + } + + let array_layer_end = desc + .range + .base_array_layer + .saturating_add(resolved_array_layer_count); + + let layer_end = texture.desc.array_layer_count(); + if array_layer_end > layer_end { + return Err(resource::CreateTextureViewError::TooManyArrayLayers { + requested: array_layer_end, + total: layer_end, + }); + }; + + // https://gpuweb.github.io/gpuweb/#abstract-opdef-renderable-texture-view + let render_extent = 'b: loop { + if !texture + .desc + .usage + .contains(wgt::TextureUsages::RENDER_ATTACHMENT) + { + break 'b Err(TextureViewNotRenderableReason::Usage(texture.desc.usage)); + } + + if !(resolved_dimension == TextureViewDimension::D2 + || (self.features.contains(wgt::Features::MULTIVIEW) + && resolved_dimension == TextureViewDimension::D2Array)) + { + break 'b Err(TextureViewNotRenderableReason::Dimension( + resolved_dimension, + )); + } + + if resolved_mip_level_count != 1 { + break 'b Err(TextureViewNotRenderableReason::MipLevelCount( + resolved_mip_level_count, + )); + } + + if resolved_array_layer_count != 1 + && !(self.features.contains(wgt::Features::MULTIVIEW)) + { + break 'b Err(TextureViewNotRenderableReason::ArrayLayerCount( + resolved_array_layer_count, + )); + } + + if aspects != hal::FormatAspects::from(texture.desc.format) { + break 'b Err(TextureViewNotRenderableReason::Aspects(aspects)); + } + + break 'b Ok(texture + .desc + .compute_render_extent(desc.range.base_mip_level)); + }; + + // filter the usages based on the other criteria + let usage = { + let mask_copy = !(hal::TextureUses::COPY_SRC | hal::TextureUses::COPY_DST); + let mask_dimension = match resolved_dimension { + wgt::TextureViewDimension::Cube | wgt::TextureViewDimension::CubeArray => { + hal::TextureUses::RESOURCE + } + wgt::TextureViewDimension::D3 => { + hal::TextureUses::RESOURCE + | hal::TextureUses::STORAGE_READ + | hal::TextureUses::STORAGE_READ_WRITE + } + _ => hal::TextureUses::all(), + }; + let mask_mip_level = if resolved_mip_level_count == 1 { + hal::TextureUses::all() + } else { + hal::TextureUses::RESOURCE + }; + texture.hal_usage & mask_copy & mask_dimension & mask_mip_level + }; + + log::debug!( + "Create view for texture {:?} filters usages to {:?}", + texture.as_info().id(), + usage + ); + + // use the combined depth-stencil format for the view + let format = if resolved_format.is_depth_stencil_component(texture.desc.format) { + texture.desc.format + } else { + resolved_format + }; + + let resolved_range = wgt::ImageSubresourceRange { + aspect: desc.range.aspect, + base_mip_level: desc.range.base_mip_level, + mip_level_count: Some(resolved_mip_level_count), + base_array_layer: desc.range.base_array_layer, + array_layer_count: Some(resolved_array_layer_count), + }; + + let hal_desc = hal::TextureViewDescriptor { + label: desc.label.to_hal(self.instance_flags), + format, + dimension: resolved_dimension, + usage, + range: resolved_range, + }; + + let raw = unsafe { + self.raw + .as_ref() + .unwrap() + .create_texture_view(texture_raw, &hal_desc) + .map_err(|_| resource::CreateTextureViewError::OutOfMemory)? + }; + + let selector = TextureSelector { + mips: desc.range.base_mip_level..mip_level_end, + layers: desc.range.base_array_layer..array_layer_end, + }; + + Ok(TextureView { + raw: Snatchable::new(raw), + parent: texture.clone(), + device: self.clone(), + desc: resource::HalTextureViewDescriptor { + texture_format: texture.desc.format, + format: resolved_format, + dimension: resolved_dimension, + range: resolved_range, + }, + format_features: texture.format_features, + render_extent, + samples: texture.desc.sample_count, + selector, + info: ResourceInfo::new(desc.label.borrow_or_default()), + }) + } + + pub(crate) fn create_sampler( + self: &Arc<Self>, + desc: &resource::SamplerDescriptor, + ) -> Result<Sampler<A>, resource::CreateSamplerError> { + if desc + .address_modes + .iter() + .any(|am| am == &wgt::AddressMode::ClampToBorder) + { + self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER)?; + } + + if desc.border_color == Some(wgt::SamplerBorderColor::Zero) { + self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO)?; + } + + if desc.lod_min_clamp < 0.0 { + return Err(resource::CreateSamplerError::InvalidLodMinClamp( + desc.lod_min_clamp, + )); + } + if desc.lod_max_clamp < desc.lod_min_clamp { + return Err(resource::CreateSamplerError::InvalidLodMaxClamp { + lod_min_clamp: desc.lod_min_clamp, + lod_max_clamp: desc.lod_max_clamp, + }); + } + + if desc.anisotropy_clamp < 1 { + return Err(resource::CreateSamplerError::InvalidAnisotropy( + desc.anisotropy_clamp, + )); + } + + if desc.anisotropy_clamp != 1 { + if !matches!(desc.min_filter, wgt::FilterMode::Linear) { + return Err( + resource::CreateSamplerError::InvalidFilterModeWithAnisotropy { + filter_type: resource::SamplerFilterErrorType::MinFilter, + filter_mode: desc.min_filter, + anisotropic_clamp: desc.anisotropy_clamp, + }, + ); + } + if !matches!(desc.mag_filter, wgt::FilterMode::Linear) { + return Err( + resource::CreateSamplerError::InvalidFilterModeWithAnisotropy { + filter_type: resource::SamplerFilterErrorType::MagFilter, + filter_mode: desc.mag_filter, + anisotropic_clamp: desc.anisotropy_clamp, + }, + ); + } + if !matches!(desc.mipmap_filter, wgt::FilterMode::Linear) { + return Err( + resource::CreateSamplerError::InvalidFilterModeWithAnisotropy { + filter_type: resource::SamplerFilterErrorType::MipmapFilter, + filter_mode: desc.mipmap_filter, + anisotropic_clamp: desc.anisotropy_clamp, + }, + ); + } + } + + let anisotropy_clamp = if self + .downlevel + .flags + .contains(wgt::DownlevelFlags::ANISOTROPIC_FILTERING) + { + // Clamp anisotropy clamp to [1, 16] per the wgpu-hal interface + desc.anisotropy_clamp.min(16) + } else { + // If it isn't supported, set this unconditionally to 1 + 1 + }; + + //TODO: check for wgt::DownlevelFlags::COMPARISON_SAMPLERS + + let hal_desc = hal::SamplerDescriptor { + label: desc.label.to_hal(self.instance_flags), + address_modes: desc.address_modes, + mag_filter: desc.mag_filter, + min_filter: desc.min_filter, + mipmap_filter: desc.mipmap_filter, + lod_clamp: desc.lod_min_clamp..desc.lod_max_clamp, + compare: desc.compare, + anisotropy_clamp, + border_color: desc.border_color, + }; + + let raw = unsafe { + self.raw + .as_ref() + .unwrap() + .create_sampler(&hal_desc) + .map_err(DeviceError::from)? + }; + Ok(Sampler { + raw: Some(raw), + device: self.clone(), + info: ResourceInfo::new(desc.label.borrow_or_default()), + comparison: desc.compare.is_some(), + filtering: desc.min_filter == wgt::FilterMode::Linear + || desc.mag_filter == wgt::FilterMode::Linear, + }) + } + + pub(crate) fn create_shader_module<'a>( + self: &Arc<Self>, + desc: &pipeline::ShaderModuleDescriptor<'a>, + source: pipeline::ShaderModuleSource<'a>, + ) -> Result<pipeline::ShaderModule<A>, pipeline::CreateShaderModuleError> { + let (module, source) = match source { + #[cfg(feature = "wgsl")] + pipeline::ShaderModuleSource::Wgsl(code) => { + profiling::scope!("naga::front::wgsl::parse_str"); + let module = naga::front::wgsl::parse_str(&code).map_err(|inner| { + pipeline::CreateShaderModuleError::Parsing(pipeline::ShaderError { + source: code.to_string(), + label: desc.label.as_ref().map(|l| l.to_string()), + inner: Box::new(inner), + }) + })?; + (Cow::Owned(module), code.into_owned()) + } + #[cfg(feature = "spirv")] + pipeline::ShaderModuleSource::SpirV(spv, options) => { + let parser = naga::front::spv::Frontend::new(spv.iter().cloned(), &options); + profiling::scope!("naga::front::spv::Frontend"); + let module = parser.parse().map_err(|inner| { + pipeline::CreateShaderModuleError::ParsingSpirV(pipeline::ShaderError { + source: String::new(), + label: desc.label.as_ref().map(|l| l.to_string()), + inner: Box::new(inner), + }) + })?; + (Cow::Owned(module), String::new()) + } + #[cfg(feature = "glsl")] + pipeline::ShaderModuleSource::Glsl(code, options) => { + let mut parser = naga::front::glsl::Frontend::default(); + profiling::scope!("naga::front::glsl::Frontend.parse"); + let module = parser.parse(&options, &code).map_err(|inner| { + pipeline::CreateShaderModuleError::ParsingGlsl(pipeline::ShaderError { + source: code.to_string(), + label: desc.label.as_ref().map(|l| l.to_string()), + inner: Box::new(inner), + }) + })?; + (Cow::Owned(module), code.into_owned()) + } + pipeline::ShaderModuleSource::Naga(module) => (module, String::new()), + pipeline::ShaderModuleSource::Dummy(_) => panic!("found `ShaderModuleSource::Dummy`"), + }; + for (_, var) in module.global_variables.iter() { + match var.binding { + Some(ref br) if br.group >= self.limits.max_bind_groups => { + return Err(pipeline::CreateShaderModuleError::InvalidGroupIndex { + bind: br.clone(), + group: br.group, + limit: self.limits.max_bind_groups, + }); + } + _ => continue, + }; + } + + use naga::valid::Capabilities as Caps; + profiling::scope!("naga::validate"); + + let mut caps = Caps::empty(); + caps.set( + Caps::PUSH_CONSTANT, + self.features.contains(wgt::Features::PUSH_CONSTANTS), + ); + caps.set( + Caps::FLOAT64, + self.features.contains(wgt::Features::SHADER_F64), + ); + caps.set( + Caps::PRIMITIVE_INDEX, + self.features + .contains(wgt::Features::SHADER_PRIMITIVE_INDEX), + ); + caps.set( + Caps::SAMPLED_TEXTURE_AND_STORAGE_BUFFER_ARRAY_NON_UNIFORM_INDEXING, + self.features.contains( + wgt::Features::SAMPLED_TEXTURE_AND_STORAGE_BUFFER_ARRAY_NON_UNIFORM_INDEXING, + ), + ); + caps.set( + Caps::UNIFORM_BUFFER_AND_STORAGE_TEXTURE_ARRAY_NON_UNIFORM_INDEXING, + self.features.contains( + wgt::Features::UNIFORM_BUFFER_AND_STORAGE_TEXTURE_ARRAY_NON_UNIFORM_INDEXING, + ), + ); + // TODO: This needs a proper wgpu feature + caps.set( + Caps::SAMPLER_NON_UNIFORM_INDEXING, + self.features.contains( + wgt::Features::SAMPLED_TEXTURE_AND_STORAGE_BUFFER_ARRAY_NON_UNIFORM_INDEXING, + ), + ); + caps.set( + Caps::STORAGE_TEXTURE_16BIT_NORM_FORMATS, + self.features + .contains(wgt::Features::TEXTURE_FORMAT_16BIT_NORM), + ); + caps.set( + Caps::MULTIVIEW, + self.features.contains(wgt::Features::MULTIVIEW), + ); + caps.set( + Caps::EARLY_DEPTH_TEST, + self.features + .contains(wgt::Features::SHADER_EARLY_DEPTH_TEST), + ); + caps.set( + Caps::MULTISAMPLED_SHADING, + self.downlevel + .flags + .contains(wgt::DownlevelFlags::MULTISAMPLED_SHADING), + ); + caps.set( + Caps::DUAL_SOURCE_BLENDING, + self.features.contains(wgt::Features::DUAL_SOURCE_BLENDING), + ); + caps.set( + Caps::CUBE_ARRAY_TEXTURES, + self.downlevel + .flags + .contains(wgt::DownlevelFlags::CUBE_ARRAY_TEXTURES), + ); + + let debug_source = + if self.instance_flags.contains(wgt::InstanceFlags::DEBUG) && !source.is_empty() { + Some(hal::DebugSource { + file_name: Cow::Owned( + desc.label + .as_ref() + .map_or("shader".to_string(), |l| l.to_string()), + ), + source_code: Cow::Owned(source.clone()), + }) + } else { + None + }; + + let info = naga::valid::Validator::new(naga::valid::ValidationFlags::all(), caps) + .validate(&module) + .map_err(|inner| { + pipeline::CreateShaderModuleError::Validation(pipeline::ShaderError { + source, + label: desc.label.as_ref().map(|l| l.to_string()), + inner: Box::new(inner), + }) + })?; + + let interface = + validation::Interface::new(&module, &info, self.limits.clone(), self.features); + let hal_shader = hal::ShaderInput::Naga(hal::NagaShader { + module, + info, + debug_source, + }); + let hal_desc = hal::ShaderModuleDescriptor { + label: desc.label.to_hal(self.instance_flags), + runtime_checks: desc.shader_bound_checks.runtime_checks(), + }; + let raw = match unsafe { + self.raw + .as_ref() + .unwrap() + .create_shader_module(&hal_desc, hal_shader) + } { + Ok(raw) => raw, + Err(error) => { + return Err(match error { + hal::ShaderError::Device(error) => { + pipeline::CreateShaderModuleError::Device(error.into()) + } + hal::ShaderError::Compilation(ref msg) => { + log::error!("Shader error: {}", msg); + pipeline::CreateShaderModuleError::Generation + } + }) + } + }; + + Ok(pipeline::ShaderModule { + raw: Some(raw), + device: self.clone(), + interface: Some(interface), + info: ResourceInfo::new(desc.label.borrow_or_default()), + label: desc.label.borrow_or_default().to_string(), + }) + } + + #[allow(unused_unsafe)] + pub(crate) unsafe fn create_shader_module_spirv<'a>( + self: &Arc<Self>, + desc: &pipeline::ShaderModuleDescriptor<'a>, + source: &'a [u32], + ) -> Result<pipeline::ShaderModule<A>, pipeline::CreateShaderModuleError> { + self.require_features(wgt::Features::SPIRV_SHADER_PASSTHROUGH)?; + let hal_desc = hal::ShaderModuleDescriptor { + label: desc.label.to_hal(self.instance_flags), + runtime_checks: desc.shader_bound_checks.runtime_checks(), + }; + let hal_shader = hal::ShaderInput::SpirV(source); + let raw = match unsafe { + self.raw + .as_ref() + .unwrap() + .create_shader_module(&hal_desc, hal_shader) + } { + Ok(raw) => raw, + Err(error) => { + return Err(match error { + hal::ShaderError::Device(error) => { + pipeline::CreateShaderModuleError::Device(error.into()) + } + hal::ShaderError::Compilation(ref msg) => { + log::error!("Shader error: {}", msg); + pipeline::CreateShaderModuleError::Generation + } + }) + } + }; + + Ok(pipeline::ShaderModule { + raw: Some(raw), + device: self.clone(), + interface: None, + info: ResourceInfo::new(desc.label.borrow_or_default()), + label: desc.label.borrow_or_default().to_string(), + }) + } + + /// Generate information about late-validated buffer bindings for pipelines. + //TODO: should this be combined with `get_introspection_bind_group_layouts` in some way? + pub(crate) fn make_late_sized_buffer_groups( + shader_binding_sizes: &FastHashMap<naga::ResourceBinding, wgt::BufferSize>, + layout: &binding_model::PipelineLayout<A>, + ) -> ArrayVec<pipeline::LateSizedBufferGroup, { hal::MAX_BIND_GROUPS }> { + // Given the shader-required binding sizes and the pipeline layout, + // return the filtered list of them in the layout order, + // removing those with given `min_binding_size`. + layout + .bind_group_layouts + .iter() + .enumerate() + .map(|(group_index, bgl)| pipeline::LateSizedBufferGroup { + shader_sizes: bgl + .entries + .values() + .filter_map(|entry| match entry.ty { + wgt::BindingType::Buffer { + min_binding_size: None, + .. + } => { + let rb = naga::ResourceBinding { + group: group_index as u32, + binding: entry.binding, + }; + let shader_size = + shader_binding_sizes.get(&rb).map_or(0, |nz| nz.get()); + Some(shader_size) + } + _ => None, + }) + .collect(), + }) + .collect() + } + + pub(crate) fn create_bind_group_layout( + self: &Arc<Self>, + label: &crate::Label, + entry_map: bgl::EntryMap, + origin: bgl::Origin, + ) -> Result<BindGroupLayout<A>, binding_model::CreateBindGroupLayoutError> { + #[derive(PartialEq)] + enum WritableStorage { + Yes, + No, + } + + for entry in entry_map.values() { + use wgt::BindingType as Bt; + + let mut required_features = wgt::Features::empty(); + let mut required_downlevel_flags = wgt::DownlevelFlags::empty(); + let (array_feature, writable_storage) = match entry.ty { + Bt::Buffer { + ty: wgt::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: _, + } => ( + Some(wgt::Features::BUFFER_BINDING_ARRAY), + WritableStorage::No, + ), + Bt::Buffer { + ty: wgt::BufferBindingType::Uniform, + has_dynamic_offset: true, + min_binding_size: _, + } => ( + Some(wgt::Features::BUFFER_BINDING_ARRAY), + WritableStorage::No, + ), + Bt::Buffer { + ty: wgt::BufferBindingType::Storage { read_only }, + .. + } => ( + Some( + wgt::Features::BUFFER_BINDING_ARRAY + | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY, + ), + match read_only { + true => WritableStorage::No, + false => WritableStorage::Yes, + }, + ), + Bt::Sampler { .. } => ( + Some(wgt::Features::TEXTURE_BINDING_ARRAY), + WritableStorage::No, + ), + Bt::Texture { + multisampled: true, + sample_type: TextureSampleType::Float { filterable: true }, + .. + } => { + return Err(binding_model::CreateBindGroupLayoutError::Entry { + binding: entry.binding, + error: + BindGroupLayoutEntryError::SampleTypeFloatFilterableBindingMultisampled, + }); + } + Bt::Texture { .. } => ( + Some(wgt::Features::TEXTURE_BINDING_ARRAY), + WritableStorage::No, + ), + Bt::StorageTexture { + access, + view_dimension, + format: _, + } => { + match view_dimension { + wgt::TextureViewDimension::Cube | wgt::TextureViewDimension::CubeArray => { + return Err(binding_model::CreateBindGroupLayoutError::Entry { + binding: entry.binding, + error: BindGroupLayoutEntryError::StorageTextureCube, + }) + } + _ => (), + } + match access { + wgt::StorageTextureAccess::ReadOnly + | wgt::StorageTextureAccess::ReadWrite + if !self.features.contains( + wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES, + ) => + { + return Err(binding_model::CreateBindGroupLayoutError::Entry { + binding: entry.binding, + error: BindGroupLayoutEntryError::StorageTextureReadWrite, + }); + } + _ => (), + } + ( + Some( + wgt::Features::TEXTURE_BINDING_ARRAY + | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY, + ), + match access { + wgt::StorageTextureAccess::WriteOnly => WritableStorage::Yes, + wgt::StorageTextureAccess::ReadOnly => { + required_features |= + wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES; + WritableStorage::No + } + wgt::StorageTextureAccess::ReadWrite => { + required_features |= + wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES; + WritableStorage::Yes + } + }, + ) + } + Bt::AccelerationStructure => todo!(), + }; + + // Validate the count parameter + if entry.count.is_some() { + required_features |= array_feature + .ok_or(BindGroupLayoutEntryError::ArrayUnsupported) + .map_err(|error| binding_model::CreateBindGroupLayoutError::Entry { + binding: entry.binding, + error, + })?; + } + + if entry.visibility.contains_invalid_bits() { + return Err( + binding_model::CreateBindGroupLayoutError::InvalidVisibility(entry.visibility), + ); + } + + if entry.visibility.contains(wgt::ShaderStages::VERTEX) { + if writable_storage == WritableStorage::Yes { + required_features |= wgt::Features::VERTEX_WRITABLE_STORAGE; + } + if let Bt::Buffer { + ty: wgt::BufferBindingType::Storage { .. }, + .. + } = entry.ty + { + required_downlevel_flags |= wgt::DownlevelFlags::VERTEX_STORAGE; + } + } + if writable_storage == WritableStorage::Yes + && entry.visibility.contains(wgt::ShaderStages::FRAGMENT) + { + required_downlevel_flags |= wgt::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE; + } + + self.require_features(required_features) + .map_err(BindGroupLayoutEntryError::MissingFeatures) + .map_err(|error| binding_model::CreateBindGroupLayoutError::Entry { + binding: entry.binding, + error, + })?; + self.require_downlevel_flags(required_downlevel_flags) + .map_err(BindGroupLayoutEntryError::MissingDownlevelFlags) + .map_err(|error| binding_model::CreateBindGroupLayoutError::Entry { + binding: entry.binding, + error, + })?; + } + + let bgl_flags = conv::bind_group_layout_flags(self.features); + + let hal_bindings = entry_map.values().copied().collect::<Vec<_>>(); + let label = label.to_hal(self.instance_flags); + let hal_desc = hal::BindGroupLayoutDescriptor { + label, + flags: bgl_flags, + entries: &hal_bindings, + }; + let raw = unsafe { + self.raw + .as_ref() + .unwrap() + .create_bind_group_layout(&hal_desc) + .map_err(DeviceError::from)? + }; + + let mut count_validator = binding_model::BindingTypeMaxCountValidator::default(); + for entry in entry_map.values() { + count_validator.add_binding(entry); + } + // If a single bind group layout violates limits, the pipeline layout is + // definitely going to violate limits too, lets catch it now. + count_validator + .validate(&self.limits) + .map_err(binding_model::CreateBindGroupLayoutError::TooManyBindings)?; + + Ok(BindGroupLayout { + raw: Some(raw), + device: self.clone(), + entries: entry_map, + origin, + binding_count_validator: count_validator, + info: ResourceInfo::new(label.unwrap_or("<BindGroupLayout>")), + label: label.unwrap_or_default().to_string(), + }) + } + + pub(crate) fn create_buffer_binding<'a>( + bb: &binding_model::BufferBinding, + binding: u32, + decl: &wgt::BindGroupLayoutEntry, + used_buffer_ranges: &mut Vec<BufferInitTrackerAction<A>>, + dynamic_binding_info: &mut Vec<binding_model::BindGroupDynamicBindingData>, + late_buffer_binding_sizes: &mut FastHashMap<u32, wgt::BufferSize>, + used: &mut BindGroupStates<A>, + storage: &'a Storage<Buffer<A>>, + limits: &wgt::Limits, + snatch_guard: &'a SnatchGuard<'a>, + ) -> Result<hal::BufferBinding<'a, A>, binding_model::CreateBindGroupError> { + use crate::binding_model::CreateBindGroupError as Error; + + let (binding_ty, dynamic, min_size) = match decl.ty { + wgt::BindingType::Buffer { + ty, + has_dynamic_offset, + min_binding_size, + } => (ty, has_dynamic_offset, min_binding_size), + _ => { + return Err(Error::WrongBindingType { + binding, + actual: decl.ty, + expected: "UniformBuffer, StorageBuffer or ReadonlyStorageBuffer", + }) + } + }; + let (pub_usage, internal_use, range_limit) = match binding_ty { + wgt::BufferBindingType::Uniform => ( + wgt::BufferUsages::UNIFORM, + hal::BufferUses::UNIFORM, + limits.max_uniform_buffer_binding_size, + ), + wgt::BufferBindingType::Storage { read_only } => ( + wgt::BufferUsages::STORAGE, + if read_only { + hal::BufferUses::STORAGE_READ + } else { + hal::BufferUses::STORAGE_READ_WRITE + }, + limits.max_storage_buffer_binding_size, + ), + }; + + let (align, align_limit_name) = + binding_model::buffer_binding_type_alignment(limits, binding_ty); + if bb.offset % align as u64 != 0 { + return Err(Error::UnalignedBufferOffset( + bb.offset, + align_limit_name, + align, + )); + } + + let buffer = used + .buffers + .add_single(storage, bb.buffer_id, internal_use) + .ok_or(Error::InvalidBuffer(bb.buffer_id))?; + + check_buffer_usage(buffer.usage, pub_usage)?; + let raw_buffer = buffer + .raw + .get(snatch_guard) + .ok_or(Error::InvalidBuffer(bb.buffer_id))?; + + let (bind_size, bind_end) = match bb.size { + Some(size) => { + let end = bb.offset + size.get(); + if end > buffer.size { + return Err(Error::BindingRangeTooLarge { + buffer: bb.buffer_id, + range: bb.offset..end, + size: buffer.size, + }); + } + (size.get(), end) + } + None => { + if buffer.size < bb.offset { + return Err(Error::BindingRangeTooLarge { + buffer: bb.buffer_id, + range: bb.offset..bb.offset, + size: buffer.size, + }); + } + (buffer.size - bb.offset, buffer.size) + } + }; + + if bind_size > range_limit as u64 { + return Err(Error::BufferRangeTooLarge { + binding, + given: bind_size as u32, + limit: range_limit, + }); + } + + // Record binding info for validating dynamic offsets + if dynamic { + dynamic_binding_info.push(binding_model::BindGroupDynamicBindingData { + binding_idx: binding, + buffer_size: buffer.size, + binding_range: bb.offset..bind_end, + maximum_dynamic_offset: buffer.size - bind_end, + binding_type: binding_ty, + }); + } + + if let Some(non_zero) = min_size { + let min_size = non_zero.get(); + if min_size > bind_size { + return Err(Error::BindingSizeTooSmall { + buffer: bb.buffer_id, + actual: bind_size, + min: min_size, + }); + } + } else { + let late_size = + wgt::BufferSize::new(bind_size).ok_or(Error::BindingZeroSize(bb.buffer_id))?; + late_buffer_binding_sizes.insert(binding, late_size); + } + + assert_eq!(bb.offset % wgt::COPY_BUFFER_ALIGNMENT, 0); + used_buffer_ranges.extend(buffer.initialization_status.read().create_action( + buffer, + bb.offset..bb.offset + bind_size, + MemoryInitKind::NeedsInitializedMemory, + )); + + Ok(hal::BufferBinding { + buffer: raw_buffer, + offset: bb.offset, + size: bb.size, + }) + } + + pub(crate) fn create_texture_binding( + view: &TextureView<A>, + internal_use: hal::TextureUses, + pub_usage: wgt::TextureUsages, + used: &mut BindGroupStates<A>, + used_texture_ranges: &mut Vec<TextureInitTrackerAction<A>>, + ) -> Result<(), binding_model::CreateBindGroupError> { + let texture = &view.parent; + let texture_id = texture.as_info().id(); + // Careful here: the texture may no longer have its own ref count, + // if it was deleted by the user. + let texture = used + .textures + .add_single(texture, Some(view.selector.clone()), internal_use) + .ok_or(binding_model::CreateBindGroupError::InvalidTexture( + texture_id, + ))?; + + if texture.device.as_info().id() != view.device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + + check_texture_usage(texture.desc.usage, pub_usage)?; + + used_texture_ranges.push(TextureInitTrackerAction { + texture: texture.clone(), + range: TextureInitRange { + mip_range: view.desc.range.mip_range(texture.desc.mip_level_count), + layer_range: view + .desc + .range + .layer_range(texture.desc.array_layer_count()), + }, + kind: MemoryInitKind::NeedsInitializedMemory, + }); + + Ok(()) + } + + // This function expects the provided bind group layout to be resolved + // (not passing a duplicate) beforehand. + pub(crate) fn create_bind_group( + self: &Arc<Self>, + layout: &Arc<BindGroupLayout<A>>, + desc: &binding_model::BindGroupDescriptor, + hub: &Hub<A>, + ) -> Result<BindGroup<A>, binding_model::CreateBindGroupError> { + use crate::binding_model::{BindingResource as Br, CreateBindGroupError as Error}; + { + // Check that the number of entries in the descriptor matches + // the number of entries in the layout. + let actual = desc.entries.len(); + let expected = layout.entries.len(); + if actual != expected { + return Err(Error::BindingsNumMismatch { expected, actual }); + } + } + + // TODO: arrayvec/smallvec, or re-use allocations + // Record binding info for dynamic offset validation + let mut dynamic_binding_info = Vec::new(); + // Map of binding -> shader reflected size + //Note: we can't collect into a vector right away because + // it needs to be in BGL iteration order, not BG entry order. + let mut late_buffer_binding_sizes = FastHashMap::default(); + // fill out the descriptors + let mut used = BindGroupStates::new(); + + let buffer_guard = hub.buffers.read(); + let texture_view_guard = hub.texture_views.read(); + let sampler_guard = hub.samplers.read(); + + let mut used_buffer_ranges = Vec::new(); + let mut used_texture_ranges = Vec::new(); + let mut hal_entries = Vec::with_capacity(desc.entries.len()); + let mut hal_buffers = Vec::new(); + let mut hal_samplers = Vec::new(); + let mut hal_textures = Vec::new(); + let snatch_guard = self.snatchable_lock.read(); + for entry in desc.entries.iter() { + let binding = entry.binding; + // Find the corresponding declaration in the layout + let decl = layout + .entries + .get(binding) + .ok_or(Error::MissingBindingDeclaration(binding))?; + let (res_index, count) = match entry.resource { + Br::Buffer(ref bb) => { + let bb = Self::create_buffer_binding( + bb, + binding, + decl, + &mut used_buffer_ranges, + &mut dynamic_binding_info, + &mut late_buffer_binding_sizes, + &mut used, + &*buffer_guard, + &self.limits, + &snatch_guard, + )?; + + let res_index = hal_buffers.len(); + hal_buffers.push(bb); + (res_index, 1) + } + Br::BufferArray(ref bindings_array) => { + let num_bindings = bindings_array.len(); + Self::check_array_binding(self.features, decl.count, num_bindings)?; + + let res_index = hal_buffers.len(); + for bb in bindings_array.iter() { + let bb = Self::create_buffer_binding( + bb, + binding, + decl, + &mut used_buffer_ranges, + &mut dynamic_binding_info, + &mut late_buffer_binding_sizes, + &mut used, + &*buffer_guard, + &self.limits, + &snatch_guard, + )?; + hal_buffers.push(bb); + } + (res_index, num_bindings) + } + Br::Sampler(id) => { + match decl.ty { + wgt::BindingType::Sampler(ty) => { + let sampler = used + .samplers + .add_single(&*sampler_guard, id) + .ok_or(Error::InvalidSampler(id))?; + + if sampler.device.as_info().id() != self.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + + // Allowed sampler values for filtering and comparison + let (allowed_filtering, allowed_comparison) = match ty { + wgt::SamplerBindingType::Filtering => (None, false), + wgt::SamplerBindingType::NonFiltering => (Some(false), false), + wgt::SamplerBindingType::Comparison => (None, true), + }; + + if let Some(allowed_filtering) = allowed_filtering { + if allowed_filtering != sampler.filtering { + return Err(Error::WrongSamplerFiltering { + binding, + layout_flt: allowed_filtering, + sampler_flt: sampler.filtering, + }); + } + } + + if allowed_comparison != sampler.comparison { + return Err(Error::WrongSamplerComparison { + binding, + layout_cmp: allowed_comparison, + sampler_cmp: sampler.comparison, + }); + } + + let res_index = hal_samplers.len(); + hal_samplers.push(sampler.raw()); + (res_index, 1) + } + _ => { + return Err(Error::WrongBindingType { + binding, + actual: decl.ty, + expected: "Sampler", + }) + } + } + } + Br::SamplerArray(ref bindings_array) => { + let num_bindings = bindings_array.len(); + Self::check_array_binding(self.features, decl.count, num_bindings)?; + + let res_index = hal_samplers.len(); + for &id in bindings_array.iter() { + let sampler = used + .samplers + .add_single(&*sampler_guard, id) + .ok_or(Error::InvalidSampler(id))?; + if sampler.device.as_info().id() != self.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + hal_samplers.push(sampler.raw()); + } + + (res_index, num_bindings) + } + Br::TextureView(id) => { + let view = used + .views + .add_single(&*texture_view_guard, id) + .ok_or(Error::InvalidTextureView(id))?; + let (pub_usage, internal_use) = self.texture_use_parameters( + binding, + decl, + view, + "SampledTexture, ReadonlyStorageTexture or WriteonlyStorageTexture", + )?; + Self::create_texture_binding( + view, + internal_use, + pub_usage, + &mut used, + &mut used_texture_ranges, + )?; + let res_index = hal_textures.len(); + hal_textures.push(hal::TextureBinding { + view: view + .raw(&snatch_guard) + .ok_or(Error::InvalidTextureView(id))?, + usage: internal_use, + }); + (res_index, 1) + } + Br::TextureViewArray(ref bindings_array) => { + let num_bindings = bindings_array.len(); + Self::check_array_binding(self.features, decl.count, num_bindings)?; + + let res_index = hal_textures.len(); + for &id in bindings_array.iter() { + let view = used + .views + .add_single(&*texture_view_guard, id) + .ok_or(Error::InvalidTextureView(id))?; + let (pub_usage, internal_use) = + self.texture_use_parameters(binding, decl, view, + "SampledTextureArray, ReadonlyStorageTextureArray or WriteonlyStorageTextureArray")?; + Self::create_texture_binding( + view, + internal_use, + pub_usage, + &mut used, + &mut used_texture_ranges, + )?; + hal_textures.push(hal::TextureBinding { + view: view + .raw(&snatch_guard) + .ok_or(Error::InvalidTextureView(id))?, + usage: internal_use, + }); + } + + (res_index, num_bindings) + } + }; + + hal_entries.push(hal::BindGroupEntry { + binding, + resource_index: res_index as u32, + count: count as u32, + }); + } + + used.optimize(); + + hal_entries.sort_by_key(|entry| entry.binding); + for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) { + if a.binding == b.binding { + return Err(Error::DuplicateBinding(a.binding)); + } + } + let hal_desc = hal::BindGroupDescriptor { + label: desc.label.to_hal(self.instance_flags), + layout: layout.raw(), + entries: &hal_entries, + buffers: &hal_buffers, + samplers: &hal_samplers, + textures: &hal_textures, + acceleration_structures: &[], + }; + let raw = unsafe { + self.raw + .as_ref() + .unwrap() + .create_bind_group(&hal_desc) + .map_err(DeviceError::from)? + }; + + Ok(BindGroup { + raw: Snatchable::new(raw), + device: self.clone(), + layout: layout.clone(), + info: ResourceInfo::new(desc.label.borrow_or_default()), + used, + used_buffer_ranges, + used_texture_ranges, + dynamic_binding_info, + // collect in the order of BGL iteration + late_buffer_binding_sizes: layout + .entries + .indices() + .flat_map(|binding| late_buffer_binding_sizes.get(&binding).cloned()) + .collect(), + }) + } + + pub(crate) fn check_array_binding( + features: wgt::Features, + count: Option<NonZeroU32>, + num_bindings: usize, + ) -> Result<(), super::binding_model::CreateBindGroupError> { + use super::binding_model::CreateBindGroupError as Error; + + if let Some(count) = count { + let count = count.get() as usize; + if count < num_bindings { + return Err(Error::BindingArrayPartialLengthMismatch { + actual: num_bindings, + expected: count, + }); + } + if count != num_bindings + && !features.contains(wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY) + { + return Err(Error::BindingArrayLengthMismatch { + actual: num_bindings, + expected: count, + }); + } + if num_bindings == 0 { + return Err(Error::BindingArrayZeroLength); + } + } else { + return Err(Error::SingleBindingExpected); + }; + + Ok(()) + } + + pub(crate) fn texture_use_parameters( + self: &Arc<Self>, + binding: u32, + decl: &wgt::BindGroupLayoutEntry, + view: &TextureView<A>, + expected: &'static str, + ) -> Result<(wgt::TextureUsages, hal::TextureUses), binding_model::CreateBindGroupError> { + use crate::binding_model::CreateBindGroupError as Error; + if view + .desc + .aspects() + .contains(hal::FormatAspects::DEPTH | hal::FormatAspects::STENCIL) + { + return Err(Error::DepthStencilAspect); + } + match decl.ty { + wgt::BindingType::Texture { + sample_type, + view_dimension, + multisampled, + } => { + use wgt::TextureSampleType as Tst; + if multisampled != (view.samples != 1) { + return Err(Error::InvalidTextureMultisample { + binding, + layout_multisampled: multisampled, + view_samples: view.samples, + }); + } + let compat_sample_type = view + .desc + .format + .sample_type(Some(view.desc.range.aspect), Some(self.features)) + .unwrap(); + match (sample_type, compat_sample_type) { + (Tst::Uint, Tst::Uint) | + (Tst::Sint, Tst::Sint) | + (Tst::Depth, Tst::Depth) | + // if we expect non-filterable, accept anything float + (Tst::Float { filterable: false }, Tst::Float { .. }) | + // if we expect filterable, require it + (Tst::Float { filterable: true }, Tst::Float { filterable: true }) | + // if we expect non-filterable, also accept depth + (Tst::Float { filterable: false }, Tst::Depth) => {} + // if we expect filterable, also accept Float that is defined as + // unfilterable if filterable feature is explicitly enabled (only hit + // if wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES is + // enabled) + (Tst::Float { filterable: true }, Tst::Float { .. }) if view.format_features.flags.contains(wgt::TextureFormatFeatureFlags::FILTERABLE) => {} + _ => { + return Err(Error::InvalidTextureSampleType { + binding, + layout_sample_type: sample_type, + view_format: view.desc.format, + }) + } + } + if view_dimension != view.desc.dimension { + return Err(Error::InvalidTextureDimension { + binding, + layout_dimension: view_dimension, + view_dimension: view.desc.dimension, + }); + } + Ok(( + wgt::TextureUsages::TEXTURE_BINDING, + hal::TextureUses::RESOURCE, + )) + } + wgt::BindingType::StorageTexture { + access, + format, + view_dimension, + } => { + if format != view.desc.format { + return Err(Error::InvalidStorageTextureFormat { + binding, + layout_format: format, + view_format: view.desc.format, + }); + } + if view_dimension != view.desc.dimension { + return Err(Error::InvalidTextureDimension { + binding, + layout_dimension: view_dimension, + view_dimension: view.desc.dimension, + }); + } + + let mip_level_count = view.selector.mips.end - view.selector.mips.start; + if mip_level_count != 1 { + return Err(Error::InvalidStorageTextureMipLevelCount { + binding, + mip_level_count, + }); + } + + let internal_use = match access { + wgt::StorageTextureAccess::WriteOnly => hal::TextureUses::STORAGE_READ_WRITE, + wgt::StorageTextureAccess::ReadOnly => { + if !view + .format_features + .flags + .contains(wgt::TextureFormatFeatureFlags::STORAGE_READ_WRITE) + { + return Err(Error::StorageReadNotSupported(view.desc.format)); + } + hal::TextureUses::STORAGE_READ + } + wgt::StorageTextureAccess::ReadWrite => { + if !view + .format_features + .flags + .contains(wgt::TextureFormatFeatureFlags::STORAGE_READ_WRITE) + { + return Err(Error::StorageReadNotSupported(view.desc.format)); + } + + hal::TextureUses::STORAGE_READ_WRITE + } + }; + Ok((wgt::TextureUsages::STORAGE_BINDING, internal_use)) + } + _ => Err(Error::WrongBindingType { + binding, + actual: decl.ty, + expected, + }), + } + } + + pub(crate) fn create_pipeline_layout( + self: &Arc<Self>, + desc: &binding_model::PipelineLayoutDescriptor, + bgl_registry: &Registry<BindGroupLayout<A>>, + ) -> Result<binding_model::PipelineLayout<A>, binding_model::CreatePipelineLayoutError> { + use crate::binding_model::CreatePipelineLayoutError as Error; + + let bind_group_layouts_count = desc.bind_group_layouts.len(); + let device_max_bind_groups = self.limits.max_bind_groups as usize; + if bind_group_layouts_count > device_max_bind_groups { + return Err(Error::TooManyGroups { + actual: bind_group_layouts_count, + max: device_max_bind_groups, + }); + } + + if !desc.push_constant_ranges.is_empty() { + self.require_features(wgt::Features::PUSH_CONSTANTS)?; + } + + let mut used_stages = wgt::ShaderStages::empty(); + for (index, pc) in desc.push_constant_ranges.iter().enumerate() { + if pc.stages.intersects(used_stages) { + return Err(Error::MoreThanOnePushConstantRangePerStage { + index, + provided: pc.stages, + intersected: pc.stages & used_stages, + }); + } + used_stages |= pc.stages; + + let device_max_pc_size = self.limits.max_push_constant_size; + if device_max_pc_size < pc.range.end { + return Err(Error::PushConstantRangeTooLarge { + index, + range: pc.range.clone(), + max: device_max_pc_size, + }); + } + + if pc.range.start % wgt::PUSH_CONSTANT_ALIGNMENT != 0 { + return Err(Error::MisalignedPushConstantRange { + index, + bound: pc.range.start, + }); + } + if pc.range.end % wgt::PUSH_CONSTANT_ALIGNMENT != 0 { + return Err(Error::MisalignedPushConstantRange { + index, + bound: pc.range.end, + }); + } + } + + let mut count_validator = binding_model::BindingTypeMaxCountValidator::default(); + + // Collect references to the BGLs + let mut bind_group_layouts = ArrayVec::new(); + for &id in desc.bind_group_layouts.iter() { + let Ok(bgl) = bgl_registry.get(id) else { + return Err(Error::InvalidBindGroupLayout(id)); + }; + + bind_group_layouts.push(bgl); + } + + // Validate total resource counts and check for a matching device + for bgl in &bind_group_layouts { + if bgl.device.as_info().id() != self.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + + count_validator.merge(&bgl.binding_count_validator); + } + + count_validator + .validate(&self.limits) + .map_err(Error::TooManyBindings)?; + + let raw_bind_group_layouts = bind_group_layouts + .iter() + .map(|bgl| bgl.raw()) + .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>(); + + let hal_desc = hal::PipelineLayoutDescriptor { + label: desc.label.to_hal(self.instance_flags), + flags: hal::PipelineLayoutFlags::FIRST_VERTEX_INSTANCE, + bind_group_layouts: &raw_bind_group_layouts, + push_constant_ranges: desc.push_constant_ranges.as_ref(), + }; + + let raw = unsafe { + self.raw + .as_ref() + .unwrap() + .create_pipeline_layout(&hal_desc) + .map_err(DeviceError::from)? + }; + + drop(raw_bind_group_layouts); + + Ok(binding_model::PipelineLayout { + raw: Some(raw), + device: self.clone(), + info: ResourceInfo::new(desc.label.borrow_or_default()), + bind_group_layouts, + push_constant_ranges: desc.push_constant_ranges.iter().cloned().collect(), + }) + } + + //TODO: refactor this. It's the only method of `Device` that registers new objects + // (the pipeline layout). + pub(crate) fn derive_pipeline_layout( + self: &Arc<Self>, + implicit_context: Option<ImplicitPipelineContext>, + mut derived_group_layouts: ArrayVec<bgl::EntryMap, { hal::MAX_BIND_GROUPS }>, + bgl_registry: &Registry<BindGroupLayout<A>>, + pipeline_layout_registry: &Registry<binding_model::PipelineLayout<A>>, + ) -> Result<Arc<binding_model::PipelineLayout<A>>, pipeline::ImplicitLayoutError> { + while derived_group_layouts + .last() + .map_or(false, |map| map.is_empty()) + { + derived_group_layouts.pop(); + } + let mut ids = implicit_context.ok_or(pipeline::ImplicitLayoutError::MissingIds(0))?; + let group_count = derived_group_layouts.len(); + if ids.group_ids.len() < group_count { + log::error!( + "Not enough bind group IDs ({}) specified for the implicit layout ({})", + ids.group_ids.len(), + derived_group_layouts.len() + ); + return Err(pipeline::ImplicitLayoutError::MissingIds(group_count as _)); + } + + for (bgl_id, map) in ids.group_ids.iter_mut().zip(derived_group_layouts) { + let bgl = self.create_bind_group_layout(&None, map, bgl::Origin::Derived)?; + bgl_registry.force_replace(*bgl_id, bgl); + } + + let layout_desc = binding_model::PipelineLayoutDescriptor { + label: None, + bind_group_layouts: Cow::Borrowed(&ids.group_ids[..group_count]), + push_constant_ranges: Cow::Borrowed(&[]), //TODO? + }; + let layout = self.create_pipeline_layout(&layout_desc, bgl_registry)?; + pipeline_layout_registry.force_replace(ids.root_id, layout); + Ok(pipeline_layout_registry.get(ids.root_id).unwrap()) + } + + pub(crate) fn create_compute_pipeline( + self: &Arc<Self>, + desc: &pipeline::ComputePipelineDescriptor, + implicit_context: Option<ImplicitPipelineContext>, + hub: &Hub<A>, + ) -> Result<pipeline::ComputePipeline<A>, pipeline::CreateComputePipelineError> { + // This has to be done first, or otherwise the IDs may be pointing to entries + // that are not even in the storage. + if let Some(ref ids) = implicit_context { + let mut pipeline_layout_guard = hub.pipeline_layouts.write(); + pipeline_layout_guard.insert_error(ids.root_id, IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL); + let mut bgl_guard = hub.bind_group_layouts.write(); + for &bgl_id in ids.group_ids.iter() { + bgl_guard.insert_error(bgl_id, IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL); + } + } + + self.require_downlevel_flags(wgt::DownlevelFlags::COMPUTE_SHADERS)?; + + let shader_module = hub + .shader_modules + .get(desc.stage.module) + .map_err(|_| validation::StageError::InvalidModule)?; + + if shader_module.device.as_info().id() != self.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + + // Get the pipeline layout from the desc if it is provided. + let pipeline_layout = match desc.layout { + Some(pipeline_layout_id) => { + let pipeline_layout = hub + .pipeline_layouts + .get(pipeline_layout_id) + .map_err(|_| pipeline::CreateComputePipelineError::InvalidLayout)?; + + if pipeline_layout.device.as_info().id() != self.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + + Some(pipeline_layout) + } + None => None, + }; + + let mut binding_layout_source = match pipeline_layout { + Some(ref pipeline_layout) => { + validation::BindingLayoutSource::Provided(pipeline_layout.get_binding_maps()) + } + None => validation::BindingLayoutSource::new_derived(&self.limits), + }; + let mut shader_binding_sizes = FastHashMap::default(); + let io = validation::StageIo::default(); + + { + let stage = wgt::ShaderStages::COMPUTE; + + if let Some(ref interface) = shader_module.interface { + let _ = interface.check_stage( + &mut binding_layout_source, + &mut shader_binding_sizes, + &desc.stage.entry_point, + stage, + io, + None, + )?; + } + } + + let pipeline_layout = match binding_layout_source { + validation::BindingLayoutSource::Provided(_) => { + drop(binding_layout_source); + pipeline_layout.unwrap() + } + validation::BindingLayoutSource::Derived(entries) => self.derive_pipeline_layout( + implicit_context, + entries, + &hub.bind_group_layouts, + &hub.pipeline_layouts, + )?, + }; + + let late_sized_buffer_groups = + Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout); + + let pipeline_desc = hal::ComputePipelineDescriptor { + label: desc.label.to_hal(self.instance_flags), + layout: pipeline_layout.raw(), + stage: hal::ProgrammableStage { + entry_point: desc.stage.entry_point.as_ref(), + module: shader_module.raw(), + }, + }; + + let raw = unsafe { + self.raw + .as_ref() + .unwrap() + .create_compute_pipeline(&pipeline_desc) + } + .map_err(|err| match err { + hal::PipelineError::Device(error) => { + pipeline::CreateComputePipelineError::Device(error.into()) + } + hal::PipelineError::Linkage(_stages, msg) => { + pipeline::CreateComputePipelineError::Internal(msg) + } + hal::PipelineError::EntryPoint(_stage) => { + pipeline::CreateComputePipelineError::Internal(ENTRYPOINT_FAILURE_ERROR.to_string()) + } + })?; + + let pipeline = pipeline::ComputePipeline { + raw: Some(raw), + layout: pipeline_layout, + device: self.clone(), + _shader_module: shader_module, + late_sized_buffer_groups, + info: ResourceInfo::new(desc.label.borrow_or_default()), + }; + Ok(pipeline) + } + + pub(crate) fn create_render_pipeline( + self: &Arc<Self>, + adapter: &Adapter<A>, + desc: &pipeline::RenderPipelineDescriptor, + implicit_context: Option<ImplicitPipelineContext>, + hub: &Hub<A>, + ) -> Result<pipeline::RenderPipeline<A>, pipeline::CreateRenderPipelineError> { + use wgt::TextureFormatFeatureFlags as Tfff; + + // This has to be done first, or otherwise the IDs may be pointing to entries + // that are not even in the storage. + if let Some(ref ids) = implicit_context { + //TODO: only lock mutable if the layout is derived + let mut pipeline_layout_guard = hub.pipeline_layouts.write(); + let mut bgl_guard = hub.bind_group_layouts.write(); + pipeline_layout_guard.insert_error(ids.root_id, IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL); + for &bgl_id in ids.group_ids.iter() { + bgl_guard.insert_error(bgl_id, IMPLICIT_BIND_GROUP_LAYOUT_ERROR_LABEL); + } + } + + let mut shader_binding_sizes = FastHashMap::default(); + + let num_attachments = desc.fragment.as_ref().map(|f| f.targets.len()).unwrap_or(0); + if num_attachments > hal::MAX_COLOR_ATTACHMENTS { + return Err(pipeline::CreateRenderPipelineError::ColorAttachment( + command::ColorAttachmentError::TooMany { + given: num_attachments, + limit: hal::MAX_COLOR_ATTACHMENTS, + }, + )); + } + + let color_targets = desc + .fragment + .as_ref() + .map_or(&[][..], |fragment| &fragment.targets); + let depth_stencil_state = desc.depth_stencil.as_ref(); + + let cts: ArrayVec<_, { hal::MAX_COLOR_ATTACHMENTS }> = + color_targets.iter().filter_map(|x| x.as_ref()).collect(); + if !cts.is_empty() && { + let first = &cts[0]; + cts[1..] + .iter() + .any(|ct| ct.write_mask != first.write_mask || ct.blend != first.blend) + } { + log::debug!("Color targets: {:?}", color_targets); + self.require_downlevel_flags(wgt::DownlevelFlags::INDEPENDENT_BLEND)?; + } + + let mut io = validation::StageIo::default(); + let mut validated_stages = wgt::ShaderStages::empty(); + + let mut vertex_steps = Vec::with_capacity(desc.vertex.buffers.len()); + let mut vertex_buffers = Vec::with_capacity(desc.vertex.buffers.len()); + let mut total_attributes = 0; + let mut shader_expects_dual_source_blending = false; + let mut pipeline_expects_dual_source_blending = false; + for (i, vb_state) in desc.vertex.buffers.iter().enumerate() { + let mut last_stride = 0; + for attribute in vb_state.attributes.iter() { + last_stride = last_stride.max(attribute.offset + attribute.format.size()); + } + vertex_steps.push(pipeline::VertexStep { + stride: vb_state.array_stride, + last_stride, + mode: vb_state.step_mode, + }); + if vb_state.attributes.is_empty() { + continue; + } + if vb_state.array_stride > self.limits.max_vertex_buffer_array_stride as u64 { + return Err(pipeline::CreateRenderPipelineError::VertexStrideTooLarge { + index: i as u32, + given: vb_state.array_stride as u32, + limit: self.limits.max_vertex_buffer_array_stride, + }); + } + if vb_state.array_stride % wgt::VERTEX_STRIDE_ALIGNMENT != 0 { + return Err(pipeline::CreateRenderPipelineError::UnalignedVertexStride { + index: i as u32, + stride: vb_state.array_stride, + }); + } + vertex_buffers.push(hal::VertexBufferLayout { + array_stride: vb_state.array_stride, + step_mode: vb_state.step_mode, + attributes: vb_state.attributes.as_ref(), + }); + + for attribute in vb_state.attributes.iter() { + if attribute.offset >= 0x10000000 { + return Err( + pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset { + location: attribute.shader_location, + offset: attribute.offset, + }, + ); + } + + if let wgt::VertexFormat::Float64 + | wgt::VertexFormat::Float64x2 + | wgt::VertexFormat::Float64x3 + | wgt::VertexFormat::Float64x4 = attribute.format + { + self.require_features(wgt::Features::VERTEX_ATTRIBUTE_64BIT)?; + } + + let previous = io.insert( + attribute.shader_location, + validation::InterfaceVar::vertex_attribute(attribute.format), + ); + + if previous.is_some() { + return Err(pipeline::CreateRenderPipelineError::ShaderLocationClash( + attribute.shader_location, + )); + } + } + total_attributes += vb_state.attributes.len(); + } + + if vertex_buffers.len() > self.limits.max_vertex_buffers as usize { + return Err(pipeline::CreateRenderPipelineError::TooManyVertexBuffers { + given: vertex_buffers.len() as u32, + limit: self.limits.max_vertex_buffers, + }); + } + if total_attributes > self.limits.max_vertex_attributes as usize { + return Err( + pipeline::CreateRenderPipelineError::TooManyVertexAttributes { + given: total_attributes as u32, + limit: self.limits.max_vertex_attributes, + }, + ); + } + + if desc.primitive.strip_index_format.is_some() && !desc.primitive.topology.is_strip() { + return Err( + pipeline::CreateRenderPipelineError::StripIndexFormatForNonStripTopology { + strip_index_format: desc.primitive.strip_index_format, + topology: desc.primitive.topology, + }, + ); + } + + if desc.primitive.unclipped_depth { + self.require_features(wgt::Features::DEPTH_CLIP_CONTROL)?; + } + + if desc.primitive.polygon_mode == wgt::PolygonMode::Line { + self.require_features(wgt::Features::POLYGON_MODE_LINE)?; + } + if desc.primitive.polygon_mode == wgt::PolygonMode::Point { + self.require_features(wgt::Features::POLYGON_MODE_POINT)?; + } + + if desc.primitive.conservative { + self.require_features(wgt::Features::CONSERVATIVE_RASTERIZATION)?; + } + + if desc.primitive.conservative && desc.primitive.polygon_mode != wgt::PolygonMode::Fill { + return Err( + pipeline::CreateRenderPipelineError::ConservativeRasterizationNonFillPolygonMode, + ); + } + + for (i, cs) in color_targets.iter().enumerate() { + if let Some(cs) = cs.as_ref() { + let error = loop { + if cs.write_mask.contains_invalid_bits() { + break Some(pipeline::ColorStateError::InvalidWriteMask(cs.write_mask)); + } + + let format_features = self.describe_format_features(adapter, cs.format)?; + if !format_features + .allowed_usages + .contains(wgt::TextureUsages::RENDER_ATTACHMENT) + { + break Some(pipeline::ColorStateError::FormatNotRenderable(cs.format)); + } + let blendable = format_features.flags.contains(Tfff::BLENDABLE); + let filterable = format_features.flags.contains(Tfff::FILTERABLE); + let adapter_specific = self + .features + .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES); + // according to WebGPU specifications the texture needs to be + // [`TextureFormatFeatureFlags::FILTERABLE`] if blending is set - use + // [`Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES`] to elude + // this limitation + if cs.blend.is_some() && (!blendable || (!filterable && !adapter_specific)) { + break Some(pipeline::ColorStateError::FormatNotBlendable(cs.format)); + } + if !hal::FormatAspects::from(cs.format).contains(hal::FormatAspects::COLOR) { + break Some(pipeline::ColorStateError::FormatNotColor(cs.format)); + } + if desc.multisample.count > 1 + && !format_features + .flags + .sample_count_supported(desc.multisample.count) + { + break Some(pipeline::ColorStateError::InvalidSampleCount( + desc.multisample.count, + cs.format, + cs.format + .guaranteed_format_features(self.features) + .flags + .supported_sample_counts(), + adapter + .get_texture_format_features(cs.format) + .flags + .supported_sample_counts(), + )); + } + if let Some(blend_mode) = cs.blend { + for factor in [ + blend_mode.color.src_factor, + blend_mode.color.dst_factor, + blend_mode.alpha.src_factor, + blend_mode.alpha.dst_factor, + ] { + if factor.ref_second_blend_source() { + self.require_features(wgt::Features::DUAL_SOURCE_BLENDING)?; + if i == 0 { + pipeline_expects_dual_source_blending = true; + break; + } else { + return Err(crate::pipeline::CreateRenderPipelineError + ::BlendFactorOnUnsupportedTarget { factor, target: i as u32 }); + } + } + } + } + break None; + }; + if let Some(e) = error { + return Err(pipeline::CreateRenderPipelineError::ColorState(i as u8, e)); + } + } + } + + if let Some(ds) = depth_stencil_state { + let error = loop { + let format_features = self.describe_format_features(adapter, ds.format)?; + if !format_features + .allowed_usages + .contains(wgt::TextureUsages::RENDER_ATTACHMENT) + { + break Some(pipeline::DepthStencilStateError::FormatNotRenderable( + ds.format, + )); + } + + let aspect = hal::FormatAspects::from(ds.format); + if ds.is_depth_enabled() && !aspect.contains(hal::FormatAspects::DEPTH) { + break Some(pipeline::DepthStencilStateError::FormatNotDepth(ds.format)); + } + if ds.stencil.is_enabled() && !aspect.contains(hal::FormatAspects::STENCIL) { + break Some(pipeline::DepthStencilStateError::FormatNotStencil( + ds.format, + )); + } + if desc.multisample.count > 1 + && !format_features + .flags + .sample_count_supported(desc.multisample.count) + { + break Some(pipeline::DepthStencilStateError::InvalidSampleCount( + desc.multisample.count, + ds.format, + ds.format + .guaranteed_format_features(self.features) + .flags + .supported_sample_counts(), + adapter + .get_texture_format_features(ds.format) + .flags + .supported_sample_counts(), + )); + } + + break None; + }; + if let Some(e) = error { + return Err(pipeline::CreateRenderPipelineError::DepthStencilState(e)); + } + + if ds.bias.clamp != 0.0 { + self.require_downlevel_flags(wgt::DownlevelFlags::DEPTH_BIAS_CLAMP)?; + } + } + + // Get the pipeline layout from the desc if it is provided. + let pipeline_layout = match desc.layout { + Some(pipeline_layout_id) => { + let pipeline_layout = hub + .pipeline_layouts + .get(pipeline_layout_id) + .map_err(|_| pipeline::CreateRenderPipelineError::InvalidLayout)?; + + if pipeline_layout.device.as_info().id() != self.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + + Some(pipeline_layout) + } + None => None, + }; + + let mut binding_layout_source = match pipeline_layout { + Some(ref pipeline_layout) => { + validation::BindingLayoutSource::Provided(pipeline_layout.get_binding_maps()) + } + None => validation::BindingLayoutSource::new_derived(&self.limits), + }; + + let samples = { + let sc = desc.multisample.count; + if sc == 0 || sc > 32 || !conv::is_power_of_two_u32(sc) { + return Err(pipeline::CreateRenderPipelineError::InvalidSampleCount(sc)); + } + sc + }; + + let vertex_shader_module; + let vertex_stage = { + let stage_desc = &desc.vertex.stage; + let stage = wgt::ShaderStages::VERTEX; + + vertex_shader_module = hub.shader_modules.get(stage_desc.module).map_err(|_| { + pipeline::CreateRenderPipelineError::Stage { + stage, + error: validation::StageError::InvalidModule, + } + })?; + if vertex_shader_module.device.as_info().id() != self.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + + if let Some(ref interface) = vertex_shader_module.interface { + io = interface + .check_stage( + &mut binding_layout_source, + &mut shader_binding_sizes, + &stage_desc.entry_point, + stage, + io, + desc.depth_stencil.as_ref().map(|d| d.depth_compare), + ) + .map_err(|error| pipeline::CreateRenderPipelineError::Stage { stage, error })?; + validated_stages |= stage; + } + + hal::ProgrammableStage { + module: vertex_shader_module.raw(), + entry_point: stage_desc.entry_point.as_ref(), + } + }; + + let mut fragment_shader_module = None; + let fragment_stage = match desc.fragment { + Some(ref fragment_state) => { + let stage = wgt::ShaderStages::FRAGMENT; + + let shader_module = fragment_shader_module.insert( + hub.shader_modules + .get(fragment_state.stage.module) + .map_err(|_| pipeline::CreateRenderPipelineError::Stage { + stage, + error: validation::StageError::InvalidModule, + })?, + ); + + if validated_stages == wgt::ShaderStages::VERTEX { + if let Some(ref interface) = shader_module.interface { + io = interface + .check_stage( + &mut binding_layout_source, + &mut shader_binding_sizes, + &fragment_state.stage.entry_point, + stage, + io, + desc.depth_stencil.as_ref().map(|d| d.depth_compare), + ) + .map_err(|error| pipeline::CreateRenderPipelineError::Stage { + stage, + error, + })?; + validated_stages |= stage; + } + } + + if let Some(ref interface) = shader_module.interface { + shader_expects_dual_source_blending = interface + .fragment_uses_dual_source_blending(&fragment_state.stage.entry_point) + .map_err(|error| pipeline::CreateRenderPipelineError::Stage { + stage, + error, + })?; + } + + Some(hal::ProgrammableStage { + module: shader_module.raw(), + entry_point: fragment_state.stage.entry_point.as_ref(), + }) + } + None => None, + }; + + if !pipeline_expects_dual_source_blending && shader_expects_dual_source_blending { + return Err( + pipeline::CreateRenderPipelineError::ShaderExpectsPipelineToUseDualSourceBlending, + ); + } + if pipeline_expects_dual_source_blending && !shader_expects_dual_source_blending { + return Err( + pipeline::CreateRenderPipelineError::PipelineExpectsShaderToUseDualSourceBlending, + ); + } + + if validated_stages.contains(wgt::ShaderStages::FRAGMENT) { + for (i, output) in io.iter() { + match color_targets.get(*i as usize) { + Some(Some(state)) => { + validation::check_texture_format(state.format, &output.ty).map_err( + |pipeline| { + pipeline::CreateRenderPipelineError::ColorState( + *i as u8, + pipeline::ColorStateError::IncompatibleFormat { + pipeline, + shader: output.ty, + }, + ) + }, + )?; + } + _ => { + log::warn!( + "The fragment stage {:?} output @location({}) values are ignored", + fragment_stage + .as_ref() + .map_or("", |stage| stage.entry_point), + i + ); + } + } + } + } + let last_stage = match desc.fragment { + Some(_) => wgt::ShaderStages::FRAGMENT, + None => wgt::ShaderStages::VERTEX, + }; + if desc.layout.is_none() && !validated_stages.contains(last_stage) { + return Err(pipeline::ImplicitLayoutError::ReflectionError(last_stage).into()); + } + + let pipeline_layout = match binding_layout_source { + validation::BindingLayoutSource::Provided(_) => { + drop(binding_layout_source); + pipeline_layout.unwrap() + } + validation::BindingLayoutSource::Derived(entries) => self.derive_pipeline_layout( + implicit_context, + entries, + &hub.bind_group_layouts, + &hub.pipeline_layouts, + )?, + }; + + // Multiview is only supported if the feature is enabled + if desc.multiview.is_some() { + self.require_features(wgt::Features::MULTIVIEW)?; + } + + if !self + .downlevel + .flags + .contains(wgt::DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED) + { + for (binding, size) in shader_binding_sizes.iter() { + if size.get() % 16 != 0 { + return Err(pipeline::CreateRenderPipelineError::UnalignedShader { + binding: binding.binding, + group: binding.group, + size: size.get(), + }); + } + } + } + + let late_sized_buffer_groups = + Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout); + + let pipeline_desc = hal::RenderPipelineDescriptor { + label: desc.label.to_hal(self.instance_flags), + layout: pipeline_layout.raw(), + vertex_buffers: &vertex_buffers, + vertex_stage, + primitive: desc.primitive, + depth_stencil: desc.depth_stencil.clone(), + multisample: desc.multisample, + fragment_stage, + color_targets, + multiview: desc.multiview, + }; + let raw = unsafe { + self.raw + .as_ref() + .unwrap() + .create_render_pipeline(&pipeline_desc) + } + .map_err(|err| match err { + hal::PipelineError::Device(error) => { + pipeline::CreateRenderPipelineError::Device(error.into()) + } + hal::PipelineError::Linkage(stage, msg) => { + pipeline::CreateRenderPipelineError::Internal { stage, error: msg } + } + hal::PipelineError::EntryPoint(stage) => { + pipeline::CreateRenderPipelineError::Internal { + stage: hal::auxil::map_naga_stage(stage), + error: ENTRYPOINT_FAILURE_ERROR.to_string(), + } + } + })?; + + let pass_context = RenderPassContext { + attachments: AttachmentData { + colors: color_targets + .iter() + .map(|state| state.as_ref().map(|s| s.format)) + .collect(), + resolves: ArrayVec::new(), + depth_stencil: depth_stencil_state.as_ref().map(|state| state.format), + }, + sample_count: samples, + multiview: desc.multiview, + }; + + let mut flags = pipeline::PipelineFlags::empty(); + for state in color_targets.iter().filter_map(|s| s.as_ref()) { + if let Some(ref bs) = state.blend { + if bs.color.uses_constant() | bs.alpha.uses_constant() { + flags |= pipeline::PipelineFlags::BLEND_CONSTANT; + } + } + } + if let Some(ds) = depth_stencil_state.as_ref() { + if ds.stencil.is_enabled() && ds.stencil.needs_ref_value() { + flags |= pipeline::PipelineFlags::STENCIL_REFERENCE; + } + if !ds.is_depth_read_only() { + flags |= pipeline::PipelineFlags::WRITES_DEPTH; + } + if !ds.is_stencil_read_only(desc.primitive.cull_mode) { + flags |= pipeline::PipelineFlags::WRITES_STENCIL; + } + } + + let shader_modules = { + let mut shader_modules = ArrayVec::new(); + shader_modules.push(vertex_shader_module); + shader_modules.extend(fragment_shader_module); + shader_modules + }; + + let pipeline = pipeline::RenderPipeline { + raw: Some(raw), + layout: pipeline_layout, + device: self.clone(), + pass_context, + _shader_modules: shader_modules, + flags, + strip_index_format: desc.primitive.strip_index_format, + vertex_steps, + late_sized_buffer_groups, + info: ResourceInfo::new(desc.label.borrow_or_default()), + }; + Ok(pipeline) + } + + pub(crate) fn get_texture_format_features( + &self, + adapter: &Adapter<A>, + format: TextureFormat, + ) -> wgt::TextureFormatFeatures { + // Variant of adapter.get_texture_format_features that takes device features into account + use wgt::TextureFormatFeatureFlags as tfsc; + let mut format_features = adapter.get_texture_format_features(format); + if (format == TextureFormat::R32Float + || format == TextureFormat::Rg32Float + || format == TextureFormat::Rgba32Float) + && !self.features.contains(wgt::Features::FLOAT32_FILTERABLE) + { + format_features.flags.set(tfsc::FILTERABLE, false); + } + format_features + } + + pub(crate) fn describe_format_features( + &self, + adapter: &Adapter<A>, + format: TextureFormat, + ) -> Result<wgt::TextureFormatFeatures, MissingFeatures> { + self.require_features(format.required_features())?; + + let using_device_features = self + .features + .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES); + // If we're running downlevel, we need to manually ask the backend what + // we can use as we can't trust WebGPU. + let downlevel = !self + .downlevel + .flags + .contains(wgt::DownlevelFlags::WEBGPU_TEXTURE_FORMAT_SUPPORT); + + if using_device_features || downlevel { + Ok(self.get_texture_format_features(adapter, format)) + } else { + Ok(format.guaranteed_format_features(self.features)) + } + } + + pub(crate) fn wait_for_submit( + &self, + submission_index: SubmissionIndex, + ) -> Result<(), WaitIdleError> { + let guard = self.fence.read(); + let fence = guard.as_ref().unwrap(); + let last_done_index = unsafe { + self.raw + .as_ref() + .unwrap() + .get_fence_value(fence) + .map_err(DeviceError::from)? + }; + if last_done_index < submission_index { + log::info!("Waiting for submission {:?}", submission_index); + unsafe { + self.raw + .as_ref() + .unwrap() + .wait(fence, submission_index, !0) + .map_err(DeviceError::from)? + }; + drop(guard); + let closures = self.lock_life().triage_submissions( + submission_index, + self.command_allocator.lock().as_mut().unwrap(), + ); + assert!( + closures.is_empty(), + "wait_for_submit is not expected to work with closures" + ); + } + Ok(()) + } + + pub(crate) fn create_query_set( + self: &Arc<Self>, + desc: &resource::QuerySetDescriptor, + ) -> Result<QuerySet<A>, resource::CreateQuerySetError> { + use resource::CreateQuerySetError as Error; + + match desc.ty { + wgt::QueryType::Occlusion => {} + wgt::QueryType::Timestamp => { + self.require_features(wgt::Features::TIMESTAMP_QUERY)?; + } + wgt::QueryType::PipelineStatistics(..) => { + self.require_features(wgt::Features::PIPELINE_STATISTICS_QUERY)?; + } + } + + if desc.count == 0 { + return Err(Error::ZeroCount); + } + + if desc.count > wgt::QUERY_SET_MAX_QUERIES { + return Err(Error::TooManyQueries { + count: desc.count, + maximum: wgt::QUERY_SET_MAX_QUERIES, + }); + } + + let hal_desc = desc.map_label(|label| label.to_hal(self.instance_flags)); + Ok(QuerySet { + raw: Some(unsafe { self.raw().create_query_set(&hal_desc).unwrap() }), + device: self.clone(), + info: ResourceInfo::new(""), + desc: desc.map_label(|_| ()), + }) + } + + pub(crate) fn lose(&self, message: &str) { + // Follow the steps at https://gpuweb.github.io/gpuweb/#lose-the-device. + + // Mark the device explicitly as invalid. This is checked in various + // places to prevent new work from being submitted. + self.valid.store(false, Ordering::Release); + + // 1) Resolve the GPUDevice device.lost promise. + let mut life_lock = self.lock_life(); + let closure = life_lock.device_lost_closure.take(); + // It's important to not hold the lock while calling the closure and while calling + // release_gpu_resources which may take the lock again. + drop(life_lock); + + if let Some(device_lost_closure) = closure { + device_lost_closure.call(DeviceLostReason::Unknown, message.to_string()); + } + + // 2) Complete any outstanding mapAsync() steps. + // 3) Complete any outstanding onSubmittedWorkDone() steps. + + // These parts are passively accomplished by setting valid to false, + // since that will prevent any new work from being added to the queues. + // Future calls to poll_devices will continue to check the work queues + // until they are cleared, and then drop the device. + + // Eagerly release GPU resources. + self.release_gpu_resources(); + } + + pub(crate) fn release_gpu_resources(&self) { + // This is called when the device is lost, which makes every associated + // resource invalid and unusable. This is an opportunity to release all of + // the underlying gpu resources, even though the objects remain visible to + // the user agent. We purge this memory naturally when resources have been + // moved into the appropriate buckets, so this function just needs to + // initiate movement into those buckets, and it can do that by calling + // "destroy" on all the resources we know about. + + // During these iterations, we discard all errors. We don't care! + let trackers = self.trackers.lock(); + for buffer in trackers.buffers.used_resources() { + let _ = buffer.destroy(); + } + for texture in trackers.textures.used_resources() { + let _ = texture.destroy(); + } + } +} + +impl<A: HalApi> Device<A> { + pub(crate) fn destroy_command_buffer(&self, mut cmd_buf: command::CommandBuffer<A>) { + let mut baked = cmd_buf.extract_baked_commands(); + unsafe { + baked.encoder.reset_all(baked.list.into_iter()); + } + unsafe { + self.raw + .as_ref() + .unwrap() + .destroy_command_encoder(baked.encoder); + } + } + + /// Wait for idle and remove resources that we can, before we die. + pub(crate) fn prepare_to_die(&self) { + self.pending_writes.lock().as_mut().unwrap().deactivate(); + let current_index = self.active_submission_index.load(Ordering::Relaxed); + if let Err(error) = unsafe { + let fence = self.fence.read(); + let fence = fence.as_ref().unwrap(); + self.raw + .as_ref() + .unwrap() + .wait(fence, current_index, CLEANUP_WAIT_MS) + } { + log::error!("failed to wait for the device: {error}"); + } + let mut life_tracker = self.lock_life(); + let _ = life_tracker.triage_submissions( + current_index, + self.command_allocator.lock().as_mut().unwrap(), + ); + if let Some(device_lost_closure) = life_tracker.device_lost_closure.take() { + // It's important to not hold the lock while calling the closure. + drop(life_tracker); + device_lost_closure.call(DeviceLostReason::Dropped, "Device is dying.".to_string()); + } + #[cfg(feature = "trace")] + { + *self.trace.lock() = None; + } + } +} + +impl<A: HalApi> Resource for Device<A> { + const TYPE: ResourceType = "Device"; + + type Marker = crate::id::markers::Device; + + fn as_info(&self) -> &ResourceInfo<Self> { + &self.info + } + + fn as_info_mut(&mut self) -> &mut ResourceInfo<Self> { + &mut self.info + } +} diff --git a/third_party/rust/wgpu-core/src/device/trace.rs b/third_party/rust/wgpu-core/src/device/trace.rs new file mode 100644 index 0000000000..0802b610d8 --- /dev/null +++ b/third_party/rust/wgpu-core/src/device/trace.rs @@ -0,0 +1,235 @@ +use crate::id; +use std::ops::Range; +#[cfg(feature = "trace")] +use std::{borrow::Cow, io::Write as _}; + +//TODO: consider a readable Id that doesn't include the backend + +type FileName = String; + +pub const FILE_NAME: &str = "trace.ron"; + +#[cfg(feature = "trace")] +pub(crate) fn new_render_bundle_encoder_descriptor<'a>( + label: crate::Label<'a>, + context: &'a super::RenderPassContext, + depth_read_only: bool, + stencil_read_only: bool, +) -> crate::command::RenderBundleEncoderDescriptor<'a> { + crate::command::RenderBundleEncoderDescriptor { + label, + color_formats: Cow::Borrowed(&context.attachments.colors), + depth_stencil: context.attachments.depth_stencil.map(|format| { + wgt::RenderBundleDepthStencil { + format, + depth_read_only, + stencil_read_only, + } + }), + sample_count: context.sample_count, + multiview: context.multiview, + } +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Action<'a> { + Init { + desc: crate::device::DeviceDescriptor<'a>, + backend: wgt::Backend, + }, + ConfigureSurface( + id::SurfaceId, + wgt::SurfaceConfiguration<Vec<wgt::TextureFormat>>, + ), + CreateBuffer(id::BufferId, crate::resource::BufferDescriptor<'a>), + FreeBuffer(id::BufferId), + DestroyBuffer(id::BufferId), + CreateTexture(id::TextureId, crate::resource::TextureDescriptor<'a>), + FreeTexture(id::TextureId), + DestroyTexture(id::TextureId), + CreateTextureView { + id: id::TextureViewId, + parent_id: id::TextureId, + desc: crate::resource::TextureViewDescriptor<'a>, + }, + DestroyTextureView(id::TextureViewId), + CreateSampler(id::SamplerId, crate::resource::SamplerDescriptor<'a>), + DestroySampler(id::SamplerId), + GetSurfaceTexture { + id: id::TextureId, + parent_id: id::SurfaceId, + }, + Present(id::SurfaceId), + DiscardSurfaceTexture(id::SurfaceId), + CreateBindGroupLayout( + id::BindGroupLayoutId, + crate::binding_model::BindGroupLayoutDescriptor<'a>, + ), + DestroyBindGroupLayout(id::BindGroupLayoutId), + CreatePipelineLayout( + id::PipelineLayoutId, + crate::binding_model::PipelineLayoutDescriptor<'a>, + ), + DestroyPipelineLayout(id::PipelineLayoutId), + CreateBindGroup( + id::BindGroupId, + crate::binding_model::BindGroupDescriptor<'a>, + ), + DestroyBindGroup(id::BindGroupId), + CreateShaderModule { + id: id::ShaderModuleId, + desc: crate::pipeline::ShaderModuleDescriptor<'a>, + data: FileName, + }, + DestroyShaderModule(id::ShaderModuleId), + CreateComputePipeline { + id: id::ComputePipelineId, + desc: crate::pipeline::ComputePipelineDescriptor<'a>, + #[cfg_attr(feature = "replay", serde(default))] + implicit_context: Option<super::ImplicitPipelineContext>, + }, + DestroyComputePipeline(id::ComputePipelineId), + CreateRenderPipeline { + id: id::RenderPipelineId, + desc: crate::pipeline::RenderPipelineDescriptor<'a>, + #[cfg_attr(feature = "replay", serde(default))] + implicit_context: Option<super::ImplicitPipelineContext>, + }, + DestroyRenderPipeline(id::RenderPipelineId), + CreateRenderBundle { + id: id::RenderBundleId, + desc: crate::command::RenderBundleEncoderDescriptor<'a>, + base: crate::command::BasePass<crate::command::RenderCommand>, + }, + DestroyRenderBundle(id::RenderBundleId), + CreateQuerySet { + id: id::QuerySetId, + desc: crate::resource::QuerySetDescriptor<'a>, + }, + DestroyQuerySet(id::QuerySetId), + WriteBuffer { + id: id::BufferId, + data: FileName, + range: Range<wgt::BufferAddress>, + queued: bool, + }, + WriteTexture { + to: crate::command::ImageCopyTexture, + data: FileName, + layout: wgt::ImageDataLayout, + size: wgt::Extent3d, + }, + Submit(crate::SubmissionIndex, Vec<Command>), +} + +#[derive(Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Command { + CopyBufferToBuffer { + src: id::BufferId, + src_offset: wgt::BufferAddress, + dst: id::BufferId, + dst_offset: wgt::BufferAddress, + size: wgt::BufferAddress, + }, + CopyBufferToTexture { + src: crate::command::ImageCopyBuffer, + dst: crate::command::ImageCopyTexture, + size: wgt::Extent3d, + }, + CopyTextureToBuffer { + src: crate::command::ImageCopyTexture, + dst: crate::command::ImageCopyBuffer, + size: wgt::Extent3d, + }, + CopyTextureToTexture { + src: crate::command::ImageCopyTexture, + dst: crate::command::ImageCopyTexture, + size: wgt::Extent3d, + }, + ClearBuffer { + dst: id::BufferId, + offset: wgt::BufferAddress, + size: Option<wgt::BufferAddress>, + }, + ClearTexture { + dst: id::TextureId, + subresource_range: wgt::ImageSubresourceRange, + }, + WriteTimestamp { + query_set_id: id::QuerySetId, + query_index: u32, + }, + ResolveQuerySet { + query_set_id: id::QuerySetId, + start_query: u32, + query_count: u32, + destination: id::BufferId, + destination_offset: wgt::BufferAddress, + }, + PushDebugGroup(String), + PopDebugGroup, + InsertDebugMarker(String), + RunComputePass { + base: crate::command::BasePass<crate::command::ComputeCommand>, + timestamp_writes: Option<crate::command::ComputePassTimestampWrites>, + }, + RunRenderPass { + base: crate::command::BasePass<crate::command::RenderCommand>, + target_colors: Vec<Option<crate::command::RenderPassColorAttachment>>, + target_depth_stencil: Option<crate::command::RenderPassDepthStencilAttachment>, + timestamp_writes: Option<crate::command::RenderPassTimestampWrites>, + occlusion_query_set_id: Option<id::QuerySetId>, + }, +} + +#[cfg(feature = "trace")] +#[derive(Debug)] +pub struct Trace { + path: std::path::PathBuf, + file: std::fs::File, + config: ron::ser::PrettyConfig, + binary_id: usize, +} + +#[cfg(feature = "trace")] +impl Trace { + pub fn new(path: &std::path::Path) -> Result<Self, std::io::Error> { + log::info!("Tracing into '{:?}'", path); + let mut file = std::fs::File::create(path.join(FILE_NAME))?; + file.write_all(b"[\n")?; + Ok(Self { + path: path.to_path_buf(), + file, + config: ron::ser::PrettyConfig::default(), + binary_id: 0, + }) + } + + pub fn make_binary(&mut self, kind: &str, data: &[u8]) -> String { + self.binary_id += 1; + let name = format!("data{}.{}", self.binary_id, kind); + let _ = std::fs::write(self.path.join(&name), data); + name + } + + pub(crate) fn add(&mut self, action: Action) { + match ron::ser::to_string_pretty(&action, self.config.clone()) { + Ok(string) => { + let _ = writeln!(self.file, "{},", string); + } + Err(e) => { + log::warn!("RON serialization failure: {:?}", e); + } + } + } +} + +#[cfg(feature = "trace")] +impl Drop for Trace { + fn drop(&mut self) { + let _ = self.file.write_all(b"]"); + } +} |