diff options
Diffstat (limited to '')
-rw-r--r-- | third_party/rust/wgpu-core/src/command/bundle.rs | 21 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/clear.rs | 7 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/compute.rs | 9 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/memory_init.rs | 11 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/render.rs | 15 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/transfer.rs | 19 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/device/global.rs | 33 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/device/life.rs | 15 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/device/mod.rs | 11 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/device/queue.rs | 17 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/device/resource.rs | 20 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/snatch.rs | 35 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/track/buffer.rs | 12 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/track/metadata.rs | 5 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/track/mod.rs | 50 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/track/texture.rs | 13 |
16 files changed, 221 insertions, 72 deletions
diff --git a/third_party/rust/wgpu-core/src/command/bundle.rs b/third_party/rust/wgpu-core/src/command/bundle.rs index ab2d18bc59..47beda8ec6 100644 --- a/third_party/rust/wgpu-core/src/command/bundle.rs +++ b/third_party/rust/wgpu-core/src/command/bundle.rs @@ -99,6 +99,7 @@ use crate::{ pipeline::{PipelineFlags, RenderPipeline, VertexStep}, resource::{Buffer, Resource, ResourceInfo, ResourceType}, resource_log, + snatch::SnatchGuard, track::RenderBundleScope, validation::check_buffer_usage, Label, LabelHelpers, @@ -165,7 +166,7 @@ fn validate_indexed_draw<A: HalApi>( ) -> Result<(), DrawError> { let last_index = first_index as u64 + index_count as u64; let index_limit = index_state.limit(); - if last_index <= index_limit { + if last_index > index_limit { return Err(DrawError::IndexBeyondLimit { last_index, index_limit, @@ -894,7 +895,11 @@ impl<A: HalApi> RenderBundle<A> { /// Note that the function isn't expected to fail, generally. /// All the validation has already been done by this point. /// The only failure condition is if some of the used buffers are destroyed. - pub(super) unsafe fn execute(&self, raw: &mut A::CommandEncoder) -> Result<(), ExecutionError> { + pub(super) unsafe fn execute( + &self, + raw: &mut A::CommandEncoder, + snatch_guard: &SnatchGuard, + ) -> Result<(), ExecutionError> { let mut offsets = self.base.dynamic_offsets.as_slice(); let mut pipeline_layout = None::<Arc<PipelineLayout<A>>>; if !self.discard_hal_labels { @@ -903,8 +908,6 @@ impl<A: HalApi> RenderBundle<A> { } } - let snatch_guard = self.device.snatchable_lock.read(); - use ArcRenderCommand as Cmd; for command in self.base.commands.iter() { match command { @@ -914,7 +917,7 @@ impl<A: HalApi> RenderBundle<A> { bind_group, } => { let raw_bg = bind_group - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(ExecutionError::InvalidBindGroup(bind_group.info.id()))?; unsafe { raw.set_bind_group( @@ -938,7 +941,7 @@ impl<A: HalApi> RenderBundle<A> { size, } => { let buffer: &A::Buffer = buffer - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(ExecutionError::DestroyedBuffer(buffer.info.id()))?; let bb = hal::BufferBinding { buffer, @@ -954,7 +957,7 @@ impl<A: HalApi> RenderBundle<A> { size, } => { let buffer = buffer - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(ExecutionError::DestroyedBuffer(buffer.info.id()))?; let bb = hal::BufferBinding { buffer, @@ -1041,7 +1044,7 @@ impl<A: HalApi> RenderBundle<A> { indexed: false, } => { let buffer = buffer - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(ExecutionError::DestroyedBuffer(buffer.info.id()))?; unsafe { raw.draw_indirect(buffer, *offset, 1) }; } @@ -1052,7 +1055,7 @@ impl<A: HalApi> RenderBundle<A> { indexed: true, } => { let buffer = buffer - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(ExecutionError::DestroyedBuffer(buffer.info.id()))?; unsafe { raw.draw_indexed_indirect(buffer, *offset, 1) }; } diff --git a/third_party/rust/wgpu-core/src/command/clear.rs b/third_party/rust/wgpu-core/src/command/clear.rs index e404fabb14..72c923f82e 100644 --- a/third_party/rust/wgpu-core/src/command/clear.rs +++ b/third_party/rust/wgpu-core/src/command/clear.rs @@ -12,6 +12,7 @@ use crate::{ id::{BufferId, CommandEncoderId, DeviceId, TextureId}, init_tracker::{MemoryInitKind, TextureInitRange}, resource::{Resource, Texture, TextureClearMode}, + snatch::SnatchGuard, track::{TextureSelector, TextureTracker}, }; @@ -239,6 +240,7 @@ impl Global { } let (encoder, tracker) = cmd_buf_data.open_encoder_and_tracker()?; + let snatch_guard = device.snatchable_lock.read(); clear_texture( &dst_texture, TextureInitRange { @@ -249,6 +251,7 @@ impl Global { &mut tracker.textures, &device.alignments, device.zero_buffer.as_ref().unwrap(), + &snatch_guard, ) } } @@ -260,10 +263,10 @@ pub(crate) fn clear_texture<A: HalApi>( texture_tracker: &mut TextureTracker<A>, alignments: &hal::Alignments, zero_buffer: &A::Buffer, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), ClearError> { - let snatch_guard = dst_texture.device.snatchable_lock.read(); let dst_raw = dst_texture - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or_else(|| ClearError::InvalidTexture(dst_texture.as_info().id()))?; // Issue the right barrier. diff --git a/third_party/rust/wgpu-core/src/command/compute.rs b/third_party/rust/wgpu-core/src/command/compute.rs index c2fd3ab397..b38324984c 100644 --- a/third_party/rust/wgpu-core/src/command/compute.rs +++ b/third_party/rust/wgpu-core/src/command/compute.rs @@ -272,14 +272,14 @@ where } } -struct State<A: HalApi> { +struct State<'a, A: HalApi> { binder: Binder<A>, pipeline: Option<id::ComputePipelineId>, - scope: UsageScope<A>, + scope: UsageScope<'a, A>, debug_scope_depth: u32, } -impl<A: HalApi> State<A> { +impl<'a, A: HalApi> State<'a, A> { fn is_ready(&self) -> Result<(), DispatchError> { let bind_mask = self.binder.invalid_mask(); if bind_mask != 0 { @@ -407,7 +407,7 @@ impl Global { let mut state = State { binder: Binder::new(), pipeline: None, - scope: UsageScope::new(&device.tracker_indices), + scope: device.new_usage_scope(), debug_scope_depth: 0, }; let mut temp_offsets = Vec::new(); @@ -868,6 +868,7 @@ impl Global { transit, &mut tracker.textures, device, + &snatch_guard, ); CommandBuffer::insert_barriers_from_tracker( transit, diff --git a/third_party/rust/wgpu-core/src/command/memory_init.rs b/third_party/rust/wgpu-core/src/command/memory_init.rs index 3bfc71f4f7..54bdedb792 100644 --- a/third_party/rust/wgpu-core/src/command/memory_init.rs +++ b/third_party/rust/wgpu-core/src/command/memory_init.rs @@ -7,6 +7,7 @@ use crate::{ hal_api::HalApi, init_tracker::*, resource::{Resource, Texture}, + snatch::SnatchGuard, track::{TextureTracker, Tracker}, FastHashMap, }; @@ -144,6 +145,7 @@ pub(crate) fn fixup_discarded_surfaces< encoder: &mut A::CommandEncoder, texture_tracker: &mut TextureTracker<A>, device: &Device<A>, + snatch_guard: &SnatchGuard<'_>, ) { for init in inits { clear_texture( @@ -156,6 +158,7 @@ pub(crate) fn fixup_discarded_surfaces< texture_tracker, &device.alignments, device.zero_buffer.as_ref().unwrap(), + snatch_guard, ) .unwrap(); } @@ -167,6 +170,7 @@ impl<A: HalApi> BakedCommands<A> { pub(crate) fn initialize_buffer_memory( &mut self, device_tracker: &mut Tracker<A>, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), DestroyedBufferError> { // Gather init ranges for each buffer so we can collapse them. // It is not possible to do this at an earlier point since previously @@ -225,16 +229,15 @@ impl<A: HalApi> BakedCommands<A> { .unwrap() .1; - let snatch_guard = buffer.device.snatchable_lock.read(); let raw_buf = buffer .raw - .get(&snatch_guard) + .get(snatch_guard) .ok_or(DestroyedBufferError(buffer_id))?; unsafe { self.encoder.transition_buffers( transition - .map(|pending| pending.into_hal(&buffer, &snatch_guard)) + .map(|pending| pending.into_hal(&buffer, snatch_guard)) .into_iter(), ); } @@ -271,6 +274,7 @@ impl<A: HalApi> BakedCommands<A> { &mut self, device_tracker: &mut Tracker<A>, device: &Device<A>, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), DestroyedTextureError> { let mut ranges: Vec<TextureInitRange> = Vec::new(); for texture_use in self.texture_memory_actions.drain_init_actions() { @@ -310,6 +314,7 @@ impl<A: HalApi> BakedCommands<A> { &mut device_tracker.textures, &device.alignments, device.zero_buffer.as_ref().unwrap(), + snatch_guard, ); // A Texture can be destroyed between the command recording diff --git a/third_party/rust/wgpu-core/src/command/render.rs b/third_party/rust/wgpu-core/src/command/render.rs index 9141ddb021..7e859e3cc8 100644 --- a/third_party/rust/wgpu-core/src/command/render.rs +++ b/third_party/rust/wgpu-core/src/command/render.rs @@ -739,9 +739,9 @@ impl<A: HalApi> TextureView<A> { const MAX_TOTAL_ATTACHMENTS: usize = hal::MAX_COLOR_ATTACHMENTS + hal::MAX_COLOR_ATTACHMENTS + 1; type AttachmentDataVec<T> = ArrayVec<T, MAX_TOTAL_ATTACHMENTS>; -struct RenderPassInfo<'a, A: HalApi> { +struct RenderPassInfo<'a, 'd, A: HalApi> { context: RenderPassContext, - usage_scope: UsageScope<A>, + usage_scope: UsageScope<'d, A>, /// All render attachments, including depth/stencil render_attachments: AttachmentDataVec<RenderAttachment<'a, A>>, is_depth_read_only: bool, @@ -754,7 +754,7 @@ struct RenderPassInfo<'a, A: HalApi> { multiview: Option<NonZeroU32>, } -impl<'a, A: HalApi> RenderPassInfo<'a, A> { +impl<'a, 'd, A: HalApi> RenderPassInfo<'a, 'd, A> { fn add_pass_texture_init_actions<V>( channel: &PassChannel<V>, texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>, @@ -790,7 +790,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { } fn start( - device: &Device<A>, + device: &'d Device<A>, label: Option<&str>, color_attachments: &[Option<RenderPassColorAttachment>], depth_stencil_attachment: Option<&RenderPassDepthStencilAttachment>, @@ -1214,7 +1214,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { Ok(Self { context, - usage_scope: UsageScope::new(&device.tracker_indices), + usage_scope: device.new_usage_scope(), render_attachments, is_depth_read_only, is_stencil_read_only, @@ -1230,7 +1230,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { mut self, raw: &mut A::CommandEncoder, snatch_guard: &SnatchGuard, - ) -> Result<(UsageScope<A>, SurfacesInDiscardState<A>), RenderPassErrorInner> { + ) -> Result<(UsageScope<'d, A>, SurfacesInDiscardState<A>), RenderPassErrorInner> { profiling::scope!("RenderPassInfo::finish"); unsafe { raw.end_render_pass(); @@ -2374,7 +2374,7 @@ impl Global { .extend(texture_memory_actions.register_init_action(action)); } - unsafe { bundle.execute(raw) } + unsafe { bundle.execute(raw, &snatch_guard) } .map_err(|e| match e { ExecutionError::DestroyedBuffer(id) => { RenderCommandError::DestroyedBuffer(id) @@ -2427,6 +2427,7 @@ impl Global { transit, &mut tracker.textures, &cmd_buf.device, + &snatch_guard, ); cmd_buf_data diff --git a/third_party/rust/wgpu-core/src/command/transfer.rs b/third_party/rust/wgpu-core/src/command/transfer.rs index 0a952dfc84..8e98a4c9b9 100644 --- a/third_party/rust/wgpu-core/src/command/transfer.rs +++ b/third_party/rust/wgpu-core/src/command/transfer.rs @@ -14,6 +14,7 @@ use crate::{ TextureInitTrackerAction, }, resource::{Resource, Texture, TextureErrorDimension}, + snatch::SnatchGuard, track::{TextureSelector, Tracker}, }; @@ -452,6 +453,7 @@ fn handle_texture_init<A: HalApi>( copy_texture: &ImageCopyTexture, copy_size: &Extent3d, texture: &Arc<Texture<A>>, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), ClearError> { let init_action = TextureInitTrackerAction { texture: texture.clone(), @@ -480,6 +482,7 @@ fn handle_texture_init<A: HalApi>( &mut trackers.textures, &device.alignments, device.zero_buffer.as_ref().unwrap(), + snatch_guard, )?; } } @@ -499,6 +502,7 @@ fn handle_src_texture_init<A: HalApi>( source: &ImageCopyTexture, copy_size: &Extent3d, texture: &Arc<Texture<A>>, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), TransferError> { handle_texture_init( MemoryInitKind::NeedsInitializedMemory, @@ -509,6 +513,7 @@ fn handle_src_texture_init<A: HalApi>( source, copy_size, texture, + snatch_guard, )?; Ok(()) } @@ -525,6 +530,7 @@ fn handle_dst_texture_init<A: HalApi>( destination: &ImageCopyTexture, copy_size: &Extent3d, texture: &Arc<Texture<A>>, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), TransferError> { // Attention: If we don't write full texture subresources, we need to a full // clear first since we don't track subrects. This means that in rare cases @@ -549,6 +555,7 @@ fn handle_dst_texture_init<A: HalApi>( destination, copy_size, texture, + snatch_guard, )?; Ok(()) } @@ -779,6 +786,8 @@ impl Global { let (dst_range, dst_base) = extract_texture_selector(destination, copy_size, &dst_texture)?; + let snatch_guard = device.snatchable_lock.read(); + // Handle texture init *before* dealing with barrier transitions so we // have an easier time inserting "immediate-inits" that may be required // by prior discards in rare cases. @@ -790,10 +799,9 @@ impl Global { destination, copy_size, &dst_texture, + &snatch_guard, )?; - let snatch_guard = device.snatchable_lock.read(); - let (src_buffer, src_pending) = { let buffer_guard = hub.buffers.read(); let src_buffer = buffer_guard @@ -935,6 +943,8 @@ impl Global { let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?; + let snatch_guard = device.snatchable_lock.read(); + // Handle texture init *before* dealing with barrier transitions so we // have an easier time inserting "immediate-inits" that may be required // by prior discards in rare cases. @@ -946,10 +956,9 @@ impl Global { source, copy_size, &src_texture, + &snatch_guard, )?; - let snatch_guard = device.snatchable_lock.read(); - let src_pending = tracker .textures .set_single(&src_texture, src_range, hal::TextureUses::COPY_SRC) @@ -1152,6 +1161,7 @@ impl Global { source, copy_size, &src_texture, + &snatch_guard, )?; handle_dst_texture_init( encoder, @@ -1161,6 +1171,7 @@ impl Global { destination, copy_size, &dst_texture, + &snatch_guard, )?; let src_pending = cmd_buf_data diff --git a/third_party/rust/wgpu-core/src/device/global.rs b/third_party/rust/wgpu-core/src/device/global.rs index 539b92e0f3..0c97e1b504 100644 --- a/third_party/rust/wgpu-core/src/device/global.rs +++ b/third_party/rust/wgpu-core/src/device/global.rs @@ -192,7 +192,15 @@ impl Global { let ptr = if map_size == 0 { std::ptr::NonNull::dangling() } else { - match map_buffer(device.raw(), &buffer, 0, map_size, HostMap::Write) { + let snatch_guard = device.snatchable_lock.read(); + match map_buffer( + device.raw(), + &buffer, + 0, + map_size, + HostMap::Write, + &snatch_guard, + ) { Ok(ptr) => ptr, Err(e) => { to_destroy.push(buffer); @@ -2008,9 +2016,10 @@ impl Global { } // Wait for all work to finish before configuring the surface. + let snatch_guard = device.snatchable_lock.read(); let fence = device.fence.read(); let fence = fence.as_ref().unwrap(); - match device.maintain(fence, wgt::Maintain::Wait) { + match device.maintain(fence, wgt::Maintain::Wait, snatch_guard) { Ok((closures, _)) => { user_callbacks = closures; } @@ -2120,9 +2129,10 @@ impl Global { device: &crate::device::Device<A>, maintain: wgt::Maintain<queue::WrappedSubmissionIndex>, ) -> Result<DevicePoll, WaitIdleError> { + let snatch_guard = device.snatchable_lock.read(); let fence = device.fence.read(); let fence = fence.as_ref().unwrap(); - let (closures, queue_empty) = device.maintain(fence, maintain)?; + let (closures, queue_empty) = device.maintain(fence, maintain, snatch_guard)?; // Some deferred destroys are scheduled in maintain so run this right after // to avoid holding on to them until the next device poll. @@ -2240,6 +2250,15 @@ impl Global { } } + // This is a test-only function to force the device into an + // invalid state by inserting an error value in its place in + // the registry. + pub fn device_make_invalid<A: HalApi>(&self, device_id: DeviceId) { + let hub = A::hub(self); + hub.devices + .force_replace_with_error(device_id, "Made invalid."); + } + pub fn device_drop<A: HalApi>(&self, device_id: DeviceId) { profiling::scope!("Device::drop"); api_log!("Device::drop {device_id:?}"); @@ -2275,7 +2294,7 @@ impl Global { ) { let hub = A::hub(self); - if let Ok(device) = hub.devices.get(device_id) { + if let Ok(Some(device)) = hub.devices.try_get(device_id) { let mut life_tracker = device.lock_life(); if let Some(existing_closure) = life_tracker.device_lost_closure.take() { // It's important to not hold the lock while calling the closure. @@ -2284,6 +2303,12 @@ impl Global { life_tracker = device.lock_life(); } life_tracker.device_lost_closure = Some(device_lost_closure); + } else { + // No device? Okay. Just like we have to call any existing closure + // before we drop it, we need to call this closure before we exit + // this function, because there's no device that is ever going to + // call it. + device_lost_closure.call(DeviceLostReason::DeviceInvalid, "".to_string()); } } diff --git a/third_party/rust/wgpu-core/src/device/life.rs b/third_party/rust/wgpu-core/src/device/life.rs index 7b06a4a30b..af345015df 100644 --- a/third_party/rust/wgpu-core/src/device/life.rs +++ b/third_party/rust/wgpu-core/src/device/life.rs @@ -12,6 +12,7 @@ use crate::{ self, Buffer, DestroyedBuffer, DestroyedTexture, QuerySet, Resource, Sampler, StagingBuffer, Texture, TextureView, }, + snatch::SnatchGuard, track::{ResourceTracker, Tracker, TrackerIndex}, FastHashMap, SubmissionIndex, }; @@ -309,12 +310,12 @@ impl<A: HalApi> LifetimeTracker<A> { } pub fn post_submit(&mut self) { - for v in self.future_suspected_buffers.drain(..).take(1) { + for v in self.future_suspected_buffers.drain(..) { self.suspected_resources .buffers .insert(v.as_info().tracker_index(), v); } - for v in self.future_suspected_textures.drain(..).take(1) { + for v in self.future_suspected_textures.drain(..) { self.suspected_resources .textures .insert(v.as_info().tracker_index(), v); @@ -780,6 +781,7 @@ impl<A: HalApi> LifetimeTracker<A> { &mut self, raw: &A::Device, trackers: &Mutex<Tracker<A>>, + snatch_guard: &SnatchGuard, ) -> Vec<super::BufferMapPendingClosure> { if self.ready_to_map.is_empty() { return Vec::new(); @@ -816,7 +818,14 @@ impl<A: HalApi> LifetimeTracker<A> { log::debug!("Buffer {tracker_index:?} map state -> Active"); let host = mapping.op.host; let size = mapping.range.end - mapping.range.start; - match super::map_buffer(raw, &buffer, mapping.range.start, size, host) { + match super::map_buffer( + raw, + &buffer, + mapping.range.start, + size, + host, + snatch_guard, + ) { Ok(ptr) => { *buffer.map_state.lock() = resource::BufferMapState::Active { ptr, diff --git a/third_party/rust/wgpu-core/src/device/mod.rs b/third_party/rust/wgpu-core/src/device/mod.rs index 7ecda830a3..e2ab6c2690 100644 --- a/third_party/rust/wgpu-core/src/device/mod.rs +++ b/third_party/rust/wgpu-core/src/device/mod.rs @@ -3,9 +3,10 @@ use crate::{ hal_api::HalApi, hub::Hub, id::{BindGroupLayoutId, PipelineLayoutId}, - resource::{Buffer, BufferAccessResult}, - resource::{BufferAccessError, BufferMapOperation}, - resource_log, Label, DOWNLEVEL_ERROR_MESSAGE, + resource::{Buffer, BufferAccessError, BufferAccessResult, BufferMapOperation}, + resource_log, + snatch::SnatchGuard, + Label, DOWNLEVEL_ERROR_MESSAGE, }; use arrayvec::ArrayVec; @@ -317,10 +318,10 @@ fn map_buffer<A: HalApi>( offset: BufferAddress, size: BufferAddress, kind: HostMap, + snatch_guard: &SnatchGuard, ) -> Result<ptr::NonNull<u8>, BufferAccessError> { - let snatch_guard = buffer.device.snatchable_lock.read(); let raw_buffer = buffer - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(BufferAccessError::Destroyed)?; let mapping = unsafe { raw.map_buffer(raw_buffer, offset..offset + size) diff --git a/third_party/rust/wgpu-core/src/device/queue.rs b/third_party/rust/wgpu-core/src/device/queue.rs index 6ebb9eb09b..3cb5f695a7 100644 --- a/third_party/rust/wgpu-core/src/device/queue.rs +++ b/third_party/rust/wgpu-core/src/device/queue.rs @@ -815,6 +815,7 @@ impl Global { &mut trackers.textures, &device.alignments, device.zero_buffer.as_ref().unwrap(), + &device.snatchable_lock.read(), ) .map_err(QueueWriteError::from)?; } @@ -1084,6 +1085,7 @@ impl Global { &mut trackers.textures, &device.alignments, device.zero_buffer.as_ref().unwrap(), + &device.snatchable_lock.read(), ) .map_err(QueueWriteError::from)?; } @@ -1147,6 +1149,9 @@ impl Global { let device = queue.device.as_ref().unwrap(); + let snatch_guard = device.snatchable_lock.read(); + + // Fence lock must be acquired after the snatch lock everywhere to avoid deadlocks. let mut fence = device.fence.write(); let fence = fence.as_mut().unwrap(); let submit_index = device @@ -1155,9 +1160,7 @@ impl Global { + 1; let mut active_executions = Vec::new(); - let mut used_surface_textures = track::TextureUsageScope::new(); - - let snatch_guard = device.snatchable_lock.read(); + let mut used_surface_textures = track::TextureUsageScope::default(); let mut submit_surface_textures_owned = SmallVec::<[_; 2]>::new(); @@ -1391,10 +1394,10 @@ impl Global { //Note: locking the trackers has to be done after the storages let mut trackers = device.trackers.lock(); baked - .initialize_buffer_memory(&mut *trackers) + .initialize_buffer_memory(&mut *trackers, &snatch_guard) .map_err(|err| QueueSubmitError::DestroyedBuffer(err.0))?; baked - .initialize_texture_memory(&mut *trackers, device) + .initialize_texture_memory(&mut *trackers, device, &snatch_guard) .map_err(|err| QueueSubmitError::DestroyedTexture(err.0))?; //Note: stateless trackers are not merged: // device already knows these resources exist. @@ -1435,7 +1438,7 @@ impl Global { baked.encoder.end_encoding().unwrap() }; baked.list.push(present); - used_surface_textures = track::TextureUsageScope::new(); + used_surface_textures = track::TextureUsageScope::default(); } // done @@ -1542,7 +1545,7 @@ impl Global { // This will schedule destruction of all resources that are no longer needed // by the user but used in the command stream, among other things. - let (closures, _) = match device.maintain(fence, wgt::Maintain::Poll) { + let (closures, _) = match device.maintain(fence, wgt::Maintain::Poll, snatch_guard) { Ok(closures) => closures, Err(WaitIdleError::Device(err)) => return Err(QueueSubmitError::Queue(err)), Err(WaitIdleError::StuckGpu) => return Err(QueueSubmitError::StuckGpu), diff --git a/third_party/rust/wgpu-core/src/device/resource.rs b/third_party/rust/wgpu-core/src/device/resource.rs index 28ba0eafb1..4892aecb75 100644 --- a/third_party/rust/wgpu-core/src/device/resource.rs +++ b/third_party/rust/wgpu-core/src/device/resource.rs @@ -28,7 +28,10 @@ use crate::{ resource_log, snatch::{SnatchGuard, SnatchLock, Snatchable}, storage::Storage, - track::{BindGroupStates, TextureSelector, Tracker, TrackerIndexAllocators}, + track::{ + BindGroupStates, TextureSelector, Tracker, TrackerIndexAllocators, UsageScope, + UsageScopePool, + }, validation::{ self, check_buffer_usage, check_texture_usage, validate_color_attachment_bytes_per_sample, }, @@ -97,6 +100,8 @@ pub struct Device<A: HalApi> { pub(crate) command_allocator: Mutex<Option<CommandAllocator<A>>>, //Note: The submission index here corresponds to the last submission that is done. pub(crate) active_submission_index: AtomicU64, //SubmissionIndex, + // NOTE: if both are needed, the `snatchable_lock` must be consistently acquired before the + // `fence` lock to avoid deadlocks. pub(crate) fence: RwLock<Option<A::Fence>>, pub(crate) snatchable_lock: SnatchLock, @@ -135,6 +140,7 @@ pub struct Device<A: HalApi> { pub(crate) deferred_destroy: Mutex<Vec<DeferredDestroy<A>>>, #[cfg(feature = "trace")] pub(crate) trace: Mutex<Option<trace::Trace>>, + pub(crate) usage_scopes: UsageScopePool<A>, } pub(crate) enum DeferredDestroy<A: HalApi> { @@ -296,6 +302,7 @@ impl<A: HalApi> Device<A> { instance_flags, pending_writes: Mutex::new(Some(pending_writes)), deferred_destroy: Mutex::new(Vec::new()), + usage_scopes: Default::default(), }) } @@ -387,6 +394,7 @@ impl<A: HalApi> Device<A> { &'this self, fence: &A::Fence, maintain: wgt::Maintain<queue::WrappedSubmissionIndex>, + snatch_guard: SnatchGuard, ) -> Result<(UserClosures, bool), WaitIdleError> { profiling::scope!("Device::maintain"); let last_done_index = if maintain.is_wait() { @@ -440,7 +448,8 @@ impl<A: HalApi> Device<A> { life_tracker.triage_mapped(); } - let mapping_closures = life_tracker.handle_mapping(self.raw(), &self.trackers); + let mapping_closures = + life_tracker.handle_mapping(self.raw(), &self.trackers, &snatch_guard); let queue_empty = life_tracker.queue_empty(); @@ -467,8 +476,9 @@ impl<A: HalApi> Device<A> { } } - // Don't hold the lock while calling release_gpu_resources. + // Don't hold the locks while calling release_gpu_resources. drop(life_tracker); + drop(snatch_guard); if should_release_gpu_resource { self.release_gpu_resources(); @@ -3568,6 +3578,10 @@ impl<A: HalApi> Device<A> { let _ = texture.destroy(); } } + + pub(crate) fn new_usage_scope(&self) -> UsageScope<'_, A> { + UsageScope::new_pooled(&self.usage_scopes, &self.tracker_indices) + } } impl<A: HalApi> Device<A> { diff --git a/third_party/rust/wgpu-core/src/snatch.rs b/third_party/rust/wgpu-core/src/snatch.rs index 2324d33574..d5cd1a3d37 100644 --- a/third_party/rust/wgpu-core/src/snatch.rs +++ b/third_party/rust/wgpu-core/src/snatch.rs @@ -1,7 +1,12 @@ #![allow(unused)] use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; -use std::cell::UnsafeCell; +use std::{ + backtrace::Backtrace, + cell::{Cell, RefCell, UnsafeCell}, + panic::{self, Location}, + thread, +}; /// A guard that provides read access to snatchable data. pub struct SnatchGuard<'a>(RwLockReadGuard<'a, ()>); @@ -59,6 +64,10 @@ impl<T> std::fmt::Debug for Snatchable<T> { unsafe impl<T> Sync for Snatchable<T> {} +thread_local! { + static READ_LOCK_LOCATION: Cell<Option<(&'static Location<'static>, Backtrace)>> = const { Cell::new(None) }; +} + /// A Device-global lock for all snatchable data. pub struct SnatchLock { lock: RwLock<()>, @@ -76,7 +85,24 @@ impl SnatchLock { } /// Request read access to snatchable resources. + #[track_caller] pub fn read(&self) -> SnatchGuard { + if cfg!(debug_assertions) { + let caller = Location::caller(); + let backtrace = Backtrace::capture(); + if let Some((prev, bt)) = READ_LOCK_LOCATION.take() { + let current = thread::current(); + let name = current.name().unwrap_or("<unnamed>"); + panic!( + "thread '{name}' attempted to acquire a snatch read lock recursively.\n + - {prev}\n{bt}\n + - {caller}\n{backtrace}" + ); + } else { + READ_LOCK_LOCATION.set(Some((caller, backtrace))); + } + } + SnatchGuard(self.lock.read()) } @@ -89,3 +115,10 @@ impl SnatchLock { ExclusiveSnatchGuard(self.lock.write()) } } + +impl Drop for SnatchGuard<'_> { + fn drop(&mut self) { + #[cfg(debug_assertions)] + READ_LOCK_LOCATION.take(); + } +} diff --git a/third_party/rust/wgpu-core/src/track/buffer.rs b/third_party/rust/wgpu-core/src/track/buffer.rs index a30ac2a225..6cf1fdda6f 100644 --- a/third_party/rust/wgpu-core/src/track/buffer.rs +++ b/third_party/rust/wgpu-core/src/track/buffer.rs @@ -108,23 +108,27 @@ impl<A: HalApi> BufferBindGroupState<A> { #[derive(Debug)] pub(crate) struct BufferUsageScope<A: HalApi> { state: Vec<BufferUses>, - metadata: ResourceMetadata<Buffer<A>>, } -impl<A: HalApi> BufferUsageScope<A> { - pub fn new() -> Self { +impl<A: HalApi> Default for BufferUsageScope<A> { + fn default() -> Self { Self { state: Vec::new(), - metadata: ResourceMetadata::new(), } } +} +impl<A: HalApi> BufferUsageScope<A> { fn tracker_assert_in_bounds(&self, index: usize) { strict_assert!(index < self.state.len()); self.metadata.tracker_assert_in_bounds(index); } + pub fn clear(&mut self) { + self.state.clear(); + self.metadata.clear(); + } /// Sets the size of all the vectors inside the tracker. /// diff --git a/third_party/rust/wgpu-core/src/track/metadata.rs b/third_party/rust/wgpu-core/src/track/metadata.rs index 744783a7fa..3e71e0e084 100644 --- a/third_party/rust/wgpu-core/src/track/metadata.rs +++ b/third_party/rust/wgpu-core/src/track/metadata.rs @@ -39,6 +39,11 @@ impl<T: Resource> ResourceMetadata<T> { resize_bitvec(&mut self.owned, size); } + pub(super) fn clear(&mut self) { + self.resources.clear(); + self.owned.clear(); + } + /// Ensures a given index is in bounds for all arrays and does /// sanity checks of the presence of a refcount. /// diff --git a/third_party/rust/wgpu-core/src/track/mod.rs b/third_party/rust/wgpu-core/src/track/mod.rs index 9ca37ebadc..374dfe7493 100644 --- a/third_party/rust/wgpu-core/src/track/mod.rs +++ b/third_party/rust/wgpu-core/src/track/mod.rs @@ -480,8 +480,8 @@ impl<A: HalApi> RenderBundleScope<A> { /// Create the render bundle scope and pull the maximum IDs from the hubs. pub fn new() -> Self { Self { - buffers: RwLock::new(BufferUsageScope::new()), - textures: RwLock::new(TextureUsageScope::new()), + buffers: RwLock::new(BufferUsageScope::default()), + textures: RwLock::new(TextureUsageScope::default()), bind_groups: RwLock::new(StatelessTracker::new()), render_pipelines: RwLock::new(StatelessTracker::new()), query_sets: RwLock::new(StatelessTracker::new()), @@ -512,28 +512,52 @@ impl<A: HalApi> RenderBundleScope<A> { } } +/// A pool for storing the memory used by [`UsageScope`]s. We take and store this memory when the +/// scope is dropped to avoid reallocating. The memory required only grows and allocation cost is +/// significant when a large number of resources have been used. +pub(crate) type UsageScopePool<A> = Mutex<Vec<(BufferUsageScope<A>, TextureUsageScope<A>)>>; + /// A usage scope tracker. Only needs to store stateful resources as stateless /// resources cannot possibly have a usage conflict. #[derive(Debug)] -pub(crate) struct UsageScope<A: HalApi> { +pub(crate) struct UsageScope<'a, A: HalApi> { + pub pool: &'a UsageScopePool<A>, pub buffers: BufferUsageScope<A>, pub textures: TextureUsageScope<A>, } -impl<A: HalApi> UsageScope<A> { - /// Create the render bundle scope and pull the maximum IDs from the hubs. - pub fn new(tracker_indices: &TrackerIndexAllocators) -> Self { - let mut value = Self { - buffers: BufferUsageScope::new(), - textures: TextureUsageScope::new(), - }; +impl<'a, A: HalApi> Drop for UsageScope<'a, A> { + fn drop(&mut self) { + // clear vecs and push into pool + self.buffers.clear(); + self.textures.clear(); + self.pool.lock().push(( + std::mem::take(&mut self.buffers), + std::mem::take(&mut self.textures), + )); + } +} - value.buffers.set_size(tracker_indices.buffers.size()); - value.textures.set_size(tracker_indices.textures.size()); +impl<A: HalApi> UsageScope<'static, A> { + pub fn new_pooled<'d>( + pool: &'d UsageScopePool<A>, + tracker_indices: &TrackerIndexAllocators, + ) -> UsageScope<'d, A> { + let pooled = pool.lock().pop().unwrap_or_default(); + + let mut scope = UsageScope::<'d, A> { + pool, + buffers: pooled.0, + textures: pooled.1, + }; - value + scope.buffers.set_size(tracker_indices.buffers.size()); + scope.textures.set_size(tracker_indices.textures.size()); + scope } +} +impl<'a, A: HalApi> UsageScope<'a, A> { /// Merge the inner contents of a bind group into the usage scope. /// /// Only stateful things are merged in here, all other resources are owned diff --git a/third_party/rust/wgpu-core/src/track/texture.rs b/third_party/rust/wgpu-core/src/track/texture.rs index e7c4707c93..3cf95ff38a 100644 --- a/third_party/rust/wgpu-core/src/track/texture.rs +++ b/third_party/rust/wgpu-core/src/track/texture.rs @@ -210,6 +210,7 @@ pub(crate) struct TextureStateSet { simple: Vec<TextureUses>, complex: FastHashMap<usize, ComplexTextureState>, } + impl TextureStateSet { fn new() -> Self { Self { @@ -235,15 +236,16 @@ pub(crate) struct TextureUsageScope<A: HalApi> { metadata: ResourceMetadata<Texture<A>>, } -impl<A: HalApi> TextureUsageScope<A> { - pub fn new() -> Self { +impl<A: HalApi> Default for TextureUsageScope<A> { + fn default() -> Self { Self { set: TextureStateSet::new(), - metadata: ResourceMetadata::new(), } } +} +impl<A: HalApi> TextureUsageScope<A> { fn tracker_assert_in_bounds(&self, index: usize) { self.metadata.tracker_assert_in_bounds(index); @@ -258,6 +260,11 @@ impl<A: HalApi> TextureUsageScope<A> { }); } + pub fn clear(&mut self) { + self.set.clear(); + self.metadata.clear(); + } + /// Sets the size of all the vectors inside the tracker. /// /// Must be called with the highest possible Texture ID before |