diff options
Diffstat (limited to 'third_party/rust/wgpu-core/src/device/global.rs')
| -rw-r--r-- | third_party/rust/wgpu-core/src/device/global.rs | 245 |
1 files changed, 136 insertions, 109 deletions
diff --git a/third_party/rust/wgpu-core/src/device/global.rs b/third_party/rust/wgpu-core/src/device/global.rs index 64fd6d4de7..539b92e0f3 100644 --- a/third_party/rust/wgpu-core/src/device/global.rs +++ b/third_party/rust/wgpu-core/src/device/global.rs @@ -26,9 +26,7 @@ use wgt::{BufferAddress, TextureFormat}; use std::{ borrow::Cow, - iter, - ops::Range, - ptr, + iter, ptr, sync::{atomic::Ordering, Arc}, }; @@ -219,7 +217,7 @@ impl Global { mapped_at_creation: false, }; let stage = match device.create_buffer(&stage_desc, true) { - Ok(stage) => stage, + Ok(stage) => Arc::new(stage), Err(e) => { to_destroy.push(buffer); break e; @@ -232,14 +230,10 @@ impl Global { Ok(mapping) => mapping, Err(e) => { to_destroy.push(buffer); - to_destroy.push(stage); break CreateBufferError::Device(e.into()); } }; - let stage_fid = hub.buffers.request(); - let stage = stage_fid.init(stage); - assert_eq!(buffer.size % wgt::COPY_BUFFER_ALIGNMENT, 0); // Zero initialize memory and then mark both staging and buffer as initialized // (it's guaranteed that this is the case by the time the buffer is usable) @@ -262,7 +256,7 @@ impl Global { .trackers .lock() .buffers - .insert_single(id, resource, buffer_use); + .insert_single(resource, buffer_use); return (id, None); }; @@ -383,7 +377,7 @@ impl Global { .buffers .get(buffer_id) .map_err(|_| BufferAccessError::Invalid)?; - check_buffer_usage(buffer.usage, wgt::BufferUsages::MAP_WRITE)?; + check_buffer_usage(buffer_id, buffer.usage, wgt::BufferUsages::MAP_WRITE)?; //assert!(buffer isn't used by the GPU); #[cfg(feature = "trace")] @@ -446,7 +440,7 @@ impl Global { .buffers .get(buffer_id) .map_err(|_| BufferAccessError::Invalid)?; - check_buffer_usage(buffer.usage, wgt::BufferUsages::MAP_READ)?; + check_buffer_usage(buffer_id, buffer.usage, wgt::BufferUsages::MAP_READ)?; //assert!(buffer isn't used by the GPU); let raw_buf = buffer @@ -529,7 +523,7 @@ impl Global { .lock_life() .suspected_resources .buffers - .insert(buffer_id, buffer); + .insert(buffer.info.tracker_index(), buffer); } if wait { @@ -573,11 +567,11 @@ impl Global { let (id, resource) = fid.assign(texture); api_log!("Device::create_texture({desc:?}) -> {id:?}"); - device.trackers.lock().textures.insert_single( - id, - resource, - hal::TextureUses::UNINITIALIZED, - ); + device + .trackers + .lock() + .textures + .insert_single(resource, hal::TextureUses::UNINITIALIZED); return (id, None); }; @@ -647,11 +641,11 @@ impl Global { let (id, resource) = fid.assign(texture); api_log!("Device::create_texture({desc:?}) -> {id:?}"); - device.trackers.lock().textures.insert_single( - id, - resource, - hal::TextureUses::UNINITIALIZED, - ); + device + .trackers + .lock() + .textures + .insert_single(resource, hal::TextureUses::UNINITIALIZED); return (id, None); }; @@ -704,7 +698,7 @@ impl Global { .trackers .lock() .buffers - .insert_single(id, buffer, hal::BufferUses::empty()); + .insert_single(buffer, hal::BufferUses::empty()); return (id, None); }; @@ -764,7 +758,7 @@ impl Global { .lock_life() .suspected_resources .textures - .insert(texture_id, texture.clone()); + .insert(texture.info.tracker_index(), texture.clone()); } } @@ -824,7 +818,7 @@ impl Global { } api_log!("Texture::create_view({texture_id:?}) -> {id:?}"); - device.trackers.lock().views.insert_single(id, resource); + device.trackers.lock().views.insert_single(resource); return (id, None); }; @@ -854,7 +848,7 @@ impl Global { .lock_life() .suspected_resources .texture_views - .insert(texture_view_id, view.clone()); + .insert(view.info.tracker_index(), view.clone()); if wait { match view.device.wait_for_submit(last_submit_index) { @@ -900,7 +894,7 @@ impl Global { let (id, resource) = fid.assign(sampler); api_log!("Device::create_sampler -> {id:?}"); - device.trackers.lock().samplers.insert_single(id, resource); + device.trackers.lock().samplers.insert_single(resource); return (id, None); }; @@ -925,7 +919,7 @@ impl Global { .lock_life() .suspected_resources .samplers - .insert(sampler_id, sampler.clone()); + .insert(sampler.info.tracker_index(), sampler.clone()); } } @@ -1024,7 +1018,7 @@ impl Global { .lock_life() .suspected_resources .bind_group_layouts - .insert(bind_group_layout_id, layout.clone()); + .insert(layout.info.tracker_index(), layout.clone()); } } @@ -1085,7 +1079,7 @@ impl Global { .lock_life() .suspected_resources .pipeline_layouts - .insert(pipeline_layout_id, layout.clone()); + .insert(layout.info.tracker_index(), layout.clone()); } } @@ -1140,11 +1134,7 @@ impl Global { api_log!("Device::create_bind_group -> {id:?}"); - device - .trackers - .lock() - .bind_groups - .insert_single(id, resource); + device.trackers.lock().bind_groups.insert_single(resource); return (id, None); }; @@ -1168,7 +1158,7 @@ impl Global { .lock_life() .suspected_resources .bind_groups - .insert(bind_group_id, bind_group.clone()); + .insert(bind_group.info.tracker_index(), bind_group.clone()); } } @@ -1332,9 +1322,8 @@ impl Global { if !device.is_valid() { break DeviceError::Lost; } - let queue = match hub.queues.get(device.queue_id.read().unwrap()) { - Ok(queue) => queue, - Err(_) => break DeviceError::InvalidQueueId, + let Some(queue) = device.get_queue() else { + break DeviceError::InvalidQueueId; }; let encoder = match device .command_allocator @@ -1379,6 +1368,7 @@ impl Global { .command_buffers .unregister(command_encoder_id.transmute()) { + cmd_buf.data.lock().as_mut().unwrap().encoder.discard(); cmd_buf .device .untrack(&cmd_buf.data.lock().as_ref().unwrap().trackers); @@ -1450,7 +1440,7 @@ impl Global { let (id, resource) = fid.assign(render_bundle); api_log!("RenderBundleEncoder::finish -> {id:?}"); - device.trackers.lock().bundles.insert_single(id, resource); + device.trackers.lock().bundles.insert_single(resource); return (id, None); }; @@ -1474,7 +1464,7 @@ impl Global { .lock_life() .suspected_resources .render_bundles - .insert(render_bundle_id, bundle.clone()); + .insert(bundle.info.tracker_index(), bundle.clone()); } } @@ -1513,11 +1503,7 @@ impl Global { let (id, resource) = fid.assign(query_set); api_log!("Device::create_query_set -> {id:?}"); - device - .trackers - .lock() - .query_sets - .insert_single(id, resource); + device.trackers.lock().query_sets.insert_single(resource); return (id, None); }; @@ -1544,7 +1530,7 @@ impl Global { .lock_life() .suspected_resources .query_sets - .insert(query_set_id, query_set.clone()); + .insert(query_set.info.tracker_index(), query_set.clone()); } } @@ -1600,7 +1586,7 @@ impl Global { .trackers .lock() .render_pipelines - .insert_single(id, resource); + .insert_single(resource); return (id, None); }; @@ -1672,18 +1658,17 @@ impl Global { let hub = A::hub(self); if let Some(pipeline) = hub.render_pipelines.unregister(render_pipeline_id) { - let layout_id = pipeline.layout.as_info().id(); let device = &pipeline.device; let mut life_lock = device.lock_life(); life_lock .suspected_resources .render_pipelines - .insert(render_pipeline_id, pipeline.clone()); + .insert(pipeline.info.tracker_index(), pipeline.clone()); - life_lock - .suspected_resources - .pipeline_layouts - .insert(layout_id, pipeline.layout.clone()); + life_lock.suspected_resources.pipeline_layouts.insert( + pipeline.layout.info.tracker_index(), + pipeline.layout.clone(), + ); } } @@ -1734,7 +1719,7 @@ impl Global { .trackers .lock() .compute_pipelines - .insert_single(id, resource); + .insert_single(resource); return (id, None); }; @@ -1804,17 +1789,16 @@ impl Global { let hub = A::hub(self); if let Some(pipeline) = hub.compute_pipelines.unregister(compute_pipeline_id) { - let layout_id = pipeline.layout.as_info().id(); let device = &pipeline.device; let mut life_lock = device.lock_life(); life_lock .suspected_resources .compute_pipelines - .insert(compute_pipeline_id, pipeline.clone()); - life_lock - .suspected_resources - .pipeline_layouts - .insert(layout_id, pipeline.layout.clone()); + .insert(pipeline.info.tracker_index(), pipeline.clone()); + life_lock.suspected_resources.pipeline_layouts.insert( + pipeline.layout.info.tracker_index(), + pipeline.layout.clone(), + ); } } @@ -2113,28 +2097,41 @@ impl Global { .get(device_id) .map_err(|_| DeviceError::Invalid)?; - let (closures, queue_empty) = { - if let wgt::Maintain::WaitForSubmissionIndex(submission_index) = maintain { - if submission_index.queue_id != device_id.transmute() { - return Err(WaitIdleError::WrongSubmissionIndex( - submission_index.queue_id, - device_id, - )); - } + if let wgt::Maintain::WaitForSubmissionIndex(submission_index) = maintain { + if submission_index.queue_id != device_id.transmute() { + return Err(WaitIdleError::WrongSubmissionIndex( + submission_index.queue_id, + device_id, + )); } + } - let fence = device.fence.read(); - let fence = fence.as_ref().unwrap(); - device.maintain(fence, maintain)? - }; + let DevicePoll { + closures, + queue_empty, + } = Self::poll_single_device(&device, maintain)?; + + closures.fire(); + + Ok(queue_empty) + } + + fn poll_single_device<A: HalApi>( + device: &crate::device::Device<A>, + maintain: wgt::Maintain<queue::WrappedSubmissionIndex>, + ) -> Result<DevicePoll, WaitIdleError> { + let fence = device.fence.read(); + let fence = fence.as_ref().unwrap(); + let (closures, queue_empty) = device.maintain(fence, maintain)?; // Some deferred destroys are scheduled in maintain so run this right after // to avoid holding on to them until the next device poll. device.deferred_resource_destruction(); - closures.fire(); - - Ok(queue_empty) + Ok(DevicePoll { + closures, + queue_empty, + }) } /// Poll all devices belonging to the backend `A`. @@ -2143,7 +2140,7 @@ impl Global { /// /// Return `all_queue_empty` indicating whether there are more queue /// submissions still in flight. - fn poll_device<A: HalApi>( + fn poll_all_devices_of_api<A: HalApi>( &self, force_wait: bool, closures: &mut UserClosures, @@ -2161,10 +2158,13 @@ impl Global { } else { wgt::Maintain::Poll }; - let fence = device.fence.read(); - let fence = fence.as_ref().unwrap(); - let (cbs, queue_empty) = device.maintain(fence, maintain)?; - all_queue_empty = all_queue_empty && queue_empty; + + let DevicePoll { + closures: cbs, + queue_empty, + } = Self::poll_single_device(device, maintain)?; + + all_queue_empty &= queue_empty; closures.extend(cbs); } @@ -2186,23 +2186,23 @@ impl Global { #[cfg(vulkan)] { - all_queue_empty = - self.poll_device::<hal::api::Vulkan>(force_wait, &mut closures)? && all_queue_empty; + all_queue_empty &= + self.poll_all_devices_of_api::<hal::api::Vulkan>(force_wait, &mut closures)?; } #[cfg(metal)] { - all_queue_empty = - self.poll_device::<hal::api::Metal>(force_wait, &mut closures)? && all_queue_empty; + all_queue_empty &= + self.poll_all_devices_of_api::<hal::api::Metal>(force_wait, &mut closures)?; } #[cfg(dx12)] { - all_queue_empty = - self.poll_device::<hal::api::Dx12>(force_wait, &mut closures)? && all_queue_empty; + all_queue_empty &= + self.poll_all_devices_of_api::<hal::api::Dx12>(force_wait, &mut closures)?; } #[cfg(gles)] { - all_queue_empty = - self.poll_device::<hal::api::Gles>(force_wait, &mut closures)? && all_queue_empty; + all_queue_empty &= + self.poll_all_devices_of_api::<hal::api::Gles>(force_wait, &mut closures)?; } closures.fire(); @@ -2336,15 +2336,18 @@ impl Global { pub fn buffer_map_async<A: HalApi>( &self, buffer_id: id::BufferId, - range: Range<BufferAddress>, + offset: BufferAddress, + size: Option<BufferAddress>, op: BufferMapOperation, ) -> BufferAccessResult { - api_log!("Buffer::map_async {buffer_id:?} range {range:?} op: {op:?}"); + api_log!("Buffer::map_async {buffer_id:?} offset {offset:?} size {size:?} op: {op:?}"); // User callbacks must not be called while holding buffer_map_async_inner's locks, so we // defer the error callback if it needs to be called immediately (typically when running // into errors). - if let Err((mut operation, err)) = self.buffer_map_async_inner::<A>(buffer_id, range, op) { + if let Err((mut operation, err)) = + self.buffer_map_async_inner::<A>(buffer_id, offset, size, op) + { if let Some(callback) = operation.callback.take() { callback.call(Err(err.clone())); } @@ -2360,7 +2363,8 @@ impl Global { fn buffer_map_async_inner<A: HalApi>( &self, buffer_id: id::BufferId, - range: Range<BufferAddress>, + offset: BufferAddress, + size: Option<BufferAddress>, op: BufferMapOperation, ) -> Result<(), (BufferMapOperation, BufferAccessError)> { profiling::scope!("Buffer::map_async"); @@ -2372,29 +2376,50 @@ impl Global { HostMap::Write => (wgt::BufferUsages::MAP_WRITE, hal::BufferUses::MAP_WRITE), }; - if range.start % wgt::MAP_ALIGNMENT != 0 || range.end % wgt::COPY_BUFFER_ALIGNMENT != 0 { - return Err((op, BufferAccessError::UnalignedRange)); - } - let buffer = { - let buffer = hub - .buffers - .get(buffer_id) - .map_err(|_| BufferAccessError::Invalid); + let buffer = hub.buffers.get(buffer_id); let buffer = match buffer { Ok(b) => b, - Err(e) => { - return Err((op, e)); + Err(_) => { + return Err((op, BufferAccessError::Invalid)); } }; + { + let snatch_guard = buffer.device.snatchable_lock.read(); + if buffer.is_destroyed(&snatch_guard) { + return Err((op, BufferAccessError::Destroyed)); + } + } + + let range_size = if let Some(size) = size { + size + } else if offset > buffer.size { + 0 + } else { + buffer.size - offset + }; + + if offset % wgt::MAP_ALIGNMENT != 0 { + return Err((op, BufferAccessError::UnalignedOffset { offset })); + } + if range_size % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err((op, BufferAccessError::UnalignedRangeSize { range_size })); + } + + let range = offset..(offset + range_size); + + if range.start % wgt::MAP_ALIGNMENT != 0 || range.end % wgt::COPY_BUFFER_ALIGNMENT != 0 + { + return Err((op, BufferAccessError::UnalignedRange)); + } let device = &buffer.device; if !device.is_valid() { return Err((op, DeviceError::Lost.into())); } - if let Err(e) = check_buffer_usage(buffer.usage, pub_usage) { + if let Err(e) = check_buffer_usage(buffer.info.id(), buffer.usage, pub_usage) { return Err((op, e.into())); } @@ -2417,11 +2442,6 @@ impl Global { )); } - let snatch_guard = device.snatchable_lock.read(); - if buffer.is_destroyed(&snatch_guard) { - return Err((op, BufferAccessError::Destroyed)); - } - { let map_state = &mut *buffer.map_state.lock(); *map_state = match *map_state { @@ -2442,6 +2462,8 @@ impl Global { }; } + let snatch_guard = buffer.device.snatchable_lock.read(); + { let mut trackers = buffer.device.as_ref().trackers.lock(); trackers.buffers.set_single(&buffer, internal_use); @@ -2557,3 +2579,8 @@ impl Global { buffer.unmap() } } + +struct DevicePoll { + closures: UserClosures, + queue_empty: bool, +} |