diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-12 05:43:14 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-12 05:43:14 +0000 |
commit | 8dd16259287f58f9273002717ec4d27e97127719 (patch) | |
tree | 3863e62a53829a84037444beab3abd4ed9dfc7d0 /third_party/rust/wgpu-core/src/command | |
parent | Releasing progress-linux version 126.0.1-1~progress7.99u1. (diff) | |
download | firefox-8dd16259287f58f9273002717ec4d27e97127719.tar.xz firefox-8dd16259287f58f9273002717ec4d27e97127719.zip |
Merging upstream version 127.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/rust/wgpu-core/src/command')
-rw-r--r-- | third_party/rust/wgpu-core/src/command/allocator.rs | 67 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/bundle.rs | 21 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/clear.rs | 9 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/compute.rs | 318 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/compute_command.rs | 322 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/mod.rs | 249 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/query.rs | 11 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/render.rs | 168 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/transfer.rs | 35 |
9 files changed, 830 insertions, 370 deletions
diff --git a/third_party/rust/wgpu-core/src/command/allocator.rs b/third_party/rust/wgpu-core/src/command/allocator.rs new file mode 100644 index 0000000000..e17fd08d76 --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/allocator.rs @@ -0,0 +1,67 @@ +use crate::hal_api::HalApi; +use crate::resource_log; +use hal::Device as _; + +use crate::lock::{rank, Mutex}; + +/// A pool of free [`wgpu_hal::CommandEncoder`]s, owned by a `Device`. +/// +/// Each encoder in this list is in the "closed" state. +/// +/// Since a raw [`CommandEncoder`][ce] is itself a pool for allocating +/// raw [`CommandBuffer`][cb]s, this is a pool of pools. +/// +/// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder +/// [ce]: hal::CommandEncoder +/// [cb]: hal::Api::CommandBuffer +pub(crate) struct CommandAllocator<A: HalApi> { + free_encoders: Mutex<Vec<A::CommandEncoder>>, +} + +impl<A: HalApi> CommandAllocator<A> { + pub(crate) fn new() -> Self { + Self { + free_encoders: Mutex::new(rank::COMMAND_ALLOCATOR_FREE_ENCODERS, Vec::new()), + } + } + + /// Return a fresh [`wgpu_hal::CommandEncoder`] in the "closed" state. + /// + /// If we have free encoders in the pool, take one of those. Otherwise, + /// create a new one on `device`. + /// + /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder + pub(crate) fn acquire_encoder( + &self, + device: &A::Device, + queue: &A::Queue, + ) -> Result<A::CommandEncoder, hal::DeviceError> { + let mut free_encoders = self.free_encoders.lock(); + match free_encoders.pop() { + Some(encoder) => Ok(encoder), + None => unsafe { + let hal_desc = hal::CommandEncoderDescriptor { label: None, queue }; + device.create_command_encoder(&hal_desc) + }, + } + } + + /// Add `encoder` back to the free pool. + pub(crate) fn release_encoder(&self, encoder: A::CommandEncoder) { + let mut free_encoders = self.free_encoders.lock(); + free_encoders.push(encoder); + } + + /// Free the pool of command encoders. + /// + /// This is only called when the `Device` is dropped. + pub(crate) fn dispose(&self, device: &A::Device) { + let mut free_encoders = self.free_encoders.lock(); + resource_log!("CommandAllocator::dispose encoders {}", free_encoders.len()); + for cmd_encoder in free_encoders.drain(..) { + unsafe { + device.destroy_command_encoder(cmd_encoder); + } + } + } +} diff --git a/third_party/rust/wgpu-core/src/command/bundle.rs b/third_party/rust/wgpu-core/src/command/bundle.rs index 47beda8ec6..d9d821c533 100644 --- a/third_party/rust/wgpu-core/src/command/bundle.rs +++ b/third_party/rust/wgpu-core/src/command/bundle.rs @@ -73,7 +73,7 @@ index format changes. [Gdcrbe]: crate::global::Global::device_create_render_bundle_encoder [Grbef]: crate::global::Global::render_bundle_encoder_finish -[wrpeb]: crate::command::render_ffi::wgpu_render_pass_execute_bundles +[wrpeb]: crate::command::render::render_commands::wgpu_render_pass_execute_bundles !*/ #![allow(clippy::reversed_empty_ranges)] @@ -113,7 +113,7 @@ use hal::CommandEncoder as _; use super::ArcRenderCommand; -/// https://gpuweb.github.io/gpuweb/#dom-gpurendercommandsmixin-draw +/// <https://gpuweb.github.io/gpuweb/#dom-gpurendercommandsmixin-draw> fn validate_draw<A: HalApi>( vertex: &[Option<VertexState<A>>], step: &[VertexStep], @@ -1548,15 +1548,14 @@ pub mod bundle_ffi { offsets: *const DynamicOffset, offset_length: usize, ) { - let redundant = unsafe { - bundle.current_bind_groups.set_and_check_redundant( - bind_group_id, - index, - &mut bundle.base.dynamic_offsets, - offsets, - offset_length, - ) - }; + let offsets = unsafe { slice::from_raw_parts(offsets, offset_length) }; + + let redundant = bundle.current_bind_groups.set_and_check_redundant( + bind_group_id, + index, + &mut bundle.base.dynamic_offsets, + offsets, + ); if redundant { return; diff --git a/third_party/rust/wgpu-core/src/command/clear.rs b/third_party/rust/wgpu-core/src/command/clear.rs index 72c923f82e..faff177928 100644 --- a/third_party/rust/wgpu-core/src/command/clear.rs +++ b/third_party/rust/wgpu-core/src/command/clear.rs @@ -104,6 +104,11 @@ impl Global { let dst_buffer = buffer_guard .get(dst) .map_err(|_| ClearError::InvalidBuffer(dst))?; + + if dst_buffer.device.as_info().id() != cmd_buf.device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + cmd_buf_data .trackers .buffers @@ -200,6 +205,10 @@ impl Global { .get(dst) .map_err(|_| ClearError::InvalidTexture(dst))?; + if dst_texture.device.as_info().id() != cmd_buf.device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + // Check if subresource aspects are valid. let clear_aspects = hal::FormatAspects::new(dst_texture.desc.format, subresource_range.aspect); diff --git a/third_party/rust/wgpu-core/src/command/compute.rs b/third_party/rust/wgpu-core/src/command/compute.rs index b38324984c..4ee48f0086 100644 --- a/third_party/rust/wgpu-core/src/command/compute.rs +++ b/third_party/rust/wgpu-core/src/command/compute.rs @@ -1,3 +1,4 @@ +use crate::command::compute_command::{ArcComputeCommand, ComputeCommand}; use crate::device::DeviceError; use crate::resource::Resource; use crate::snatch::SnatchGuard; @@ -20,7 +21,6 @@ use crate::{ hal_label, id, id::DeviceId, init_tracker::MemoryInitKind, - pipeline, resource::{self}, storage::Storage, track::{Tracker, UsageConflict, UsageScope}, @@ -36,61 +36,9 @@ use serde::Serialize; use thiserror::Error; +use std::sync::Arc; use std::{fmt, mem, str}; -#[doc(hidden)] -#[derive(Clone, Copy, Debug)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub enum ComputeCommand { - SetBindGroup { - index: u32, - num_dynamic_offsets: usize, - bind_group_id: id::BindGroupId, - }, - SetPipeline(id::ComputePipelineId), - - /// Set a range of push constants to values stored in [`BasePass::push_constant_data`]. - SetPushConstant { - /// The byte offset within the push constant storage to write to. This - /// must be a multiple of four. - offset: u32, - - /// The number of bytes to write. This must be a multiple of four. - size_bytes: u32, - - /// Index in [`BasePass::push_constant_data`] of the start of the data - /// to be written. - /// - /// Note: this is not a byte offset like `offset`. Rather, it is the - /// index of the first `u32` element in `push_constant_data` to read. - values_offset: u32, - }, - - Dispatch([u32; 3]), - DispatchIndirect { - buffer_id: id::BufferId, - offset: wgt::BufferAddress, - }, - PushDebugGroup { - color: u32, - len: usize, - }, - PopDebugGroup, - InsertDebugMarker { - color: u32, - len: usize, - }, - WriteTimestamp { - query_set_id: id::QuerySetId, - query_index: u32, - }, - BeginPipelineStatisticsQuery { - query_set_id: id::QuerySetId, - query_index: u32, - }, - EndPipelineStatisticsQuery, -} - #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct ComputePass { base: BasePass<ComputeCommand>, @@ -184,7 +132,7 @@ pub enum ComputePassErrorInner { #[error(transparent)] Encoder(#[from] CommandEncoderError), #[error("Bind group at index {0:?} is invalid")] - InvalidBindGroup(usize), + InvalidBindGroup(u32), #[error("Device {0:?} is invalid")] InvalidDevice(DeviceId), #[error("Bind group index {index} is greater than the device's requested `max_bind_group` limit {max}")] @@ -249,7 +197,7 @@ impl PrettyError for ComputePassErrorInner { pub struct ComputePassError { pub scope: PassErrorScope, #[source] - inner: ComputePassErrorInner, + pub(super) inner: ComputePassErrorInner, } impl PrettyError for ComputePassError { fn fmt_pretty(&self, fmt: &mut ErrorFormatter) { @@ -346,7 +294,8 @@ impl Global { encoder_id: id::CommandEncoderId, pass: &ComputePass, ) -> Result<(), ComputePassError> { - self.command_encoder_run_compute_pass_impl::<A>( + // TODO: This should go directly to `command_encoder_run_compute_pass_impl` by means of storing `ArcComputeCommand` internally. + self.command_encoder_run_compute_pass_with_unresolved_commands::<A>( encoder_id, pass.base.as_ref(), pass.timestamp_writes.as_ref(), @@ -354,18 +303,41 @@ impl Global { } #[doc(hidden)] - pub fn command_encoder_run_compute_pass_impl<A: HalApi>( + pub fn command_encoder_run_compute_pass_with_unresolved_commands<A: HalApi>( &self, encoder_id: id::CommandEncoderId, base: BasePassRef<ComputeCommand>, timestamp_writes: Option<&ComputePassTimestampWrites>, ) -> Result<(), ComputePassError> { + let resolved_commands = + ComputeCommand::resolve_compute_command_ids(A::hub(self), base.commands)?; + + self.command_encoder_run_compute_pass_impl::<A>( + encoder_id, + BasePassRef { + label: base.label, + commands: &resolved_commands, + dynamic_offsets: base.dynamic_offsets, + string_data: base.string_data, + push_constant_data: base.push_constant_data, + }, + timestamp_writes, + ) + } + + fn command_encoder_run_compute_pass_impl<A: HalApi>( + &self, + encoder_id: id::CommandEncoderId, + base: BasePassRef<ArcComputeCommand<A>>, + timestamp_writes: Option<&ComputePassTimestampWrites>, + ) -> Result<(), ComputePassError> { profiling::scope!("CommandEncoder::run_compute_pass"); let pass_scope = PassErrorScope::Pass(encoder_id); let hub = A::hub(self); - let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id).map_pass_err(pass_scope)?; + let cmd_buf: Arc<CommandBuffer<A>> = + CommandBuffer::get_encoder(hub, encoder_id).map_pass_err(pass_scope)?; let device = &cmd_buf.device; if !device.is_valid() { return Err(ComputePassErrorInner::InvalidDevice( @@ -380,7 +352,13 @@ impl Global { #[cfg(feature = "trace")] if let Some(ref mut list) = cmd_buf_data.commands { list.push(crate::device::trace::Command::RunComputePass { - base: BasePass::from_ref(base), + base: BasePass { + label: base.label.map(str::to_string), + commands: base.commands.iter().map(Into::into).collect(), + dynamic_offsets: base.dynamic_offsets.to_vec(), + string_data: base.string_data.to_vec(), + push_constant_data: base.push_constant_data.to_vec(), + }, timestamp_writes: timestamp_writes.cloned(), }); } @@ -400,9 +378,7 @@ impl Global { let raw = encoder.open().map_pass_err(pass_scope)?; let bind_group_guard = hub.bind_groups.read(); - let pipeline_guard = hub.compute_pipelines.read(); let query_set_guard = hub.query_sets.read(); - let buffer_guard = hub.buffers.read(); let mut state = State { binder: Binder::new(), @@ -480,19 +456,21 @@ impl Global { // be inserted before texture reads. let mut pending_discard_init_fixups = SurfacesInDiscardState::new(); + // TODO: We should be draining the commands here, avoiding extra copies in the process. + // (A command encoder can't be executed twice!) for command in base.commands { - match *command { - ComputeCommand::SetBindGroup { + match command { + ArcComputeCommand::SetBindGroup { index, num_dynamic_offsets, - bind_group_id, + bind_group, } => { - let scope = PassErrorScope::SetBindGroup(bind_group_id); + let scope = PassErrorScope::SetBindGroup(bind_group.as_info().id()); let max_bind_groups = cmd_buf.limits.max_bind_groups; - if index >= max_bind_groups { + if index >= &max_bind_groups { return Err(ComputePassErrorInner::BindGroupIndexOutOfRange { - index, + index: *index, max: max_bind_groups, }) .map_pass_err(scope); @@ -505,13 +483,9 @@ impl Global { ); dynamic_offset_count += num_dynamic_offsets; - let bind_group = tracker - .bind_groups - .add_single(&*bind_group_guard, bind_group_id) - .ok_or(ComputePassErrorInner::InvalidBindGroup(index as usize)) - .map_pass_err(scope)?; + let bind_group = tracker.bind_groups.insert_single(bind_group.clone()); bind_group - .validate_dynamic_bindings(index, &temp_offsets, &cmd_buf.limits) + .validate_dynamic_bindings(*index, &temp_offsets, &cmd_buf.limits) .map_pass_err(scope)?; buffer_memory_init_actions.extend( @@ -533,14 +507,14 @@ impl Global { let entries = state .binder - .assign_group(index as usize, bind_group, &temp_offsets); + .assign_group(*index as usize, bind_group, &temp_offsets); if !entries.is_empty() && pipeline_layout.is_some() { let pipeline_layout = pipeline_layout.as_ref().unwrap().raw(); for (i, e) in entries.iter().enumerate() { if let Some(group) = e.group.as_ref() { let raw_bg = group .raw(&snatch_guard) - .ok_or(ComputePassErrorInner::InvalidBindGroup(i)) + .ok_or(ComputePassErrorInner::InvalidBindGroup(i as u32)) .map_pass_err(scope)?; unsafe { raw.set_bind_group( @@ -554,16 +528,13 @@ impl Global { } } } - ComputeCommand::SetPipeline(pipeline_id) => { + ArcComputeCommand::SetPipeline(pipeline) => { + let pipeline_id = pipeline.as_info().id(); let scope = PassErrorScope::SetPipelineCompute(pipeline_id); state.pipeline = Some(pipeline_id); - let pipeline: &pipeline::ComputePipeline<A> = tracker - .compute_pipelines - .add_single(&*pipeline_guard, pipeline_id) - .ok_or(ComputePassErrorInner::InvalidPipeline(pipeline_id)) - .map_pass_err(scope)?; + tracker.compute_pipelines.insert_single(pipeline.clone()); unsafe { raw.set_compute_pipeline(pipeline.raw()); @@ -587,7 +558,7 @@ impl Global { if let Some(group) = e.group.as_ref() { let raw_bg = group .raw(&snatch_guard) - .ok_or(ComputePassErrorInner::InvalidBindGroup(i)) + .ok_or(ComputePassErrorInner::InvalidBindGroup(i as u32)) .map_pass_err(scope)?; unsafe { raw.set_bind_group( @@ -623,7 +594,7 @@ impl Global { } } } - ComputeCommand::SetPushConstant { + ArcComputeCommand::SetPushConstant { offset, size_bytes, values_offset, @@ -634,7 +605,7 @@ impl Global { let values_end_offset = (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; let data_slice = - &base.push_constant_data[(values_offset as usize)..values_end_offset]; + &base.push_constant_data[(*values_offset as usize)..values_end_offset]; let pipeline_layout = state .binder @@ -649,7 +620,7 @@ impl Global { pipeline_layout .validate_push_constant_ranges( wgt::ShaderStages::COMPUTE, - offset, + *offset, end_offset_bytes, ) .map_pass_err(scope)?; @@ -658,12 +629,12 @@ impl Global { raw.set_push_constants( pipeline_layout.raw(), wgt::ShaderStages::COMPUTE, - offset, + *offset, data_slice, ); } } - ComputeCommand::Dispatch(groups) => { + ArcComputeCommand::Dispatch(groups) => { let scope = PassErrorScope::Dispatch { indirect: false, pipeline: state.pipeline, @@ -688,7 +659,7 @@ impl Global { { return Err(ComputePassErrorInner::Dispatch( DispatchError::InvalidGroupSize { - current: groups, + current: *groups, limit: groups_size_limit, }, )) @@ -696,10 +667,11 @@ impl Global { } unsafe { - raw.dispatch(groups); + raw.dispatch(*groups); } } - ComputeCommand::DispatchIndirect { buffer_id, offset } => { + ArcComputeCommand::DispatchIndirect { buffer, offset } => { + let buffer_id = buffer.as_info().id(); let scope = PassErrorScope::Dispatch { indirect: true, pipeline: state.pipeline, @@ -711,29 +683,25 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer = state + state .scope .buffers - .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .insert_merge_single(buffer.clone(), hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; + check_buffer_usage(buffer_id, buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; - check_buffer_usage( - buffer_id, - indirect_buffer.usage, - wgt::BufferUsages::INDIRECT, - ) - .map_pass_err(scope)?; let end_offset = offset + mem::size_of::<wgt::DispatchIndirectArgs>() as u64; - if end_offset > indirect_buffer.size { + if end_offset > buffer.size { return Err(ComputePassErrorInner::IndirectBufferOverrun { - offset, + offset: *offset, end_offset, - buffer_size: indirect_buffer.size, + buffer_size: buffer.size, }) .map_pass_err(scope); } - let buf_raw = indirect_buffer + let buf_raw = buffer .raw .get(&snatch_guard) .ok_or(ComputePassErrorInner::InvalidIndirectBuffer(buffer_id)) @@ -742,9 +710,9 @@ impl Global { let stride = 3 * 4; // 3 integers, x/y/z group size buffer_memory_init_actions.extend( - indirect_buffer.initialization_status.read().create_action( - indirect_buffer, - offset..(offset + stride), + buffer.initialization_status.read().create_action( + buffer, + *offset..(*offset + stride), MemoryInitKind::NeedsInitializedMemory, ), ); @@ -754,15 +722,15 @@ impl Global { raw, &mut intermediate_trackers, &*bind_group_guard, - Some(indirect_buffer.as_info().tracker_index()), + Some(buffer.as_info().tracker_index()), &snatch_guard, ) .map_pass_err(scope)?; unsafe { - raw.dispatch_indirect(buf_raw, offset); + raw.dispatch_indirect(buf_raw, *offset); } } - ComputeCommand::PushDebugGroup { color: _, len } => { + ArcComputeCommand::PushDebugGroup { color: _, len } => { state.debug_scope_depth += 1; if !discard_hal_labels { let label = @@ -774,7 +742,7 @@ impl Global { } string_offset += len; } - ComputeCommand::PopDebugGroup => { + ArcComputeCommand::PopDebugGroup => { let scope = PassErrorScope::PopDebugGroup; if state.debug_scope_depth == 0 { @@ -788,7 +756,7 @@ impl Global { } } } - ComputeCommand::InsertDebugMarker { color: _, len } => { + ArcComputeCommand::InsertDebugMarker { color: _, len } => { if !discard_hal_labels { let label = str::from_utf8(&base.string_data[string_offset..string_offset + len]) @@ -797,49 +765,43 @@ impl Global { } string_offset += len; } - ComputeCommand::WriteTimestamp { - query_set_id, + ArcComputeCommand::WriteTimestamp { + query_set, query_index, } => { + let query_set_id = query_set.as_info().id(); let scope = PassErrorScope::WriteTimestamp; device .require_features(wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES) .map_pass_err(scope)?; - let query_set: &resource::QuerySet<A> = tracker - .query_sets - .add_single(&*query_set_guard, query_set_id) - .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id)) - .map_pass_err(scope)?; + let query_set = tracker.query_sets.insert_single(query_set.clone()); query_set - .validate_and_write_timestamp(raw, query_set_id, query_index, None) + .validate_and_write_timestamp(raw, query_set_id, *query_index, None) .map_pass_err(scope)?; } - ComputeCommand::BeginPipelineStatisticsQuery { - query_set_id, + ArcComputeCommand::BeginPipelineStatisticsQuery { + query_set, query_index, } => { + let query_set_id = query_set.as_info().id(); let scope = PassErrorScope::BeginPipelineStatisticsQuery; - let query_set: &resource::QuerySet<A> = tracker - .query_sets - .add_single(&*query_set_guard, query_set_id) - .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id)) - .map_pass_err(scope)?; + let query_set = tracker.query_sets.insert_single(query_set.clone()); query_set .validate_and_begin_pipeline_statistics_query( raw, query_set_id, - query_index, + *query_index, None, &mut active_query, ) .map_pass_err(scope)?; } - ComputeCommand::EndPipelineStatisticsQuery => { + ArcComputeCommand::EndPipelineStatisticsQuery => { let scope = PassErrorScope::EndPipelineStatisticsQuery; end_pipeline_statistics_query(raw, &*query_set_guard, &mut active_query) @@ -883,33 +845,24 @@ impl Global { } } -pub mod compute_ffi { +pub mod compute_commands { use super::{ComputeCommand, ComputePass}; - use crate::{id, RawString}; - use std::{convert::TryInto, ffi, slice}; + use crate::id; + use std::convert::TryInto; use wgt::{BufferAddress, DynamicOffset}; - /// # Safety - /// - /// This function is unsafe as there is no guarantee that the given pointer is - /// valid for `offset_length` elements. - #[no_mangle] - pub unsafe extern "C" fn wgpu_compute_pass_set_bind_group( + pub fn wgpu_compute_pass_set_bind_group( pass: &mut ComputePass, index: u32, bind_group_id: id::BindGroupId, - offsets: *const DynamicOffset, - offset_length: usize, + offsets: &[DynamicOffset], ) { - let redundant = unsafe { - pass.current_bind_groups.set_and_check_redundant( - bind_group_id, - index, - &mut pass.base.dynamic_offsets, - offsets, - offset_length, - ) - }; + let redundant = pass.current_bind_groups.set_and_check_redundant( + bind_group_id, + index, + &mut pass.base.dynamic_offsets, + offsets, + ); if redundant { return; @@ -917,13 +870,12 @@ pub mod compute_ffi { pass.base.commands.push(ComputeCommand::SetBindGroup { index, - num_dynamic_offsets: offset_length, + num_dynamic_offsets: offsets.len(), bind_group_id, }); } - #[no_mangle] - pub extern "C" fn wgpu_compute_pass_set_pipeline( + pub fn wgpu_compute_pass_set_pipeline( pass: &mut ComputePass, pipeline_id: id::ComputePipelineId, ) { @@ -936,47 +888,34 @@ pub mod compute_ffi { .push(ComputeCommand::SetPipeline(pipeline_id)); } - /// # Safety - /// - /// This function is unsafe as there is no guarantee that the given pointer is - /// valid for `size_bytes` bytes. - #[no_mangle] - pub unsafe extern "C" fn wgpu_compute_pass_set_push_constant( - pass: &mut ComputePass, - offset: u32, - size_bytes: u32, - data: *const u8, - ) { + pub fn wgpu_compute_pass_set_push_constant(pass: &mut ComputePass, offset: u32, data: &[u8]) { assert_eq!( offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), 0, "Push constant offset must be aligned to 4 bytes." ); assert_eq!( - size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + data.len() as u32 & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), 0, "Push constant size must be aligned to 4 bytes." ); - let data_slice = unsafe { slice::from_raw_parts(data, size_bytes as usize) }; let value_offset = pass.base.push_constant_data.len().try_into().expect( "Ran out of push constant space. Don't set 4gb of push constants per ComputePass.", ); pass.base.push_constant_data.extend( - data_slice - .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize) + data.chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize) .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])), ); pass.base.commands.push(ComputeCommand::SetPushConstant { offset, - size_bytes, + size_bytes: data.len() as u32, values_offset: value_offset, }); } - #[no_mangle] - pub extern "C" fn wgpu_compute_pass_dispatch_workgroups( + pub fn wgpu_compute_pass_dispatch_workgroups( pass: &mut ComputePass, groups_x: u32, groups_y: u32, @@ -987,8 +926,7 @@ pub mod compute_ffi { .push(ComputeCommand::Dispatch([groups_x, groups_y, groups_z])); } - #[no_mangle] - pub extern "C" fn wgpu_compute_pass_dispatch_workgroups_indirect( + pub fn wgpu_compute_pass_dispatch_workgroups_indirect( pass: &mut ComputePass, buffer_id: id::BufferId, offset: BufferAddress, @@ -998,17 +936,8 @@ pub mod compute_ffi { .push(ComputeCommand::DispatchIndirect { buffer_id, offset }); } - /// # Safety - /// - /// This function is unsafe as there is no guarantee that the given `label` - /// is a valid null-terminated string. - #[no_mangle] - pub unsafe extern "C" fn wgpu_compute_pass_push_debug_group( - pass: &mut ComputePass, - label: RawString, - color: u32, - ) { - let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes(); + pub fn wgpu_compute_pass_push_debug_group(pass: &mut ComputePass, label: &str, color: u32) { + let bytes = label.as_bytes(); pass.base.string_data.extend_from_slice(bytes); pass.base.commands.push(ComputeCommand::PushDebugGroup { @@ -1017,22 +946,12 @@ pub mod compute_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_compute_pass_pop_debug_group(pass: &mut ComputePass) { + pub fn wgpu_compute_pass_pop_debug_group(pass: &mut ComputePass) { pass.base.commands.push(ComputeCommand::PopDebugGroup); } - /// # Safety - /// - /// This function is unsafe as there is no guarantee that the given `label` - /// is a valid null-terminated string. - #[no_mangle] - pub unsafe extern "C" fn wgpu_compute_pass_insert_debug_marker( - pass: &mut ComputePass, - label: RawString, - color: u32, - ) { - let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes(); + pub fn wgpu_compute_pass_insert_debug_marker(pass: &mut ComputePass, label: &str, color: u32) { + let bytes = label.as_bytes(); pass.base.string_data.extend_from_slice(bytes); pass.base.commands.push(ComputeCommand::InsertDebugMarker { @@ -1041,8 +960,7 @@ pub mod compute_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_compute_pass_write_timestamp( + pub fn wgpu_compute_pass_write_timestamp( pass: &mut ComputePass, query_set_id: id::QuerySetId, query_index: u32, @@ -1053,8 +971,7 @@ pub mod compute_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_compute_pass_begin_pipeline_statistics_query( + pub fn wgpu_compute_pass_begin_pipeline_statistics_query( pass: &mut ComputePass, query_set_id: id::QuerySetId, query_index: u32, @@ -1067,8 +984,7 @@ pub mod compute_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_compute_pass_end_pipeline_statistics_query(pass: &mut ComputePass) { + pub fn wgpu_compute_pass_end_pipeline_statistics_query(pass: &mut ComputePass) { pass.base .commands .push(ComputeCommand::EndPipelineStatisticsQuery); diff --git a/third_party/rust/wgpu-core/src/command/compute_command.rs b/third_party/rust/wgpu-core/src/command/compute_command.rs new file mode 100644 index 0000000000..49fdbbec24 --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/compute_command.rs @@ -0,0 +1,322 @@ +use std::sync::Arc; + +use crate::{ + binding_model::BindGroup, + hal_api::HalApi, + id, + pipeline::ComputePipeline, + resource::{Buffer, QuerySet}, +}; + +use super::{ComputePassError, ComputePassErrorInner, PassErrorScope}; + +#[derive(Clone, Copy, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ComputeCommand { + SetBindGroup { + index: u32, + num_dynamic_offsets: usize, + bind_group_id: id::BindGroupId, + }, + + SetPipeline(id::ComputePipelineId), + + /// Set a range of push constants to values stored in `push_constant_data`. + SetPushConstant { + /// The byte offset within the push constant storage to write to. This + /// must be a multiple of four. + offset: u32, + + /// The number of bytes to write. This must be a multiple of four. + size_bytes: u32, + + /// Index in `push_constant_data` of the start of the data + /// to be written. + /// + /// Note: this is not a byte offset like `offset`. Rather, it is the + /// index of the first `u32` element in `push_constant_data` to read. + values_offset: u32, + }, + + Dispatch([u32; 3]), + + DispatchIndirect { + buffer_id: id::BufferId, + offset: wgt::BufferAddress, + }, + + PushDebugGroup { + color: u32, + len: usize, + }, + + PopDebugGroup, + + InsertDebugMarker { + color: u32, + len: usize, + }, + + WriteTimestamp { + query_set_id: id::QuerySetId, + query_index: u32, + }, + + BeginPipelineStatisticsQuery { + query_set_id: id::QuerySetId, + query_index: u32, + }, + + EndPipelineStatisticsQuery, +} + +impl ComputeCommand { + /// Resolves all ids in a list of commands into the corresponding resource Arc. + /// + // TODO: Once resolving is done on-the-fly during recording, this function should be only needed with the replay feature: + // #[cfg(feature = "replay")] + pub fn resolve_compute_command_ids<A: HalApi>( + hub: &crate::hub::Hub<A>, + commands: &[ComputeCommand], + ) -> Result<Vec<ArcComputeCommand<A>>, ComputePassError> { + let buffers_guard = hub.buffers.read(); + let bind_group_guard = hub.bind_groups.read(); + let query_set_guard = hub.query_sets.read(); + let pipelines_guard = hub.compute_pipelines.read(); + + let resolved_commands: Vec<ArcComputeCommand<A>> = commands + .iter() + .map(|c| -> Result<ArcComputeCommand<A>, ComputePassError> { + Ok(match *c { + ComputeCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group_id, + } => ArcComputeCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group: bind_group_guard.get_owned(bind_group_id).map_err(|_| { + ComputePassError { + scope: PassErrorScope::SetBindGroup(bind_group_id), + inner: ComputePassErrorInner::InvalidBindGroup(index), + } + })?, + }, + + ComputeCommand::SetPipeline(pipeline_id) => ArcComputeCommand::SetPipeline( + pipelines_guard + .get_owned(pipeline_id) + .map_err(|_| ComputePassError { + scope: PassErrorScope::SetPipelineCompute(pipeline_id), + inner: ComputePassErrorInner::InvalidPipeline(pipeline_id), + })?, + ), + + ComputeCommand::SetPushConstant { + offset, + size_bytes, + values_offset, + } => ArcComputeCommand::SetPushConstant { + offset, + size_bytes, + values_offset, + }, + + ComputeCommand::Dispatch(dim) => ArcComputeCommand::Dispatch(dim), + + ComputeCommand::DispatchIndirect { buffer_id, offset } => { + ArcComputeCommand::DispatchIndirect { + buffer: buffers_guard.get_owned(buffer_id).map_err(|_| { + ComputePassError { + scope: PassErrorScope::Dispatch { + indirect: true, + pipeline: None, // TODO: not used right now, but once we do the resolve during recording we can use this again. + }, + inner: ComputePassErrorInner::InvalidBuffer(buffer_id), + } + })?, + offset, + } + } + + ComputeCommand::PushDebugGroup { color, len } => { + ArcComputeCommand::PushDebugGroup { color, len } + } + + ComputeCommand::PopDebugGroup => ArcComputeCommand::PopDebugGroup, + + ComputeCommand::InsertDebugMarker { color, len } => { + ArcComputeCommand::InsertDebugMarker { color, len } + } + + ComputeCommand::WriteTimestamp { + query_set_id, + query_index, + } => ArcComputeCommand::WriteTimestamp { + query_set: query_set_guard.get_owned(query_set_id).map_err(|_| { + ComputePassError { + scope: PassErrorScope::WriteTimestamp, + inner: ComputePassErrorInner::InvalidQuerySet(query_set_id), + } + })?, + query_index, + }, + + ComputeCommand::BeginPipelineStatisticsQuery { + query_set_id, + query_index, + } => ArcComputeCommand::BeginPipelineStatisticsQuery { + query_set: query_set_guard.get_owned(query_set_id).map_err(|_| { + ComputePassError { + scope: PassErrorScope::BeginPipelineStatisticsQuery, + inner: ComputePassErrorInner::InvalidQuerySet(query_set_id), + } + })?, + query_index, + }, + + ComputeCommand::EndPipelineStatisticsQuery => { + ArcComputeCommand::EndPipelineStatisticsQuery + } + }) + }) + .collect::<Result<Vec<_>, ComputePassError>>()?; + Ok(resolved_commands) + } +} + +/// Equivalent to `ComputeCommand` but the Ids resolved into resource Arcs. +#[derive(Clone, Debug)] +pub enum ArcComputeCommand<A: HalApi> { + SetBindGroup { + index: u32, + num_dynamic_offsets: usize, + bind_group: Arc<BindGroup<A>>, + }, + + SetPipeline(Arc<ComputePipeline<A>>), + + /// Set a range of push constants to values stored in `push_constant_data`. + SetPushConstant { + /// The byte offset within the push constant storage to write to. This + /// must be a multiple of four. + offset: u32, + + /// The number of bytes to write. This must be a multiple of four. + size_bytes: u32, + + /// Index in `push_constant_data` of the start of the data + /// to be written. + /// + /// Note: this is not a byte offset like `offset`. Rather, it is the + /// index of the first `u32` element in `push_constant_data` to read. + values_offset: u32, + }, + + Dispatch([u32; 3]), + + DispatchIndirect { + buffer: Arc<Buffer<A>>, + offset: wgt::BufferAddress, + }, + + PushDebugGroup { + color: u32, + len: usize, + }, + + PopDebugGroup, + + InsertDebugMarker { + color: u32, + len: usize, + }, + + WriteTimestamp { + query_set: Arc<QuerySet<A>>, + query_index: u32, + }, + + BeginPipelineStatisticsQuery { + query_set: Arc<QuerySet<A>>, + query_index: u32, + }, + + EndPipelineStatisticsQuery, +} + +#[cfg(feature = "trace")] +impl<A: HalApi> From<&ArcComputeCommand<A>> for ComputeCommand { + fn from(value: &ArcComputeCommand<A>) -> Self { + use crate::resource::Resource as _; + + match value { + ArcComputeCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group, + } => ComputeCommand::SetBindGroup { + index: *index, + num_dynamic_offsets: *num_dynamic_offsets, + bind_group_id: bind_group.as_info().id(), + }, + + ArcComputeCommand::SetPipeline(pipeline) => { + ComputeCommand::SetPipeline(pipeline.as_info().id()) + } + + ArcComputeCommand::SetPushConstant { + offset, + size_bytes, + values_offset, + } => ComputeCommand::SetPushConstant { + offset: *offset, + size_bytes: *size_bytes, + values_offset: *values_offset, + }, + + ArcComputeCommand::Dispatch(dim) => ComputeCommand::Dispatch(*dim), + + ArcComputeCommand::DispatchIndirect { buffer, offset } => { + ComputeCommand::DispatchIndirect { + buffer_id: buffer.as_info().id(), + offset: *offset, + } + } + + ArcComputeCommand::PushDebugGroup { color, len } => ComputeCommand::PushDebugGroup { + color: *color, + len: *len, + }, + + ArcComputeCommand::PopDebugGroup => ComputeCommand::PopDebugGroup, + + ArcComputeCommand::InsertDebugMarker { color, len } => { + ComputeCommand::InsertDebugMarker { + color: *color, + len: *len, + } + } + + ArcComputeCommand::WriteTimestamp { + query_set, + query_index, + } => ComputeCommand::WriteTimestamp { + query_set_id: query_set.as_info().id(), + query_index: *query_index, + }, + + ArcComputeCommand::BeginPipelineStatisticsQuery { + query_set, + query_index, + } => ComputeCommand::BeginPipelineStatisticsQuery { + query_set_id: query_set.as_info().id(), + query_index: *query_index, + }, + + ArcComputeCommand::EndPipelineStatisticsQuery => { + ComputeCommand::EndPipelineStatisticsQuery + } + } + } +} diff --git a/third_party/rust/wgpu-core/src/command/mod.rs b/third_party/rust/wgpu-core/src/command/mod.rs index febed4fc97..d53f47bf42 100644 --- a/third_party/rust/wgpu-core/src/command/mod.rs +++ b/third_party/rust/wgpu-core/src/command/mod.rs @@ -1,20 +1,23 @@ +mod allocator; mod bind; mod bundle; mod clear; mod compute; +mod compute_command; mod draw; mod memory_init; mod query; mod render; mod transfer; -use std::slice; use std::sync::Arc; pub(crate) use self::clear::clear_texture; pub use self::{ - bundle::*, clear::ClearError, compute::*, draw::*, query::*, render::*, transfer::*, + bundle::*, clear::ClearError, compute::*, compute_command::ComputeCommand, draw::*, query::*, + render::*, transfer::*, }; +pub(crate) use allocator::CommandAllocator; use self::memory_init::CommandBufferTextureMemoryActions; @@ -22,6 +25,7 @@ use crate::device::{Device, DeviceError}; use crate::error::{ErrorFormatter, PrettyError}; use crate::hub::Hub; use crate::id::CommandBufferId; +use crate::lock::{rank, Mutex}; use crate::snatch::SnatchGuard; use crate::init_tracker::BufferInitTrackerAction; @@ -30,7 +34,6 @@ use crate::track::{Tracker, UsageScope}; use crate::{api_log, global::Global, hal_api::HalApi, id, resource_log, Label}; use hal::CommandEncoder as _; -use parking_lot::Mutex; use thiserror::Error; #[cfg(feature = "trace")] @@ -38,23 +41,122 @@ use crate::device::trace::Command as TraceCommand; const PUSH_CONSTANT_CLEAR_ARRAY: &[u32] = &[0_u32; 64]; +/// The current state of a [`CommandBuffer`]. #[derive(Debug)] pub(crate) enum CommandEncoderStatus { + /// Ready to record commands. An encoder's initial state. + /// + /// Command building methods like [`command_encoder_clear_buffer`] and + /// [`command_encoder_run_compute_pass`] require the encoder to be in this + /// state. + /// + /// [`command_encoder_clear_buffer`]: Global::command_encoder_clear_buffer + /// [`command_encoder_run_compute_pass`]: Global::command_encoder_run_compute_pass Recording, + + /// Command recording is complete, and the buffer is ready for submission. + /// + /// [`Global::command_encoder_finish`] transitions a + /// `CommandBuffer` from the `Recording` state into this state. + /// + /// [`Global::queue_submit`] drops command buffers unless they are + /// in this state. Finished, + + /// An error occurred while recording a compute or render pass. + /// + /// When a `CommandEncoder` is left in this state, we have also + /// returned an error result from the function that encountered + /// the problem. Future attempts to use the encoder (that is, + /// calls to [`CommandBuffer::get_encoder`]) will also return + /// errors. + /// + /// Calling [`Global::command_encoder_finish`] in this state + /// discards the command buffer under construction. Error, } +/// A raw [`CommandEncoder`][rce], and the raw [`CommandBuffer`][rcb]s built from it. +/// +/// Each wgpu-core [`CommandBuffer`] owns an instance of this type, which is +/// where the commands are actually stored. +/// +/// This holds a `Vec` of raw [`CommandBuffer`][rcb]s, not just one. We are not +/// always able to record commands in the order in which they must ultimately be +/// submitted to the queue, but raw command buffers don't permit inserting new +/// commands into the middle of a recorded stream. However, hal queue submission +/// accepts a series of command buffers at once, so we can simply break the +/// stream up into multiple buffers, and then reorder the buffers. See +/// [`CommandEncoder::close_and_swap`] for a specific example of this. +/// +/// Note that a [`CommandEncoderId`] actually refers to a [`CommandBuffer`]. +/// Methods that take a command encoder id actually look up the command buffer, +/// and then use its encoder. +/// +/// [rce]: hal::Api::CommandEncoder +/// [rcb]: hal::Api::CommandBuffer +/// [`CommandEncoderId`]: crate::id::CommandEncoderId pub(crate) struct CommandEncoder<A: HalApi> { + /// The underlying `wgpu_hal` [`CommandEncoder`]. + /// + /// Successfully executed command buffers' encoders are saved in a + /// [`CommandAllocator`] for recycling. + /// + /// [`CommandEncoder`]: hal::Api::CommandEncoder + /// [`CommandAllocator`]: crate::command::CommandAllocator raw: A::CommandEncoder, + + /// All the raw command buffers for our owning [`CommandBuffer`], in + /// submission order. + /// + /// These command buffers were all constructed with `raw`. The + /// [`wgpu_hal::CommandEncoder`] trait forbids these from outliving `raw`, + /// and requires that we provide all of these when we call + /// [`raw.reset_all()`][CE::ra], so the encoder and its buffers travel + /// together. + /// + /// [CE::ra]: hal::CommandEncoder::reset_all + /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder list: Vec<A::CommandBuffer>, + + /// True if `raw` is in the "recording" state. + /// + /// See the documentation for [`wgpu_hal::CommandEncoder`] for + /// details on the states `raw` can be in. + /// + /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder is_open: bool, + label: Option<String>, } //TODO: handle errors better impl<A: HalApi> CommandEncoder<A> { - /// Closes the live encoder + /// Finish the current command buffer, if any, and place it + /// at the second-to-last position in our list. + /// + /// If we have opened this command encoder, finish its current + /// command buffer, and insert it just before the last element in + /// [`self.list`][l]. If this command buffer is closed, do nothing. + /// + /// On return, the underlying hal encoder is closed. + /// + /// What is this for? + /// + /// The `wgpu_hal` contract requires that each render or compute pass's + /// commands be preceded by calls to [`transition_buffers`] and + /// [`transition_textures`], to put the resources the pass operates on in + /// the appropriate state. Unfortunately, we don't know which transitions + /// are needed until we're done recording the pass itself. Rather than + /// iterating over the pass twice, we note the necessary transitions as we + /// record its commands, finish the raw command buffer for the actual pass, + /// record a new raw command buffer for the transitions, and jam that buffer + /// in just before the pass's. This is the function that jams in the + /// transitions' command buffer. + /// + /// [l]: CommandEncoder::list + /// [`transition_buffers`]: hal::CommandEncoder::transition_buffers + /// [`transition_textures`]: hal::CommandEncoder::transition_textures fn close_and_swap(&mut self) -> Result<(), DeviceError> { if self.is_open { self.is_open = false; @@ -65,6 +167,16 @@ impl<A: HalApi> CommandEncoder<A> { Ok(()) } + /// Finish the current command buffer, if any, and add it to the + /// end of [`self.list`][l]. + /// + /// If we have opened this command encoder, finish its current + /// command buffer, and push it onto the end of [`self.list`][l]. + /// If this command buffer is closed, do nothing. + /// + /// On return, the underlying hal encoder is closed. + /// + /// [l]: CommandEncoder::list fn close(&mut self) -> Result<(), DeviceError> { if self.is_open { self.is_open = false; @@ -75,6 +187,9 @@ impl<A: HalApi> CommandEncoder<A> { Ok(()) } + /// Discard the command buffer under construction, if any. + /// + /// The underlying hal encoder is closed, if it was recording. pub(crate) fn discard(&mut self) { if self.is_open { self.is_open = false; @@ -82,7 +197,10 @@ impl<A: HalApi> CommandEncoder<A> { } } - fn open(&mut self) -> Result<&mut A::CommandEncoder, DeviceError> { + /// Begin recording a new command buffer, if we haven't already. + /// + /// The underlying hal encoder is put in the "recording" state. + pub(crate) fn open(&mut self) -> Result<&mut A::CommandEncoder, DeviceError> { if !self.is_open { self.is_open = true; let label = self.label.as_deref(); @@ -92,6 +210,10 @@ impl<A: HalApi> CommandEncoder<A> { Ok(&mut self.raw) } + /// Begin recording a new command buffer for a render pass, with + /// its own label. + /// + /// The underlying hal encoder is put in the "recording" state. fn open_pass(&mut self, label: Option<&str>) -> Result<(), DeviceError> { self.is_open = true; unsafe { self.raw.begin_encoding(label)? }; @@ -100,7 +222,7 @@ impl<A: HalApi> CommandEncoder<A> { } } -pub struct BakedCommands<A: HalApi> { +pub(crate) struct BakedCommands<A: HalApi> { pub(crate) encoder: A::CommandEncoder, pub(crate) list: Vec<A::CommandBuffer>, pub(crate) trackers: Tracker<A>, @@ -111,12 +233,29 @@ pub struct BakedCommands<A: HalApi> { pub(crate) struct DestroyedBufferError(pub id::BufferId); pub(crate) struct DestroyedTextureError(pub id::TextureId); +/// The mutable state of a [`CommandBuffer`]. pub struct CommandBufferMutable<A: HalApi> { + /// The [`wgpu_hal::Api::CommandBuffer`]s we've built so far, and the encoder + /// they belong to. + /// + /// [`wgpu_hal::Api::CommandBuffer`]: hal::Api::CommandBuffer pub(crate) encoder: CommandEncoder<A>, + + /// The current state of this command buffer's encoder. status: CommandEncoderStatus, + + /// All the resources that the commands recorded so far have referred to. pub(crate) trackers: Tracker<A>, + + /// The regions of buffers and textures these commands will read and write. + /// + /// This is used to determine which portions of which + /// buffers/textures we actually need to initialize. If we're + /// definitely going to write to something before we read from it, + /// we don't need to clear its contents. buffer_memory_init_actions: Vec<BufferInitTrackerAction<A>>, texture_memory_actions: CommandBufferTextureMemoryActions<A>, + pub(crate) pending_query_resets: QueryResetMap<A>, #[cfg(feature = "trace")] pub(crate) commands: Option<Vec<TraceCommand>>, @@ -133,11 +272,36 @@ impl<A: HalApi> CommandBufferMutable<A> { } } +/// A buffer of commands to be submitted to the GPU for execution. +/// +/// Whereas the WebGPU API uses two separate types for command buffers and +/// encoders, this type is a fusion of the two: +/// +/// - During command recording, this holds a [`CommandEncoder`] accepting this +/// buffer's commands. In this state, the [`CommandBuffer`] type behaves like +/// a WebGPU `GPUCommandEncoder`. +/// +/// - Once command recording is finished by calling +/// [`Global::command_encoder_finish`], no further recording is allowed. The +/// internal [`CommandEncoder`] is retained solely as a storage pool for the +/// raw command buffers. In this state, the value behaves like a WebGPU +/// `GPUCommandBuffer`. +/// +/// - Once a command buffer is submitted to the queue, it is removed from the id +/// registry, and its contents are taken to construct a [`BakedCommands`], +/// whose contents eventually become the property of the submission queue. pub struct CommandBuffer<A: HalApi> { pub(crate) device: Arc<Device<A>>, limits: wgt::Limits, support_clear_texture: bool, pub(crate) info: ResourceInfo<CommandBuffer<A>>, + + /// The mutable state of this command buffer. + /// + /// This `Option` is populated when the command buffer is first created. + /// When this is submitted, dropped, or destroyed, its contents are + /// extracted into a [`BakedCommands`] by + /// [`CommandBuffer::extract_baked_commands`]. pub(crate) data: Mutex<Option<CommandBufferMutable<A>>>, } @@ -176,25 +340,28 @@ impl<A: HalApi> CommandBuffer<A> { .as_str(), None, ), - data: Mutex::new(Some(CommandBufferMutable { - encoder: CommandEncoder { - raw: encoder, - is_open: false, - list: Vec::new(), - label, - }, - status: CommandEncoderStatus::Recording, - trackers: Tracker::new(), - buffer_memory_init_actions: Default::default(), - texture_memory_actions: Default::default(), - pending_query_resets: QueryResetMap::new(), - #[cfg(feature = "trace")] - commands: if enable_tracing { - Some(Vec::new()) - } else { - None - }, - })), + data: Mutex::new( + rank::COMMAND_BUFFER_DATA, + Some(CommandBufferMutable { + encoder: CommandEncoder { + raw: encoder, + is_open: false, + list: Vec::new(), + label, + }, + status: CommandEncoderStatus::Recording, + trackers: Tracker::new(), + buffer_memory_init_actions: Default::default(), + texture_memory_actions: Default::default(), + pending_query_resets: QueryResetMap::new(), + #[cfg(feature = "trace")] + commands: if enable_tracing { + Some(Vec::new()) + } else { + None + }, + }), + ), } } @@ -248,12 +415,18 @@ impl<A: HalApi> CommandBuffer<A> { } impl<A: HalApi> CommandBuffer<A> { + /// Return the [`CommandBuffer`] for `id`, for recording new commands. + /// + /// In `wgpu_core`, the [`CommandBuffer`] type serves both as encoder and + /// buffer, which is why this function takes an [`id::CommandEncoderId`] but + /// returns a [`CommandBuffer`]. The returned command buffer must be in the + /// "recording" state. Otherwise, an error is returned. fn get_encoder( hub: &Hub<A>, id: id::CommandEncoderId, ) -> Result<Arc<Self>, CommandEncoderError> { let storage = hub.command_buffers.read(); - match storage.get(id.transmute()) { + match storage.get(id.into_command_buffer_id()) { Ok(cmd_buf) => match cmd_buf.data.lock().as_ref().unwrap().status { CommandEncoderStatus::Recording => Ok(cmd_buf.clone()), CommandEncoderStatus::Finished => Err(CommandEncoderError::NotRecording), @@ -286,11 +459,9 @@ impl<A: HalApi> CommandBuffer<A> { } pub(crate) fn from_arc_into_baked(self: Arc<Self>) -> BakedCommands<A> { - if let Some(mut command_buffer) = Arc::into_inner(self) { - command_buffer.extract_baked_commands() - } else { - panic!("CommandBuffer cannot be destroyed because is still in use"); - } + let mut command_buffer = Arc::into_inner(self) + .expect("CommandBuffer cannot be destroyed because is still in use"); + command_buffer.extract_baked_commands() } } @@ -418,7 +589,7 @@ impl Global { let hub = A::hub(self); - let error = match hub.command_buffers.get(encoder_id.transmute()) { + let error = match hub.command_buffers.get(encoder_id.into_command_buffer_id()) { Ok(cmd_buf) => { let mut cmd_buf_data = cmd_buf.data.lock(); let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); @@ -444,7 +615,7 @@ impl Global { Err(_) => Some(CommandEncoderError::Invalid), }; - (encoder_id.transmute(), error) + (encoder_id.into_command_buffer_id(), error) } pub fn command_encoder_push_debug_group<A: HalApi>( @@ -599,16 +770,15 @@ impl BindGroupStateChange { } } - unsafe fn set_and_check_redundant( + fn set_and_check_redundant( &mut self, bind_group_id: id::BindGroupId, index: u32, dynamic_offsets: &mut Vec<u32>, - offsets: *const wgt::DynamicOffset, - offset_length: usize, + offsets: &[wgt::DynamicOffset], ) -> bool { // For now never deduplicate bind groups with dynamic offsets. - if offset_length == 0 { + if offsets.is_empty() { // If this get returns None, that means we're well over the limit, // so let the call through to get a proper error if let Some(current_bind_group) = self.last_states.get_mut(index as usize) { @@ -624,8 +794,7 @@ impl BindGroupStateChange { if let Some(current_bind_group) = self.last_states.get_mut(index as usize) { current_bind_group.reset(); } - dynamic_offsets - .extend_from_slice(unsafe { slice::from_raw_parts(offsets, offset_length) }); + dynamic_offsets.extend_from_slice(offsets); } false } @@ -700,7 +869,7 @@ impl PrettyError for PassErrorScope { // This error is not in the error chain, only notes are needed match *self { Self::Pass(id) => { - fmt.command_buffer_label(&id.transmute()); + fmt.command_buffer_label(&id.into_command_buffer_id()); } Self::SetBindGroup(id) => { fmt.bind_group_label(&id); diff --git a/third_party/rust/wgpu-core/src/command/query.rs b/third_party/rust/wgpu-core/src/command/query.rs index 89cba6fbf3..fd3360cc00 100644 --- a/third_party/rust/wgpu-core/src/command/query.rs +++ b/third_party/rust/wgpu-core/src/command/query.rs @@ -9,7 +9,7 @@ use crate::{ hal_api::HalApi, id::{self, Id}, init_tracker::MemoryInitKind, - resource::QuerySet, + resource::{QuerySet, Resource}, storage::Storage, Epoch, FastHashMap, Index, }; @@ -429,11 +429,20 @@ impl Global { .add_single(&*query_set_guard, query_set_id) .ok_or(QueryError::InvalidQuerySet(query_set_id))?; + if query_set.device.as_info().id() != cmd_buf.device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + let (dst_buffer, dst_pending) = { let buffer_guard = hub.buffers.read(); let dst_buffer = buffer_guard .get(destination) .map_err(|_| QueryError::InvalidBuffer(destination))?; + + if dst_buffer.device.as_info().id() != cmd_buf.device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + tracker .buffers .set_single(dst_buffer, hal::BufferUses::COPY_DST) diff --git a/third_party/rust/wgpu-core/src/command/render.rs b/third_party/rust/wgpu-core/src/command/render.rs index 7e859e3cc8..87dc9aac16 100644 --- a/third_party/rust/wgpu-core/src/command/render.rs +++ b/third_party/rust/wgpu-core/src/command/render.rs @@ -1343,7 +1343,8 @@ impl Global { let hub = A::hub(self); - let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id).map_pass_err(pass_scope)?; + let cmd_buf: Arc<CommandBuffer<A>> = + CommandBuffer::get_encoder(hub, encoder_id).map_pass_err(pass_scope)?; let device = &cmd_buf.device; let snatch_guard = device.snatchable_lock.read(); @@ -2409,7 +2410,10 @@ impl Global { (trackers, pending_discard_init_fixups) }; - let cmd_buf = hub.command_buffers.get(encoder_id.transmute()).unwrap(); + let cmd_buf = hub + .command_buffers + .get(encoder_id.into_command_buffer_id()) + .unwrap(); let mut cmd_buf_data = cmd_buf.data.lock(); let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); @@ -2455,36 +2459,27 @@ impl Global { } } -pub mod render_ffi { +pub mod render_commands { use super::{ super::{Rect, RenderCommand}, RenderPass, }; - use crate::{id, RawString}; - use std::{convert::TryInto, ffi, num::NonZeroU32, slice}; + use crate::id; + use std::{convert::TryInto, num::NonZeroU32}; use wgt::{BufferAddress, BufferSize, Color, DynamicOffset, IndexFormat}; - /// # Safety - /// - /// This function is unsafe as there is no guarantee that the given pointer is - /// valid for `offset_length` elements. - #[no_mangle] - pub unsafe extern "C" fn wgpu_render_pass_set_bind_group( + pub fn wgpu_render_pass_set_bind_group( pass: &mut RenderPass, index: u32, bind_group_id: id::BindGroupId, - offsets: *const DynamicOffset, - offset_length: usize, + offsets: &[DynamicOffset], ) { - let redundant = unsafe { - pass.current_bind_groups.set_and_check_redundant( - bind_group_id, - index, - &mut pass.base.dynamic_offsets, - offsets, - offset_length, - ) - }; + let redundant = pass.current_bind_groups.set_and_check_redundant( + bind_group_id, + index, + &mut pass.base.dynamic_offsets, + offsets, + ); if redundant { return; @@ -2492,16 +2487,12 @@ pub mod render_ffi { pass.base.commands.push(RenderCommand::SetBindGroup { index, - num_dynamic_offsets: offset_length, + num_dynamic_offsets: offsets.len(), bind_group_id, }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_set_pipeline( - pass: &mut RenderPass, - pipeline_id: id::RenderPipelineId, - ) { + pub fn wgpu_render_pass_set_pipeline(pass: &mut RenderPass, pipeline_id: id::RenderPipelineId) { if pass.current_pipeline.set_and_check_redundant(pipeline_id) { return; } @@ -2511,8 +2502,7 @@ pub mod render_ffi { .push(RenderCommand::SetPipeline(pipeline_id)); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_set_vertex_buffer( + pub fn wgpu_render_pass_set_vertex_buffer( pass: &mut RenderPass, slot: u32, buffer_id: id::BufferId, @@ -2527,8 +2517,7 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_set_index_buffer( + pub fn wgpu_render_pass_set_index_buffer( pass: &mut RenderPass, buffer: id::BufferId, index_format: IndexFormat, @@ -2538,22 +2527,19 @@ pub mod render_ffi { pass.set_index_buffer(buffer, index_format, offset, size); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_set_blend_constant(pass: &mut RenderPass, color: &Color) { + pub fn wgpu_render_pass_set_blend_constant(pass: &mut RenderPass, color: &Color) { pass.base .commands .push(RenderCommand::SetBlendConstant(*color)); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_set_stencil_reference(pass: &mut RenderPass, value: u32) { + pub fn wgpu_render_pass_set_stencil_reference(pass: &mut RenderPass, value: u32) { pass.base .commands .push(RenderCommand::SetStencilReference(value)); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_set_viewport( + pub fn wgpu_render_pass_set_viewport( pass: &mut RenderPass, x: f32, y: f32, @@ -2569,8 +2555,7 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_set_scissor_rect( + pub fn wgpu_render_pass_set_scissor_rect( pass: &mut RenderPass, x: u32, y: u32, @@ -2582,17 +2567,11 @@ pub mod render_ffi { .push(RenderCommand::SetScissor(Rect { x, y, w, h })); } - /// # Safety - /// - /// This function is unsafe as there is no guarantee that the given pointer is - /// valid for `size_bytes` bytes. - #[no_mangle] - pub unsafe extern "C" fn wgpu_render_pass_set_push_constants( + pub fn wgpu_render_pass_set_push_constants( pass: &mut RenderPass, stages: wgt::ShaderStages, offset: u32, - size_bytes: u32, - data: *const u8, + data: &[u8], ) { assert_eq!( offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), @@ -2600,31 +2579,28 @@ pub mod render_ffi { "Push constant offset must be aligned to 4 bytes." ); assert_eq!( - size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + data.len() as u32 & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), 0, "Push constant size must be aligned to 4 bytes." ); - let data_slice = unsafe { slice::from_raw_parts(data, size_bytes as usize) }; let value_offset = pass.base.push_constant_data.len().try_into().expect( "Ran out of push constant space. Don't set 4gb of push constants per RenderPass.", ); pass.base.push_constant_data.extend( - data_slice - .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize) + data.chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize) .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])), ); pass.base.commands.push(RenderCommand::SetPushConstant { stages, offset, - size_bytes, + size_bytes: data.len() as u32, values_offset: Some(value_offset), }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_draw( + pub fn wgpu_render_pass_draw( pass: &mut RenderPass, vertex_count: u32, instance_count: u32, @@ -2639,8 +2615,7 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_draw_indexed( + pub fn wgpu_render_pass_draw_indexed( pass: &mut RenderPass, index_count: u32, instance_count: u32, @@ -2657,8 +2632,7 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_draw_indirect( + pub fn wgpu_render_pass_draw_indirect( pass: &mut RenderPass, buffer_id: id::BufferId, offset: BufferAddress, @@ -2671,8 +2645,7 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_draw_indexed_indirect( + pub fn wgpu_render_pass_draw_indexed_indirect( pass: &mut RenderPass, buffer_id: id::BufferId, offset: BufferAddress, @@ -2685,8 +2658,7 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_multi_draw_indirect( + pub fn wgpu_render_pass_multi_draw_indirect( pass: &mut RenderPass, buffer_id: id::BufferId, offset: BufferAddress, @@ -2700,8 +2672,7 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_multi_draw_indexed_indirect( + pub fn wgpu_render_pass_multi_draw_indexed_indirect( pass: &mut RenderPass, buffer_id: id::BufferId, offset: BufferAddress, @@ -2715,8 +2686,7 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_multi_draw_indirect_count( + pub fn wgpu_render_pass_multi_draw_indirect_count( pass: &mut RenderPass, buffer_id: id::BufferId, offset: BufferAddress, @@ -2736,8 +2706,7 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_multi_draw_indexed_indirect_count( + pub fn wgpu_render_pass_multi_draw_indexed_indirect_count( pass: &mut RenderPass, buffer_id: id::BufferId, offset: BufferAddress, @@ -2757,17 +2726,8 @@ pub mod render_ffi { }); } - /// # Safety - /// - /// This function is unsafe as there is no guarantee that the given `label` - /// is a valid null-terminated string. - #[no_mangle] - pub unsafe extern "C" fn wgpu_render_pass_push_debug_group( - pass: &mut RenderPass, - label: RawString, - color: u32, - ) { - let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes(); + pub fn wgpu_render_pass_push_debug_group(pass: &mut RenderPass, label: &str, color: u32) { + let bytes = label.as_bytes(); pass.base.string_data.extend_from_slice(bytes); pass.base.commands.push(RenderCommand::PushDebugGroup { @@ -2776,22 +2736,12 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_pop_debug_group(pass: &mut RenderPass) { + pub fn wgpu_render_pass_pop_debug_group(pass: &mut RenderPass) { pass.base.commands.push(RenderCommand::PopDebugGroup); } - /// # Safety - /// - /// This function is unsafe as there is no guarantee that the given `label` - /// is a valid null-terminated string. - #[no_mangle] - pub unsafe extern "C" fn wgpu_render_pass_insert_debug_marker( - pass: &mut RenderPass, - label: RawString, - color: u32, - ) { - let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes(); + pub fn wgpu_render_pass_insert_debug_marker(pass: &mut RenderPass, label: &str, color: u32) { + let bytes = label.as_bytes(); pass.base.string_data.extend_from_slice(bytes); pass.base.commands.push(RenderCommand::InsertDebugMarker { @@ -2800,8 +2750,7 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_write_timestamp( + pub fn wgpu_render_pass_write_timestamp( pass: &mut RenderPass, query_set_id: id::QuerySetId, query_index: u32, @@ -2812,23 +2761,17 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_begin_occlusion_query( - pass: &mut RenderPass, - query_index: u32, - ) { + pub fn wgpu_render_pass_begin_occlusion_query(pass: &mut RenderPass, query_index: u32) { pass.base .commands .push(RenderCommand::BeginOcclusionQuery { query_index }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_end_occlusion_query(pass: &mut RenderPass) { + pub fn wgpu_render_pass_end_occlusion_query(pass: &mut RenderPass) { pass.base.commands.push(RenderCommand::EndOcclusionQuery); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_begin_pipeline_statistics_query( + pub fn wgpu_render_pass_begin_pipeline_statistics_query( pass: &mut RenderPass, query_set_id: id::QuerySetId, query_index: u32, @@ -2841,26 +2784,17 @@ pub mod render_ffi { }); } - #[no_mangle] - pub extern "C" fn wgpu_render_pass_end_pipeline_statistics_query(pass: &mut RenderPass) { + pub fn wgpu_render_pass_end_pipeline_statistics_query(pass: &mut RenderPass) { pass.base .commands .push(RenderCommand::EndPipelineStatisticsQuery); } - /// # Safety - /// - /// This function is unsafe as there is no guarantee that the given pointer is - /// valid for `render_bundle_ids_length` elements. - #[no_mangle] - pub unsafe extern "C" fn wgpu_render_pass_execute_bundles( + pub fn wgpu_render_pass_execute_bundles( pass: &mut RenderPass, - render_bundle_ids: *const id::RenderBundleId, - render_bundle_ids_length: usize, + render_bundle_ids: &[id::RenderBundleId], ) { - for &bundle_id in - unsafe { slice::from_raw_parts(render_bundle_ids, render_bundle_ids_length) } - { + for &bundle_id in render_bundle_ids { pass.base .commands .push(RenderCommand::ExecuteBundle(bundle_id)); diff --git a/third_party/rust/wgpu-core/src/command/transfer.rs b/third_party/rust/wgpu-core/src/command/transfer.rs index 8e98a4c9b9..84bc88e723 100644 --- a/third_party/rust/wgpu-core/src/command/transfer.rs +++ b/third_party/rust/wgpu-core/src/command/transfer.rs @@ -607,6 +607,11 @@ impl Global { let src_buffer = buffer_guard .get(source) .map_err(|_| TransferError::InvalidBuffer(source))?; + + if src_buffer.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + cmd_buf_data .trackers .buffers @@ -628,6 +633,11 @@ impl Global { let dst_buffer = buffer_guard .get(destination) .map_err(|_| TransferError::InvalidBuffer(destination))?; + + if dst_buffer.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + cmd_buf_data .trackers .buffers @@ -777,6 +787,10 @@ impl Global { .get(destination.texture) .map_err(|_| TransferError::InvalidTexture(destination.texture))?; + if dst_texture.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + let (hal_copy_size, array_layer_count) = validate_texture_copy_range( destination, &dst_texture.desc, @@ -807,6 +821,11 @@ impl Global { let src_buffer = buffer_guard .get(source.buffer) .map_err(|_| TransferError::InvalidBuffer(source.buffer))?; + + if src_buffer.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + tracker .buffers .set_single(src_buffer, hal::BufferUses::COPY_SRC) @@ -938,6 +957,10 @@ impl Global { .get(source.texture) .map_err(|_| TransferError::InvalidTexture(source.texture))?; + if src_texture.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + let (hal_copy_size, array_layer_count) = validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?; @@ -989,6 +1012,11 @@ impl Global { let dst_buffer = buffer_guard .get(destination.buffer) .map_err(|_| TransferError::InvalidBuffer(destination.buffer))?; + + if dst_buffer.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + tracker .buffers .set_single(dst_buffer, hal::BufferUses::COPY_DST) @@ -1117,6 +1145,13 @@ impl Global { .get(destination.texture) .map_err(|_| TransferError::InvalidTexture(source.texture))?; + if src_texture.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + if dst_texture.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice.into()); + } + // src and dst texture format must be copy-compatible // https://gpuweb.github.io/gpuweb/#copy-compatible if src_texture.desc.format.remove_srgb_suffix() |