diff options
Diffstat (limited to 'third_party/rust/wgpu-core/src/command')
-rw-r--r-- | third_party/rust/wgpu-core/src/command/bind.rs | 451 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/bundle.rs | 1722 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/clear.rs | 493 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/compute.rs | 1074 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/draw.rs | 247 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/memory_init.rs | 342 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/mod.rs | 732 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/query.rs | 501 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/render.rs | 2866 | ||||
-rw-r--r-- | third_party/rust/wgpu-core/src/command/transfer.rs | 1229 |
10 files changed, 9657 insertions, 0 deletions
diff --git a/third_party/rust/wgpu-core/src/command/bind.rs b/third_party/rust/wgpu-core/src/command/bind.rs new file mode 100644 index 0000000000..7b2ac54552 --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/bind.rs @@ -0,0 +1,451 @@ +use std::sync::Arc; + +use crate::{ + binding_model::{BindGroup, LateMinBufferBindingSizeMismatch, PipelineLayout}, + device::SHADER_STAGE_COUNT, + hal_api::HalApi, + id::BindGroupId, + pipeline::LateSizedBufferGroup, + resource::Resource, +}; + +use arrayvec::ArrayVec; + +type BindGroupMask = u8; + +mod compat { + use arrayvec::ArrayVec; + + use crate::{binding_model::BindGroupLayout, device::bgl, hal_api::HalApi, resource::Resource}; + use std::{ops::Range, sync::Arc}; + + #[derive(Debug, Clone)] + struct Entry<A: HalApi> { + assigned: Option<Arc<BindGroupLayout<A>>>, + expected: Option<Arc<BindGroupLayout<A>>>, + } + + impl<A: HalApi> Entry<A> { + fn empty() -> Self { + Self { + assigned: None, + expected: None, + } + } + fn is_active(&self) -> bool { + self.assigned.is_some() && self.expected.is_some() + } + + fn is_valid(&self) -> bool { + if self.expected.is_none() { + return true; + } + if let Some(expected_bgl) = self.expected.as_ref() { + if let Some(assigned_bgl) = self.assigned.as_ref() { + if expected_bgl.is_equal(assigned_bgl) { + return true; + } + } + } + false + } + + fn is_incompatible(&self) -> bool { + self.expected.is_none() || !self.is_valid() + } + + // Describe how bind group layouts are incompatible, for validation + // error message. + fn bgl_diff(&self) -> Vec<String> { + let mut diff = Vec::new(); + + if let Some(expected_bgl) = self.expected.as_ref() { + let expected_bgl_type = match expected_bgl.origin { + bgl::Origin::Derived => "implicit", + bgl::Origin::Pool => "explicit", + }; + let expected_label = expected_bgl.label(); + diff.push(format!( + "Should be compatible an with an {expected_bgl_type} bind group layout {}", + if expected_label.is_empty() { + "without label".to_string() + } else { + format!("with label = `{}`", expected_label) + } + )); + if let Some(assigned_bgl) = self.assigned.as_ref() { + let assigned_bgl_type = match assigned_bgl.origin { + bgl::Origin::Derived => "implicit", + bgl::Origin::Pool => "explicit", + }; + let assigned_label = assigned_bgl.label(); + diff.push(format!( + "Assigned {assigned_bgl_type} bind group layout {}", + if assigned_label.is_empty() { + "without label".to_string() + } else { + format!("with label = `{}`", assigned_label) + } + )); + for (id, e_entry) in expected_bgl.entries.iter() { + if let Some(a_entry) = assigned_bgl.entries.get(*id) { + if a_entry.binding != e_entry.binding { + diff.push(format!( + "Entry {id} binding expected {}, got {}", + e_entry.binding, a_entry.binding + )); + } + if a_entry.count != e_entry.count { + diff.push(format!( + "Entry {id} count expected {:?}, got {:?}", + e_entry.count, a_entry.count + )); + } + if a_entry.ty != e_entry.ty { + diff.push(format!( + "Entry {id} type expected {:?}, got {:?}", + e_entry.ty, a_entry.ty + )); + } + if a_entry.visibility != e_entry.visibility { + diff.push(format!( + "Entry {id} visibility expected {:?}, got {:?}", + e_entry.visibility, a_entry.visibility + )); + } + } else { + diff.push(format!( + "Entry {id} not found in assigned bind group layout" + )) + } + } + + assigned_bgl.entries.iter().for_each(|(id, _e_entry)| { + if !expected_bgl.entries.contains_key(*id) { + diff.push(format!( + "Entry {id} not found in expected bind group layout" + )) + } + }); + + if expected_bgl.origin != assigned_bgl.origin { + diff.push(format!("Expected {expected_bgl_type} bind group layout, got {assigned_bgl_type}")) + } + } else { + diff.push("Assigned bind group layout not found (internal error)".to_owned()); + } + } else { + diff.push("Expected bind group layout not found (internal error)".to_owned()); + } + + diff + } + } + + #[derive(Debug, Default)] + pub(crate) struct BoundBindGroupLayouts<A: HalApi> { + entries: ArrayVec<Entry<A>, { hal::MAX_BIND_GROUPS }>, + } + + impl<A: HalApi> BoundBindGroupLayouts<A> { + pub fn new() -> Self { + Self { + entries: (0..hal::MAX_BIND_GROUPS).map(|_| Entry::empty()).collect(), + } + } + fn make_range(&self, start_index: usize) -> Range<usize> { + // find first incompatible entry + let end = self + .entries + .iter() + .position(|e| e.is_incompatible()) + .unwrap_or(self.entries.len()); + start_index..end.max(start_index) + } + + pub fn update_expectations( + &mut self, + expectations: &[Arc<BindGroupLayout<A>>], + ) -> Range<usize> { + let start_index = self + .entries + .iter() + .zip(expectations) + .position(|(e, expect)| { + e.expected.is_none() || !e.expected.as_ref().unwrap().is_equal(expect) + }) + .unwrap_or(expectations.len()); + for (e, expect) in self.entries[start_index..] + .iter_mut() + .zip(expectations[start_index..].iter()) + { + e.expected = Some(expect.clone()); + } + for e in self.entries[expectations.len()..].iter_mut() { + e.expected = None; + } + self.make_range(start_index) + } + + pub fn assign(&mut self, index: usize, value: Arc<BindGroupLayout<A>>) -> Range<usize> { + self.entries[index].assigned = Some(value); + self.make_range(index) + } + + pub fn list_active(&self) -> impl Iterator<Item = usize> + '_ { + self.entries + .iter() + .enumerate() + .filter_map(|(i, e)| if e.is_active() { Some(i) } else { None }) + } + + pub fn invalid_mask(&self) -> super::BindGroupMask { + self.entries.iter().enumerate().fold(0, |mask, (i, entry)| { + if entry.is_valid() { + mask + } else { + mask | 1u8 << i + } + }) + } + + pub fn bgl_diff(&self) -> Vec<String> { + for e in &self.entries { + if !e.is_valid() { + return e.bgl_diff(); + } + } + vec![String::from("No differences detected? (internal error)")] + } + } +} + +#[derive(Debug)] +struct LateBufferBinding { + shader_expect_size: wgt::BufferAddress, + bound_size: wgt::BufferAddress, +} + +#[derive(Debug)] +pub(super) struct EntryPayload<A: HalApi> { + pub(super) group: Option<Arc<BindGroup<A>>>, + pub(super) dynamic_offsets: Vec<wgt::DynamicOffset>, + late_buffer_bindings: Vec<LateBufferBinding>, + /// Since `LateBufferBinding` may contain information about the bindings + /// not used by the pipeline, we need to know when to stop validating. + pub(super) late_bindings_effective_count: usize, +} + +impl<A: HalApi> Default for EntryPayload<A> { + fn default() -> Self { + Self { + group: None, + dynamic_offsets: Default::default(), + late_buffer_bindings: Default::default(), + late_bindings_effective_count: Default::default(), + } + } +} + +impl<A: HalApi> EntryPayload<A> { + fn reset(&mut self) { + self.group = None; + self.dynamic_offsets.clear(); + self.late_buffer_bindings.clear(); + self.late_bindings_effective_count = 0; + } +} + +#[derive(Debug, Default)] +pub(super) struct Binder<A: HalApi> { + pub(super) pipeline_layout: Option<Arc<PipelineLayout<A>>>, + manager: compat::BoundBindGroupLayouts<A>, + payloads: [EntryPayload<A>; hal::MAX_BIND_GROUPS], +} + +impl<A: HalApi> Binder<A> { + pub(super) fn new() -> Self { + Self { + pipeline_layout: None, + manager: compat::BoundBindGroupLayouts::new(), + payloads: Default::default(), + } + } + pub(super) fn reset(&mut self) { + self.pipeline_layout = None; + self.manager = compat::BoundBindGroupLayouts::new(); + for payload in self.payloads.iter_mut() { + payload.reset(); + } + } + + pub(super) fn change_pipeline_layout<'a>( + &'a mut self, + new: &Arc<PipelineLayout<A>>, + late_sized_buffer_groups: &[LateSizedBufferGroup], + ) -> (usize, &'a [EntryPayload<A>]) { + let old_id_opt = self.pipeline_layout.replace(new.clone()); + + let mut bind_range = self.manager.update_expectations(&new.bind_group_layouts); + + // Update the buffer binding sizes that are required by shaders. + for (payload, late_group) in self.payloads.iter_mut().zip(late_sized_buffer_groups) { + payload.late_bindings_effective_count = late_group.shader_sizes.len(); + for (late_binding, &shader_expect_size) in payload + .late_buffer_bindings + .iter_mut() + .zip(late_group.shader_sizes.iter()) + { + late_binding.shader_expect_size = shader_expect_size; + } + if late_group.shader_sizes.len() > payload.late_buffer_bindings.len() { + for &shader_expect_size in + late_group.shader_sizes[payload.late_buffer_bindings.len()..].iter() + { + payload.late_buffer_bindings.push(LateBufferBinding { + shader_expect_size, + bound_size: 0, + }); + } + } + } + + if let Some(old) = old_id_opt { + // root constants are the base compatibility property + if old.push_constant_ranges != new.push_constant_ranges { + bind_range.start = 0; + } + } + + (bind_range.start, &self.payloads[bind_range]) + } + + pub(super) fn assign_group<'a>( + &'a mut self, + index: usize, + bind_group: &Arc<BindGroup<A>>, + offsets: &[wgt::DynamicOffset], + ) -> &'a [EntryPayload<A>] { + let bind_group_id = bind_group.as_info().id(); + log::trace!("\tBinding [{}] = group {:?}", index, bind_group_id); + debug_assert_eq!(A::VARIANT, bind_group_id.backend()); + + let payload = &mut self.payloads[index]; + payload.group = Some(bind_group.clone()); + payload.dynamic_offsets.clear(); + payload.dynamic_offsets.extend_from_slice(offsets); + + // Fill out the actual binding sizes for buffers, + // whose layout doesn't specify `min_binding_size`. + for (late_binding, late_size) in payload + .late_buffer_bindings + .iter_mut() + .zip(bind_group.late_buffer_binding_sizes.iter()) + { + late_binding.bound_size = late_size.get(); + } + if bind_group.late_buffer_binding_sizes.len() > payload.late_buffer_bindings.len() { + for late_size in + bind_group.late_buffer_binding_sizes[payload.late_buffer_bindings.len()..].iter() + { + payload.late_buffer_bindings.push(LateBufferBinding { + shader_expect_size: 0, + bound_size: late_size.get(), + }); + } + } + + let bind_range = self.manager.assign(index, bind_group.layout.clone()); + &self.payloads[bind_range] + } + + pub(super) fn list_active(&self) -> impl Iterator<Item = BindGroupId> + '_ { + let payloads = &self.payloads; + self.manager + .list_active() + .map(move |index| payloads[index].group.as_ref().unwrap().as_info().id()) + } + + pub(super) fn invalid_mask(&self) -> BindGroupMask { + self.manager.invalid_mask() + } + + pub(super) fn bgl_diff(&self) -> Vec<String> { + self.manager.bgl_diff() + } + + /// Scan active buffer bindings corresponding to layouts without `min_binding_size` specified. + pub(super) fn check_late_buffer_bindings( + &self, + ) -> Result<(), LateMinBufferBindingSizeMismatch> { + for group_index in self.manager.list_active() { + let payload = &self.payloads[group_index]; + for (compact_index, late_binding) in payload.late_buffer_bindings + [..payload.late_bindings_effective_count] + .iter() + .enumerate() + { + if late_binding.bound_size < late_binding.shader_expect_size { + return Err(LateMinBufferBindingSizeMismatch { + group_index: group_index as u32, + compact_index, + shader_size: late_binding.shader_expect_size, + bound_size: late_binding.bound_size, + }); + } + } + } + Ok(()) + } +} + +struct PushConstantChange { + stages: wgt::ShaderStages, + offset: u32, + enable: bool, +} + +/// Break up possibly overlapping push constant ranges into a set of +/// non-overlapping ranges which contain all the stage flags of the +/// original ranges. This allows us to zero out (or write any value) +/// to every possible value. +pub fn compute_nonoverlapping_ranges( + ranges: &[wgt::PushConstantRange], +) -> ArrayVec<wgt::PushConstantRange, { SHADER_STAGE_COUNT * 2 }> { + if ranges.is_empty() { + return ArrayVec::new(); + } + debug_assert!(ranges.len() <= SHADER_STAGE_COUNT); + + let mut breaks: ArrayVec<PushConstantChange, { SHADER_STAGE_COUNT * 2 }> = ArrayVec::new(); + for range in ranges { + breaks.push(PushConstantChange { + stages: range.stages, + offset: range.range.start, + enable: true, + }); + breaks.push(PushConstantChange { + stages: range.stages, + offset: range.range.end, + enable: false, + }); + } + breaks.sort_unstable_by_key(|change| change.offset); + + let mut output_ranges = ArrayVec::new(); + let mut position = 0_u32; + let mut stages = wgt::ShaderStages::NONE; + + for bk in breaks { + if bk.offset - position > 0 && !stages.is_empty() { + output_ranges.push(wgt::PushConstantRange { + stages, + range: position..bk.offset, + }) + } + position = bk.offset; + stages.set(bk.stages, bk.enable); + } + + output_ranges +} diff --git a/third_party/rust/wgpu-core/src/command/bundle.rs b/third_party/rust/wgpu-core/src/command/bundle.rs new file mode 100644 index 0000000000..9d80c62f85 --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/bundle.rs @@ -0,0 +1,1722 @@ +/*! Render Bundles + +A render bundle is a prerecorded sequence of commands that can be replayed on a +command encoder with a single call. A single bundle can replayed any number of +times, on different encoders. Constructing a render bundle lets `wgpu` validate +and analyze its commands up front, so that replaying a bundle can be more +efficient than simply re-recording its commands each time. + +Not all commands are available in bundles; for example, a render bundle may not +contain a [`RenderCommand::SetViewport`] command. + +Most of `wgpu`'s backend graphics APIs have something like bundles. For example, +Vulkan calls them "secondary command buffers", and Metal calls them "indirect +command buffers". Although we plan to take advantage of these platform features +at some point in the future, for now `wgpu`'s implementation of render bundles +does not use them: at the hal level, `wgpu` render bundles just replay the +commands. + +## Render Bundle Isolation + +One important property of render bundles is that the draw calls in a render +bundle depend solely on the pipeline and state established within the render +bundle itself. A draw call in a bundle will never use a vertex buffer, say, that +was set in the `RenderPass` before executing the bundle. We call this property +'isolation', in that a render bundle is somewhat isolated from the passes that +use it. + +Render passes are also isolated from the effects of bundles. After executing a +render bundle, a render pass's pipeline, bind groups, and vertex and index +buffers are are unset, so the bundle cannot affect later draw calls in the pass. + +A render pass is not fully isolated from a bundle's effects on push constant +values. Draw calls following a bundle's execution will see whatever values the +bundle writes to push constant storage. Setting a pipeline initializes any push +constant storage it could access to zero, and this initialization may also be +visible after bundle execution. + +## Render Bundle Lifecycle + +To create a render bundle: + +1) Create a [`RenderBundleEncoder`] by calling + [`Global::device_create_render_bundle_encoder`][Gdcrbe]. + +2) Record commands in the `RenderBundleEncoder` using functions from the + [`bundle_ffi`] module. + +3) Call [`Global::render_bundle_encoder_finish`][Grbef], which analyzes and cleans up + the command stream and returns a `RenderBundleId`. + +4) Then, any number of times, call [`wgpu_render_pass_execute_bundles`][wrpeb] to + execute the bundle as part of some render pass. + +## Implementation + +The most complex part of render bundles is the "finish" step, mostly implemented +in [`RenderBundleEncoder::finish`]. This consumes the commands stored in the +encoder's [`BasePass`], while validating everything, tracking the state, +dropping redundant or unnecessary commands, and presenting the results as a new +[`RenderBundle`]. It doesn't actually execute any commands. + +This step also enforces the 'isolation' property mentioned above: every draw +call is checked to ensure that the resources it uses on were established since +the last time the pipeline was set. This means the bundle can be executed +verbatim without any state tracking. + +### Execution + +When the bundle is used in an actual render pass, `RenderBundle::execute` is +called. It goes through the commands and issues them into the native command +buffer. Thanks to isolation, it doesn't track any bind group invalidations or +index format changes. + +[Gdcrbe]: crate::global::Global::device_create_render_bundle_encoder +[Grbef]: crate::global::Global::render_bundle_encoder_finish +[wrpeb]: crate::command::render_ffi::wgpu_render_pass_execute_bundles +!*/ + +#![allow(clippy::reversed_empty_ranges)] + +#[cfg(feature = "trace")] +use crate::device::trace; +use crate::{ + binding_model::{buffer_binding_type_alignment, BindGroup, BindGroupLayout, PipelineLayout}, + command::{ + BasePass, BindGroupStateChange, ColorAttachmentError, DrawError, MapPassErr, + PassErrorScope, RenderCommand, RenderCommandError, StateChange, + }, + conv, + device::{ + AttachmentData, Device, DeviceError, MissingDownlevelFlags, + RenderPassCompatibilityCheckType, RenderPassContext, SHADER_STAGE_COUNT, + }, + error::{ErrorFormatter, PrettyError}, + hal_api::HalApi, + hub::Hub, + id, + init_tracker::{BufferInitTrackerAction, MemoryInitKind, TextureInitTrackerAction}, + pipeline::{PipelineFlags, RenderPipeline, VertexStep}, + resource::{Resource, ResourceInfo, ResourceType}, + resource_log, + track::RenderBundleScope, + validation::check_buffer_usage, + Label, LabelHelpers, +}; +use arrayvec::ArrayVec; + +use std::{borrow::Cow, mem, num::NonZeroU32, ops::Range, sync::Arc}; +use thiserror::Error; + +use hal::CommandEncoder as _; + +/// https://gpuweb.github.io/gpuweb/#dom-gpurendercommandsmixin-draw +fn validate_draw( + vertex: &[Option<VertexState>], + step: &[VertexStep], + first_vertex: u32, + vertex_count: u32, + first_instance: u32, + instance_count: u32, +) -> Result<(), DrawError> { + let vertices_end = first_vertex as u64 + vertex_count as u64; + let instances_end = first_instance as u64 + instance_count as u64; + + for (idx, (vbs, step)) in vertex.iter().zip(step).enumerate() { + let Some(vbs) = vbs else { + continue; + }; + + let stride_count = match step.mode { + wgt::VertexStepMode::Vertex => vertices_end, + wgt::VertexStepMode::Instance => instances_end, + }; + + if stride_count == 0 { + continue; + } + + let offset = (stride_count - 1) * step.stride + step.last_stride; + let limit = vbs.range.end - vbs.range.start; + if offset > limit { + return Err(DrawError::VertexOutOfBounds { + step_mode: step.mode, + offset, + limit, + slot: idx as u32, + }); + } + } + + Ok(()) +} + +// See https://gpuweb.github.io/gpuweb/#dom-gpurendercommandsmixin-drawindexed +fn validate_indexed_draw( + vertex: &[Option<VertexState>], + step: &[VertexStep], + index_state: &IndexState, + first_index: u32, + index_count: u32, + first_instance: u32, + instance_count: u32, +) -> Result<(), DrawError> { + let last_index = first_index as u64 + index_count as u64; + let index_limit = index_state.limit(); + if last_index <= index_limit { + return Err(DrawError::IndexBeyondLimit { + last_index, + index_limit, + }); + } + + let stride_count = first_instance as u64 + instance_count as u64; + for (idx, (vbs, step)) in vertex.iter().zip(step).enumerate() { + let Some(vbs) = vbs else { + continue; + }; + + if stride_count == 0 || step.mode != wgt::VertexStepMode::Instance { + continue; + } + + let offset = (stride_count - 1) * step.stride + step.last_stride; + let limit = vbs.range.end - vbs.range.start; + if offset > limit { + return Err(DrawError::VertexOutOfBounds { + step_mode: step.mode, + offset, + limit, + slot: idx as u32, + }); + } + } + + Ok(()) +} + +/// Describes a [`RenderBundleEncoder`]. +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct RenderBundleEncoderDescriptor<'a> { + /// Debug label of the render bundle encoder. + /// + /// This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// The formats of the color attachments that this render bundle is capable + /// to rendering to. + /// + /// This must match the formats of the color attachments in the + /// renderpass this render bundle is executed in. + pub color_formats: Cow<'a, [Option<wgt::TextureFormat>]>, + /// Information about the depth attachment that this render bundle is + /// capable to rendering to. + /// + /// The format must match the format of the depth attachments in the + /// renderpass this render bundle is executed in. + pub depth_stencil: Option<wgt::RenderBundleDepthStencil>, + /// Sample count this render bundle is capable of rendering to. + /// + /// This must match the pipelines and the renderpasses it is used in. + pub sample_count: u32, + /// If this render bundle will rendering to multiple array layers in the + /// attachments at the same time. + pub multiview: Option<NonZeroU32>, +} + +#[derive(Debug)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct RenderBundleEncoder { + base: BasePass<RenderCommand>, + parent_id: id::DeviceId, + pub(crate) context: RenderPassContext, + pub(crate) is_depth_read_only: bool, + pub(crate) is_stencil_read_only: bool, + + // Resource binding dedupe state. + #[cfg_attr(feature = "serde", serde(skip))] + current_bind_groups: BindGroupStateChange, + #[cfg_attr(feature = "serde", serde(skip))] + current_pipeline: StateChange<id::RenderPipelineId>, +} + +impl RenderBundleEncoder { + pub fn new( + desc: &RenderBundleEncoderDescriptor, + parent_id: id::DeviceId, + base: Option<BasePass<RenderCommand>>, + ) -> Result<Self, CreateRenderBundleError> { + let (is_depth_read_only, is_stencil_read_only) = match desc.depth_stencil { + Some(ds) => { + let aspects = hal::FormatAspects::from(ds.format); + ( + !aspects.contains(hal::FormatAspects::DEPTH) || ds.depth_read_only, + !aspects.contains(hal::FormatAspects::STENCIL) || ds.stencil_read_only, + ) + } + // There's no depth/stencil attachment, so these values just don't + // matter. Choose the most accommodating value, to simplify + // validation. + None => (true, true), + }; + + //TODO: validate that attachment formats are renderable, + // have expected aspects, support multisampling. + Ok(Self { + base: base.unwrap_or_else(|| BasePass::new(&desc.label)), + parent_id, + context: RenderPassContext { + attachments: AttachmentData { + colors: if desc.color_formats.len() > hal::MAX_COLOR_ATTACHMENTS { + return Err(CreateRenderBundleError::ColorAttachment( + ColorAttachmentError::TooMany { + given: desc.color_formats.len(), + limit: hal::MAX_COLOR_ATTACHMENTS, + }, + )); + } else { + desc.color_formats.iter().cloned().collect() + }, + resolves: ArrayVec::new(), + depth_stencil: desc.depth_stencil.map(|ds| ds.format), + }, + sample_count: { + let sc = desc.sample_count; + if sc == 0 || sc > 32 || !conv::is_power_of_two_u32(sc) { + return Err(CreateRenderBundleError::InvalidSampleCount(sc)); + } + sc + }, + multiview: desc.multiview, + }, + + is_depth_read_only, + is_stencil_read_only, + current_bind_groups: BindGroupStateChange::new(), + current_pipeline: StateChange::new(), + }) + } + + pub fn dummy(parent_id: id::DeviceId) -> Self { + Self { + base: BasePass::new(&None), + parent_id, + context: RenderPassContext { + attachments: AttachmentData { + colors: ArrayVec::new(), + resolves: ArrayVec::new(), + depth_stencil: None, + }, + sample_count: 0, + multiview: None, + }, + is_depth_read_only: false, + is_stencil_read_only: false, + + current_bind_groups: BindGroupStateChange::new(), + current_pipeline: StateChange::new(), + } + } + + #[cfg(feature = "trace")] + pub(crate) fn to_base_pass(&self) -> BasePass<RenderCommand> { + BasePass::from_ref(self.base.as_ref()) + } + + pub fn parent(&self) -> id::DeviceId { + self.parent_id + } + + /// Convert this encoder's commands into a [`RenderBundle`]. + /// + /// We want executing a [`RenderBundle`] to be quick, so we take + /// this opportunity to clean up the [`RenderBundleEncoder`]'s + /// command stream and gather metadata about it that will help + /// keep [`ExecuteBundle`] simple and fast. We remove redundant + /// commands (along with their side data), note resource usage, + /// and accumulate buffer and texture initialization actions. + /// + /// [`ExecuteBundle`]: RenderCommand::ExecuteBundle + pub(crate) fn finish<A: HalApi>( + self, + desc: &RenderBundleDescriptor, + device: &Arc<Device<A>>, + hub: &Hub<A>, + ) -> Result<RenderBundle<A>, RenderBundleError> { + let bind_group_guard = hub.bind_groups.read(); + let pipeline_guard = hub.render_pipelines.read(); + let query_set_guard = hub.query_sets.read(); + let buffer_guard = hub.buffers.read(); + let texture_guard = hub.textures.read(); + + let mut state = State { + trackers: RenderBundleScope::new( + &*buffer_guard, + &*texture_guard, + &*bind_group_guard, + &*pipeline_guard, + &*query_set_guard, + ), + pipeline: None, + bind: (0..hal::MAX_BIND_GROUPS).map(|_| None).collect(), + vertex: (0..hal::MAX_VERTEX_BUFFERS).map(|_| None).collect(), + index: None, + flat_dynamic_offsets: Vec::new(), + }; + let mut commands = Vec::new(); + let mut buffer_memory_init_actions = Vec::new(); + let mut texture_memory_init_actions = Vec::new(); + + let base = self.base.as_ref(); + let mut next_dynamic_offset = 0; + + for &command in base.commands { + match command { + RenderCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group_id, + } => { + let scope = PassErrorScope::SetBindGroup(bind_group_id); + + let bind_group = state + .trackers + .bind_groups + .write() + .add_single(&*bind_group_guard, bind_group_id) + .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id)) + .map_pass_err(scope)?; + self.check_valid_to_use(bind_group.device.info.id()) + .map_pass_err(scope)?; + + let max_bind_groups = device.limits.max_bind_groups; + if index >= max_bind_groups { + return Err(RenderCommandError::BindGroupIndexOutOfRange { + index, + max: max_bind_groups, + }) + .map_pass_err(scope); + } + + // Identify the next `num_dynamic_offsets` entries from `base.dynamic_offsets`. + let num_dynamic_offsets = num_dynamic_offsets; + let offsets_range = + next_dynamic_offset..next_dynamic_offset + num_dynamic_offsets; + next_dynamic_offset = offsets_range.end; + let offsets = &base.dynamic_offsets[offsets_range.clone()]; + + if bind_group.dynamic_binding_info.len() != offsets.len() { + return Err(RenderCommandError::InvalidDynamicOffsetCount { + actual: offsets.len(), + expected: bind_group.dynamic_binding_info.len(), + }) + .map_pass_err(scope); + } + + // Check for misaligned offsets. + for (offset, info) in offsets + .iter() + .map(|offset| *offset as wgt::BufferAddress) + .zip(bind_group.dynamic_binding_info.iter()) + { + let (alignment, limit_name) = + buffer_binding_type_alignment(&device.limits, info.binding_type); + if offset % alignment as u64 != 0 { + return Err(RenderCommandError::UnalignedBufferOffset( + offset, limit_name, alignment, + )) + .map_pass_err(scope); + } + } + + buffer_memory_init_actions.extend_from_slice(&bind_group.used_buffer_ranges); + texture_memory_init_actions.extend_from_slice(&bind_group.used_texture_ranges); + + state.set_bind_group(index, bind_group_guard.get(bind_group_id).as_ref().unwrap(), &bind_group.layout, offsets_range); + unsafe { + state + .trackers + .merge_bind_group(&bind_group.used) + .map_pass_err(scope)? + }; + //Note: stateless trackers are not merged: the lifetime reference + // is held to the bind group itself. + } + RenderCommand::SetPipeline(pipeline_id) => { + let scope = PassErrorScope::SetPipelineRender(pipeline_id); + + let pipeline = state + .trackers + .render_pipelines + .write() + .add_single(&*pipeline_guard, pipeline_id) + .ok_or(RenderCommandError::InvalidPipeline(pipeline_id)) + .map_pass_err(scope)?; + self.check_valid_to_use(pipeline.device.info.id()) + .map_pass_err(scope)?; + + self.context + .check_compatible(&pipeline.pass_context, RenderPassCompatibilityCheckType::RenderPipeline) + .map_err(RenderCommandError::IncompatiblePipelineTargets) + .map_pass_err(scope)?; + + if (pipeline.flags.contains(PipelineFlags::WRITES_DEPTH) + && self.is_depth_read_only) + || (pipeline.flags.contains(PipelineFlags::WRITES_STENCIL) + && self.is_stencil_read_only) + { + return Err(RenderCommandError::IncompatiblePipelineRods) + .map_pass_err(scope); + } + + let pipeline_state = PipelineState::new(pipeline); + + commands.push(command); + + // If this pipeline uses push constants, zero out their values. + if let Some(iter) = pipeline_state.zero_push_constants() { + commands.extend(iter) + } + + state.invalidate_bind_groups(&pipeline_state, &pipeline.layout); + state.pipeline = Some(pipeline_state); + } + RenderCommand::SetIndexBuffer { + buffer_id, + index_format, + offset, + size, + } => { + let scope = PassErrorScope::SetIndexBuffer(buffer_id); + let buffer = state + .trackers + .buffers + .write() + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) + .map_pass_err(scope)?; + self.check_valid_to_use(buffer.device.info.id()) + .map_pass_err(scope)?; + check_buffer_usage(buffer.usage, wgt::BufferUsages::INDEX) + .map_pass_err(scope)?; + + let end = match size { + Some(s) => offset + s.get(), + None => buffer.size, + }; + buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action( + buffer, + offset..end, + MemoryInitKind::NeedsInitializedMemory, + )); + state.set_index_buffer(buffer_id, index_format, offset..end); + } + RenderCommand::SetVertexBuffer { + slot, + buffer_id, + offset, + size, + } => { + let scope = PassErrorScope::SetVertexBuffer(buffer_id); + + let max_vertex_buffers = device.limits.max_vertex_buffers; + if slot >= max_vertex_buffers { + return Err(RenderCommandError::VertexBufferIndexOutOfRange { + index: slot, + max: max_vertex_buffers, + }) + .map_pass_err(scope); + } + + let buffer = state + .trackers + .buffers + .write() + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) + .map_pass_err(scope)?; + self.check_valid_to_use(buffer.device.info.id()) + .map_pass_err(scope)?; + check_buffer_usage(buffer.usage, wgt::BufferUsages::VERTEX) + .map_pass_err(scope)?; + + let end = match size { + Some(s) => offset + s.get(), + None => buffer.size, + }; + buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action( + buffer, + offset..end, + MemoryInitKind::NeedsInitializedMemory, + )); + state.vertex[slot as usize] = Some(VertexState::new(buffer_id, offset..end)); + } + RenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset: _, + } => { + let scope = PassErrorScope::SetPushConstant; + let end_offset = offset + size_bytes; + + let pipeline_state = state.pipeline(scope)?; + + pipeline_state.pipeline.layout + .validate_push_constant_ranges(stages, offset, end_offset) + .map_pass_err(scope)?; + + commands.push(command); + } + RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + } => { + let scope = PassErrorScope::Draw { + indexed: false, + indirect: false, + pipeline: state.pipeline_id(), + }; + let pipeline = state.pipeline(scope)?; + let used_bind_groups = pipeline.used_bind_groups; + + validate_draw( + &state.vertex[..], + &pipeline.steps, + first_vertex, + vertex_count, + first_instance, + instance_count, + ).map_pass_err(scope)?; + + if instance_count > 0 && vertex_count > 0 { + commands.extend(state.flush_vertices()); + commands.extend(state.flush_binds(used_bind_groups, base.dynamic_offsets)); + commands.push(command); + } + } + RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex: _, + first_instance, + } => { + let scope = PassErrorScope::Draw { + indexed: true, + indirect: false, + pipeline: state.pipeline_id(), + }; + let pipeline = state.pipeline(scope)?; + let used_bind_groups = pipeline.used_bind_groups; + let index = match state.index { + Some(ref index) => index, + None => return Err(DrawError::MissingIndexBuffer).map_pass_err(scope), + }; + + validate_indexed_draw( + &state.vertex[..], + &pipeline.steps, + index, + first_index, + index_count, + first_instance, + instance_count, + ).map_pass_err(scope)?; + + if instance_count > 0 && index_count > 0 { + commands.extend(state.flush_index()); + commands.extend(state.flush_vertices()); + commands.extend(state.flush_binds(used_bind_groups, base.dynamic_offsets)); + commands.push(command); + } + } + RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: false, + } => { + let scope = PassErrorScope::Draw { + indexed: false, + indirect: true, + pipeline: state.pipeline_id(), + }; + device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) + .map_pass_err(scope)?; + + let pipeline = state.pipeline(scope)?; + let used_bind_groups = pipeline.used_bind_groups; + + let buffer = state + .trackers + .buffers + .write() + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; + self.check_valid_to_use(buffer.device.info.id()) + .map_pass_err(scope)?; + check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) + .map_pass_err(scope)?; + + buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action( + buffer, + offset..(offset + mem::size_of::<wgt::DrawIndirectArgs>() as u64), + MemoryInitKind::NeedsInitializedMemory, + )); + + commands.extend(state.flush_vertices()); + commands.extend(state.flush_binds(used_bind_groups, base.dynamic_offsets)); + commands.push(command); + } + RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: true, + } => { + let scope = PassErrorScope::Draw { + indexed: true, + indirect: true, + pipeline: state.pipeline_id(), + }; + device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) + .map_pass_err(scope)?; + + let pipeline = state.pipeline(scope)?; + let used_bind_groups = pipeline.used_bind_groups; + + let buffer = state + .trackers + .buffers + .write() + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; + self.check_valid_to_use(buffer.device.info.id()) + .map_pass_err(scope)?; + check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) + .map_pass_err(scope)?; + + buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action( + buffer, + offset..(offset + mem::size_of::<wgt::DrawIndirectArgs>() as u64), + MemoryInitKind::NeedsInitializedMemory, + )); + + let index = match state.index { + Some(ref mut index) => index, + None => return Err(DrawError::MissingIndexBuffer).map_pass_err(scope), + }; + + commands.extend(index.flush()); + commands.extend(state.flush_vertices()); + commands.extend(state.flush_binds(used_bind_groups, base.dynamic_offsets)); + commands.push(command); + } + RenderCommand::MultiDrawIndirect { .. } + | RenderCommand::MultiDrawIndirectCount { .. } => unimplemented!(), + RenderCommand::PushDebugGroup { color: _, len: _ } => unimplemented!(), + RenderCommand::InsertDebugMarker { color: _, len: _ } => unimplemented!(), + RenderCommand::PopDebugGroup => unimplemented!(), + RenderCommand::WriteTimestamp { .. } // Must check the TIMESTAMP_QUERY_INSIDE_PASSES feature + | RenderCommand::BeginOcclusionQuery { .. } + | RenderCommand::EndOcclusionQuery + | RenderCommand::BeginPipelineStatisticsQuery { .. } + | RenderCommand::EndPipelineStatisticsQuery => unimplemented!(), + RenderCommand::ExecuteBundle(_) + | RenderCommand::SetBlendConstant(_) + | RenderCommand::SetStencilReference(_) + | RenderCommand::SetViewport { .. } + | RenderCommand::SetScissor(_) => unreachable!("not supported by a render bundle"), + } + } + + Ok(RenderBundle { + base: BasePass { + label: desc.label.as_ref().map(|cow| cow.to_string()), + commands, + dynamic_offsets: state.flat_dynamic_offsets, + string_data: Vec::new(), + push_constant_data: Vec::new(), + }, + is_depth_read_only: self.is_depth_read_only, + is_stencil_read_only: self.is_stencil_read_only, + device: device.clone(), + used: state.trackers, + buffer_memory_init_actions, + texture_memory_init_actions, + context: self.context, + info: ResourceInfo::new(desc.label.borrow_or_default()), + discard_hal_labels: device + .instance_flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS), + }) + } + + fn check_valid_to_use(&self, device_id: id::DeviceId) -> Result<(), RenderBundleErrorInner> { + if device_id != self.parent_id { + return Err(RenderBundleErrorInner::NotValidToUse); + } + + Ok(()) + } + + pub fn set_index_buffer( + &mut self, + buffer_id: id::BufferId, + index_format: wgt::IndexFormat, + offset: wgt::BufferAddress, + size: Option<wgt::BufferSize>, + ) { + self.base.commands.push(RenderCommand::SetIndexBuffer { + buffer_id, + index_format, + offset, + size, + }); + } +} + +/// Error type returned from `RenderBundleEncoder::new` if the sample count is invalid. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateRenderBundleError { + #[error(transparent)] + ColorAttachment(#[from] ColorAttachmentError), + #[error("Invalid number of samples {0}")] + InvalidSampleCount(u32), +} + +/// Error type returned from `RenderBundleEncoder::new` if the sample count is invalid. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum ExecutionError { + #[error("Buffer {0:?} is destroyed")] + DestroyedBuffer(id::BufferId), + #[error("BindGroup {0:?} is invalid")] + InvalidBindGroup(id::BindGroupId), + #[error("Using {0} in a render bundle is not implemented")] + Unimplemented(&'static str), +} +impl PrettyError for ExecutionError { + fn fmt_pretty(&self, fmt: &mut ErrorFormatter) { + fmt.error(self); + match *self { + Self::DestroyedBuffer(id) => { + fmt.buffer_label(&id); + } + Self::InvalidBindGroup(id) => { + fmt.bind_group_label(&id); + } + Self::Unimplemented(_reason) => {} + }; + } +} + +pub type RenderBundleDescriptor<'a> = wgt::RenderBundleDescriptor<Label<'a>>; + +//Note: here, `RenderBundle` is just wrapping a raw stream of render commands. +// The plan is to back it by an actual Vulkan secondary buffer, D3D12 Bundle, +// or Metal indirect command buffer. +#[derive(Debug)] +pub struct RenderBundle<A: HalApi> { + // Normalized command stream. It can be executed verbatim, + // without re-binding anything on the pipeline change. + base: BasePass<RenderCommand>, + pub(super) is_depth_read_only: bool, + pub(super) is_stencil_read_only: bool, + pub(crate) device: Arc<Device<A>>, + pub(crate) used: RenderBundleScope<A>, + pub(super) buffer_memory_init_actions: Vec<BufferInitTrackerAction<A>>, + pub(super) texture_memory_init_actions: Vec<TextureInitTrackerAction<A>>, + pub(super) context: RenderPassContext, + pub(crate) info: ResourceInfo<RenderBundle<A>>, + discard_hal_labels: bool, +} + +impl<A: HalApi> Drop for RenderBundle<A> { + fn drop(&mut self) { + resource_log!("Destroy raw RenderBundle {:?}", self.info.label()); + + #[cfg(feature = "trace")] + if let Some(t) = self.device.trace.lock().as_mut() { + t.add(trace::Action::DestroyRenderBundle(self.info.id())); + } + } +} + +#[cfg(send_sync)] +unsafe impl<A: HalApi> Send for RenderBundle<A> {} +#[cfg(send_sync)] +unsafe impl<A: HalApi> Sync for RenderBundle<A> {} + +impl<A: HalApi> RenderBundle<A> { + /// Actually encode the contents into a native command buffer. + /// + /// This is partially duplicating the logic of `command_encoder_run_render_pass`. + /// However the point of this function is to be lighter, since we already had + /// a chance to go through the commands in `render_bundle_encoder_finish`. + /// + /// Note that the function isn't expected to fail, generally. + /// All the validation has already been done by this point. + /// The only failure condition is if some of the used buffers are destroyed. + pub(super) unsafe fn execute(&self, raw: &mut A::CommandEncoder) -> Result<(), ExecutionError> { + let trackers = &self.used; + let mut offsets = self.base.dynamic_offsets.as_slice(); + let mut pipeline_layout = None::<Arc<PipelineLayout<A>>>; + if !self.discard_hal_labels { + if let Some(ref label) = self.base.label { + unsafe { raw.begin_debug_marker(label) }; + } + } + + let snatch_guard = self.device.snatchable_lock.read(); + + for command in self.base.commands.iter() { + match *command { + RenderCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group_id, + } => { + let bind_groups = trackers.bind_groups.read(); + let bind_group = bind_groups.get(bind_group_id).unwrap(); + let raw_bg = bind_group + .raw(&snatch_guard) + .ok_or(ExecutionError::InvalidBindGroup(bind_group_id))?; + unsafe { + raw.set_bind_group( + pipeline_layout.as_ref().unwrap().raw(), + index, + raw_bg, + &offsets[..num_dynamic_offsets], + ) + }; + offsets = &offsets[num_dynamic_offsets..]; + } + RenderCommand::SetPipeline(pipeline_id) => { + let render_pipelines = trackers.render_pipelines.read(); + let pipeline = render_pipelines.get(pipeline_id).unwrap(); + unsafe { raw.set_render_pipeline(pipeline.raw()) }; + + pipeline_layout = Some(pipeline.layout.clone()); + } + RenderCommand::SetIndexBuffer { + buffer_id, + index_format, + offset, + size, + } => { + let buffers = trackers.buffers.read(); + let buffer: &A::Buffer = buffers + .get(buffer_id) + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))? + .raw(&snatch_guard) + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?; + let bb = hal::BufferBinding { + buffer, + offset, + size, + }; + unsafe { raw.set_index_buffer(bb, index_format) }; + } + RenderCommand::SetVertexBuffer { + slot, + buffer_id, + offset, + size, + } => { + let buffers = trackers.buffers.read(); + let buffer = buffers + .get(buffer_id) + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))? + .raw(&snatch_guard) + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?; + let bb = hal::BufferBinding { + buffer, + offset, + size, + }; + unsafe { raw.set_vertex_buffer(slot, bb) }; + } + RenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset, + } => { + let pipeline_layout = pipeline_layout.as_ref().unwrap(); + + if let Some(values_offset) = values_offset { + let values_end_offset = + (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; + let data_slice = &self.base.push_constant_data + [(values_offset as usize)..values_end_offset]; + + unsafe { + raw.set_push_constants( + pipeline_layout.raw(), + stages, + offset, + data_slice, + ) + } + } else { + super::push_constant_clear( + offset, + size_bytes, + |clear_offset, clear_data| { + unsafe { + raw.set_push_constants( + pipeline_layout.raw(), + stages, + clear_offset, + clear_data, + ) + }; + }, + ); + } + } + RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + } => { + unsafe { raw.draw(first_vertex, vertex_count, first_instance, instance_count) }; + } + RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + } => { + unsafe { + raw.draw_indexed( + first_index, + index_count, + base_vertex, + first_instance, + instance_count, + ) + }; + } + RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: false, + } => { + let buffers = trackers.buffers.read(); + let buffer = buffers + .get(buffer_id) + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))? + .raw(&snatch_guard) + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?; + unsafe { raw.draw_indirect(buffer, offset, 1) }; + } + RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: true, + } => { + let buffers = trackers.buffers.read(); + let buffer = buffers + .get(buffer_id) + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))? + .raw(&snatch_guard) + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?; + unsafe { raw.draw_indexed_indirect(buffer, offset, 1) }; + } + RenderCommand::MultiDrawIndirect { .. } + | RenderCommand::MultiDrawIndirectCount { .. } => { + return Err(ExecutionError::Unimplemented("multi-draw-indirect")) + } + RenderCommand::PushDebugGroup { .. } + | RenderCommand::InsertDebugMarker { .. } + | RenderCommand::PopDebugGroup => { + return Err(ExecutionError::Unimplemented("debug-markers")) + } + RenderCommand::WriteTimestamp { .. } + | RenderCommand::BeginOcclusionQuery { .. } + | RenderCommand::EndOcclusionQuery + | RenderCommand::BeginPipelineStatisticsQuery { .. } + | RenderCommand::EndPipelineStatisticsQuery => { + return Err(ExecutionError::Unimplemented("queries")) + } + RenderCommand::ExecuteBundle(_) + | RenderCommand::SetBlendConstant(_) + | RenderCommand::SetStencilReference(_) + | RenderCommand::SetViewport { .. } + | RenderCommand::SetScissor(_) => unreachable!(), + } + } + + if !self.discard_hal_labels { + if let Some(_) = self.base.label { + unsafe { raw.end_debug_marker() }; + } + } + + Ok(()) + } +} + +impl<A: HalApi> Resource for RenderBundle<A> { + const TYPE: ResourceType = "RenderBundle"; + + type Marker = crate::id::markers::RenderBundle; + + fn as_info(&self) -> &ResourceInfo<Self> { + &self.info + } + + fn as_info_mut(&mut self) -> &mut ResourceInfo<Self> { + &mut self.info + } +} + +/// A render bundle's current index buffer state. +/// +/// [`RenderBundleEncoder::finish`] records the currently set index buffer here, +/// and calls [`State::flush_index`] before any indexed draw command to produce +/// a `SetIndexBuffer` command if one is necessary. +#[derive(Debug)] +struct IndexState { + buffer: id::BufferId, + format: wgt::IndexFormat, + range: Range<wgt::BufferAddress>, + is_dirty: bool, +} + +impl IndexState { + /// Return the number of entries in the current index buffer. + /// + /// Panic if no index buffer has been set. + fn limit(&self) -> u64 { + let bytes_per_index = match self.format { + wgt::IndexFormat::Uint16 => 2, + wgt::IndexFormat::Uint32 => 4, + }; + + (self.range.end - self.range.start) / bytes_per_index + } + + /// Generate a `SetIndexBuffer` command to prepare for an indexed draw + /// command, if needed. + fn flush(&mut self) -> Option<RenderCommand> { + if self.is_dirty { + self.is_dirty = false; + Some(RenderCommand::SetIndexBuffer { + buffer_id: self.buffer, + index_format: self.format, + offset: self.range.start, + size: wgt::BufferSize::new(self.range.end - self.range.start), + }) + } else { + None + } + } +} + +/// The state of a single vertex buffer slot during render bundle encoding. +/// +/// [`RenderBundleEncoder::finish`] uses this to drop redundant +/// `SetVertexBuffer` commands from the final [`RenderBundle`]. It +/// records one vertex buffer slot's state changes here, and then +/// calls this type's [`flush`] method just before any draw command to +/// produce a `SetVertexBuffer` commands if one is necessary. +/// +/// [`flush`]: IndexState::flush +#[derive(Debug)] +struct VertexState { + buffer: id::BufferId, + range: Range<wgt::BufferAddress>, + is_dirty: bool, +} + +impl VertexState { + fn new(buffer: id::BufferId, range: Range<wgt::BufferAddress>) -> Self { + Self { + buffer, + range, + is_dirty: true, + } + } + + /// Generate a `SetVertexBuffer` command for this slot, if necessary. + /// + /// `slot` is the index of the vertex buffer slot that `self` tracks. + fn flush(&mut self, slot: u32) -> Option<RenderCommand> { + if self.is_dirty { + self.is_dirty = false; + Some(RenderCommand::SetVertexBuffer { + slot, + buffer_id: self.buffer, + offset: self.range.start, + size: wgt::BufferSize::new(self.range.end - self.range.start), + }) + } else { + None + } + } +} + +/// A bind group that has been set at a particular index during render bundle encoding. +#[derive(Debug)] +struct BindState<A: HalApi> { + /// The id of the bind group set at this index. + bind_group: Arc<BindGroup<A>>, + + /// The layout of `group`. + layout: Arc<BindGroupLayout<A>>, + + /// The range of dynamic offsets for this bind group, in the original + /// command stream's `BassPass::dynamic_offsets` array. + dynamic_offsets: Range<usize>, + + /// True if this index's contents have been changed since the last time we + /// generated a `SetBindGroup` command. + is_dirty: bool, +} + +/// The bundle's current pipeline, and some cached information needed for validation. +struct PipelineState<A: HalApi> { + /// The pipeline + pipeline: Arc<RenderPipeline<A>>, + + /// How this pipeline's vertex shader traverses each vertex buffer, indexed + /// by vertex buffer slot number. + steps: Vec<VertexStep>, + + /// Ranges of push constants this pipeline uses, copied from the pipeline + /// layout. + push_constant_ranges: ArrayVec<wgt::PushConstantRange, { SHADER_STAGE_COUNT }>, + + /// The number of bind groups this pipeline uses. + used_bind_groups: usize, +} + +impl<A: HalApi> PipelineState<A> { + fn new(pipeline: &Arc<RenderPipeline<A>>) -> Self { + Self { + pipeline: pipeline.clone(), + steps: pipeline.vertex_steps.to_vec(), + push_constant_ranges: pipeline + .layout + .push_constant_ranges + .iter() + .cloned() + .collect(), + used_bind_groups: pipeline.layout.bind_group_layouts.len(), + } + } + + /// Return a sequence of commands to zero the push constant ranges this + /// pipeline uses. If no initialization is necessary, return `None`. + fn zero_push_constants(&self) -> Option<impl Iterator<Item = RenderCommand>> { + if !self.push_constant_ranges.is_empty() { + let nonoverlapping_ranges = + super::bind::compute_nonoverlapping_ranges(&self.push_constant_ranges); + + Some( + nonoverlapping_ranges + .into_iter() + .map(|range| RenderCommand::SetPushConstant { + stages: range.stages, + offset: range.range.start, + size_bytes: range.range.end - range.range.start, + values_offset: None, // write zeros + }), + ) + } else { + None + } + } +} + +/// State for analyzing and cleaning up bundle command streams. +/// +/// To minimize state updates, [`RenderBundleEncoder::finish`] +/// actually just applies commands like [`SetBindGroup`] and +/// [`SetIndexBuffer`] to the simulated state stored here, and then +/// calls the `flush_foo` methods before draw calls to produce the +/// update commands we actually need. +/// +/// [`SetBindGroup`]: RenderCommand::SetBindGroup +/// [`SetIndexBuffer`]: RenderCommand::SetIndexBuffer +struct State<A: HalApi> { + /// Resources used by this bundle. This will become [`RenderBundle::used`]. + trackers: RenderBundleScope<A>, + + /// The currently set pipeline, if any. + pipeline: Option<PipelineState<A>>, + + /// The bind group set at each index, if any. + bind: ArrayVec<Option<BindState<A>>, { hal::MAX_BIND_GROUPS }>, + + /// The state of each vertex buffer slot. + vertex: ArrayVec<Option<VertexState>, { hal::MAX_VERTEX_BUFFERS }>, + + /// The current index buffer, if one has been set. We flush this state + /// before indexed draw commands. + index: Option<IndexState>, + + /// Dynamic offset values used by the cleaned-up command sequence. + /// + /// This becomes the final [`RenderBundle`]'s [`BasePass`]'s + /// [`dynamic_offsets`] list. + /// + /// [`dynamic_offsets`]: BasePass::dynamic_offsets + flat_dynamic_offsets: Vec<wgt::DynamicOffset>, +} + +impl<A: HalApi> State<A> { + /// Return the id of the current pipeline, if any. + fn pipeline_id(&self) -> Option<id::RenderPipelineId> { + self.pipeline.as_ref().map(|p| p.pipeline.as_info().id()) + } + + /// Return the current pipeline state. Return an error if none is set. + fn pipeline(&self, scope: PassErrorScope) -> Result<&PipelineState<A>, RenderBundleError> { + self.pipeline + .as_ref() + .ok_or(DrawError::MissingPipeline) + .map_pass_err(scope) + } + + /// Mark all non-empty bind group table entries from `index` onwards as dirty. + fn invalidate_bind_group_from(&mut self, index: usize) { + for contents in self.bind[index..].iter_mut().flatten() { + contents.is_dirty = true; + } + } + + fn set_bind_group( + &mut self, + slot: u32, + bind_group: &Arc<BindGroup<A>>, + layout: &Arc<BindGroupLayout<A>>, + dynamic_offsets: Range<usize>, + ) { + // If this call wouldn't actually change this index's state, we can + // return early. (If there are dynamic offsets, the range will always + // be different.) + if dynamic_offsets.is_empty() { + if let Some(ref contents) = self.bind[slot as usize] { + if contents.bind_group.is_equal(bind_group) { + return; + } + } + } + + // Record the index's new state. + self.bind[slot as usize] = Some(BindState { + bind_group: bind_group.clone(), + layout: layout.clone(), + dynamic_offsets, + is_dirty: true, + }); + + // Once we've changed the bind group at a particular index, all + // subsequent indices need to be rewritten. + self.invalidate_bind_group_from(slot as usize + 1); + } + + /// Determine which bind group slots need to be re-set after a pipeline change. + /// + /// Given that we are switching from the current pipeline state to `new`, + /// whose layout is `layout`, mark all the bind group slots that we need to + /// emit new `SetBindGroup` commands for as dirty. + /// + /// According to `wgpu_hal`'s rules: + /// + /// - If the layout of any bind group slot changes, then that slot and + /// all following slots must have their bind groups re-established. + /// + /// - Changing the push constant ranges at all requires re-establishing + /// all bind groups. + fn invalidate_bind_groups(&mut self, new: &PipelineState<A>, layout: &PipelineLayout<A>) { + match self.pipeline { + None => { + // Establishing entirely new pipeline state. + self.invalidate_bind_group_from(0); + } + Some(ref old) => { + if old.pipeline.is_equal(&new.pipeline) { + // Everything is derived from the pipeline, so if the id has + // not changed, there's no need to consider anything else. + return; + } + + // Any push constant change invalidates all groups. + if old.push_constant_ranges != new.push_constant_ranges { + self.invalidate_bind_group_from(0); + } else { + let first_changed = self.bind.iter().zip(&layout.bind_group_layouts).position( + |(entry, layout)| match *entry { + Some(ref contents) => !contents.layout.is_equal(layout), + None => false, + }, + ); + if let Some(slot) = first_changed { + self.invalidate_bind_group_from(slot); + } + } + } + } + } + + /// Set the bundle's current index buffer and its associated parameters. + fn set_index_buffer( + &mut self, + buffer: id::BufferId, + format: wgt::IndexFormat, + range: Range<wgt::BufferAddress>, + ) { + match self.index { + Some(ref current) + if current.buffer == buffer + && current.format == format + && current.range == range => + { + return + } + _ => (), + } + + self.index = Some(IndexState { + buffer, + format, + range, + is_dirty: true, + }); + } + + /// Generate a `SetIndexBuffer` command to prepare for an indexed draw + /// command, if needed. + fn flush_index(&mut self) -> Option<RenderCommand> { + self.index.as_mut().and_then(|index| index.flush()) + } + + fn flush_vertices(&mut self) -> impl Iterator<Item = RenderCommand> + '_ { + self.vertex + .iter_mut() + .enumerate() + .flat_map(|(i, vs)| vs.as_mut().and_then(|vs| vs.flush(i as u32))) + } + + /// Generate `SetBindGroup` commands for any bind groups that need to be updated. + fn flush_binds( + &mut self, + used_bind_groups: usize, + dynamic_offsets: &[wgt::DynamicOffset], + ) -> impl Iterator<Item = RenderCommand> + '_ { + // Append each dirty bind group's dynamic offsets to `flat_dynamic_offsets`. + for contents in self.bind[..used_bind_groups].iter().flatten() { + if contents.is_dirty { + self.flat_dynamic_offsets + .extend_from_slice(&dynamic_offsets[contents.dynamic_offsets.clone()]); + } + } + + // Then, generate `SetBindGroup` commands to update the dirty bind + // groups. After this, all bind groups are clean. + self.bind[..used_bind_groups] + .iter_mut() + .enumerate() + .flat_map(|(i, entry)| { + if let Some(ref mut contents) = *entry { + if contents.is_dirty { + contents.is_dirty = false; + let offsets = &contents.dynamic_offsets; + return Some(RenderCommand::SetBindGroup { + index: i.try_into().unwrap(), + bind_group_id: contents.bind_group.as_info().id(), + num_dynamic_offsets: offsets.end - offsets.start, + }); + } + } + None + }) + } +} + +/// Error encountered when finishing recording a render bundle. +#[derive(Clone, Debug, Error)] +pub(super) enum RenderBundleErrorInner { + #[error("Resource is not valid to use with this render bundle because the resource and the bundle come from different devices")] + NotValidToUse, + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + RenderCommand(RenderCommandError), + #[error(transparent)] + Draw(#[from] DrawError), + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), +} + +impl<T> From<T> for RenderBundleErrorInner +where + T: Into<RenderCommandError>, +{ + fn from(t: T) -> Self { + Self::RenderCommand(t.into()) + } +} + +/// Error encountered when finishing recording a render bundle. +#[derive(Clone, Debug, Error)] +#[error("{scope}")] +pub struct RenderBundleError { + pub scope: PassErrorScope, + #[source] + inner: RenderBundleErrorInner, +} + +impl RenderBundleError { + pub(crate) const INVALID_DEVICE: Self = RenderBundleError { + scope: PassErrorScope::Bundle, + inner: RenderBundleErrorInner::Device(DeviceError::Invalid), + }; +} +impl PrettyError for RenderBundleError { + fn fmt_pretty(&self, fmt: &mut ErrorFormatter) { + // This error is wrapper for the inner error, + // but the scope has useful labels + fmt.error(self); + self.scope.fmt_pretty(fmt); + } +} + +impl<T, E> MapPassErr<T, RenderBundleError> for Result<T, E> +where + E: Into<RenderBundleErrorInner>, +{ + fn map_pass_err(self, scope: PassErrorScope) -> Result<T, RenderBundleError> { + self.map_err(|inner| RenderBundleError { + scope, + inner: inner.into(), + }) + } +} + +pub mod bundle_ffi { + use super::{RenderBundleEncoder, RenderCommand}; + use crate::{id, RawString}; + use std::{convert::TryInto, slice}; + use wgt::{BufferAddress, BufferSize, DynamicOffset, IndexFormat}; + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `offset_length` elements. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_bundle_set_bind_group( + bundle: &mut RenderBundleEncoder, + index: u32, + bind_group_id: id::BindGroupId, + offsets: *const DynamicOffset, + offset_length: usize, + ) { + let redundant = unsafe { + bundle.current_bind_groups.set_and_check_redundant( + bind_group_id, + index, + &mut bundle.base.dynamic_offsets, + offsets, + offset_length, + ) + }; + + if redundant { + return; + } + + bundle.base.commands.push(RenderCommand::SetBindGroup { + index, + num_dynamic_offsets: offset_length, + bind_group_id, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_set_pipeline( + bundle: &mut RenderBundleEncoder, + pipeline_id: id::RenderPipelineId, + ) { + if bundle.current_pipeline.set_and_check_redundant(pipeline_id) { + return; + } + + bundle + .base + .commands + .push(RenderCommand::SetPipeline(pipeline_id)); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_set_vertex_buffer( + bundle: &mut RenderBundleEncoder, + slot: u32, + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option<BufferSize>, + ) { + bundle.base.commands.push(RenderCommand::SetVertexBuffer { + slot, + buffer_id, + offset, + size, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_set_index_buffer( + encoder: &mut RenderBundleEncoder, + buffer: id::BufferId, + index_format: IndexFormat, + offset: BufferAddress, + size: Option<BufferSize>, + ) { + encoder.set_index_buffer(buffer, index_format, offset, size); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `data` elements. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_bundle_set_push_constants( + pass: &mut RenderBundleEncoder, + stages: wgt::ShaderStages, + offset: u32, + size_bytes: u32, + data: *const u8, + ) { + assert_eq!( + offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant offset must be aligned to 4 bytes." + ); + assert_eq!( + size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant size must be aligned to 4 bytes." + ); + let data_slice = unsafe { slice::from_raw_parts(data, size_bytes as usize) }; + let value_offset = pass.base.push_constant_data.len().try_into().expect( + "Ran out of push constant space. Don't set 4gb of push constants per RenderBundle.", + ); + + pass.base.push_constant_data.extend( + data_slice + .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize) + .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])), + ); + + pass.base.commands.push(RenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset: Some(value_offset), + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_draw( + bundle: &mut RenderBundleEncoder, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, + ) { + bundle.base.commands.push(RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_draw_indexed( + bundle: &mut RenderBundleEncoder, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, + ) { + bundle.base.commands.push(RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_draw_indirect( + bundle: &mut RenderBundleEncoder, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + bundle.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: false, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_draw_indexed_indirect( + bundle: &mut RenderBundleEncoder, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + bundle.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: true, + }); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given `label` + /// is a valid null-terminated string. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_bundle_push_debug_group( + _bundle: &mut RenderBundleEncoder, + _label: RawString, + ) { + //TODO + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_pop_debug_group(_bundle: &mut RenderBundleEncoder) { + //TODO + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given `label` + /// is a valid null-terminated string. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_bundle_insert_debug_marker( + _bundle: &mut RenderBundleEncoder, + _label: RawString, + ) { + //TODO + } +} diff --git a/third_party/rust/wgpu-core/src/command/clear.rs b/third_party/rust/wgpu-core/src/command/clear.rs new file mode 100644 index 0000000000..2569fea1a4 --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/clear.rs @@ -0,0 +1,493 @@ +use std::{ops::Range, sync::Arc}; + +#[cfg(feature = "trace")] +use crate::device::trace::Command as TraceCommand; +use crate::{ + api_log, + command::CommandBuffer, + device::DeviceError, + get_lowest_common_denom, + global::Global, + hal_api::HalApi, + id::{BufferId, CommandEncoderId, DeviceId, TextureId}, + init_tracker::{MemoryInitKind, TextureInitRange}, + resource::{Resource, Texture, TextureClearMode}, + track::{TextureSelector, TextureTracker}, +}; + +use hal::CommandEncoder as _; +use thiserror::Error; +use wgt::{math::align_to, BufferAddress, BufferUsages, ImageSubresourceRange, TextureAspect}; + +/// Error encountered while attempting a clear. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum ClearError { + #[error("To use clear_texture the CLEAR_TEXTURE feature needs to be enabled")] + MissingClearTextureFeature, + #[error("Command encoder {0:?} is invalid")] + InvalidCommandEncoder(CommandEncoderId), + #[error("Device {0:?} is invalid")] + InvalidDevice(DeviceId), + #[error("Buffer {0:?} is invalid or destroyed")] + InvalidBuffer(BufferId), + #[error("Texture {0:?} is invalid or destroyed")] + InvalidTexture(TextureId), + #[error("Texture {0:?} can not be cleared")] + NoValidTextureClearMode(TextureId), + #[error("Buffer clear size {0:?} is not a multiple of `COPY_BUFFER_ALIGNMENT`")] + UnalignedFillSize(BufferAddress), + #[error("Buffer offset {0:?} is not a multiple of `COPY_BUFFER_ALIGNMENT`")] + UnalignedBufferOffset(BufferAddress), + #[error("Clear of {start_offset}..{end_offset} would end up overrunning the bounds of the buffer of size {buffer_size}")] + BufferOverrun { + start_offset: BufferAddress, + end_offset: BufferAddress, + buffer_size: BufferAddress, + }, + #[error("Destination buffer is missing the `COPY_DST` usage flag")] + MissingCopyDstUsageFlag(Option<BufferId>, Option<TextureId>), + #[error("Texture lacks the aspects that were specified in the image subresource range. Texture with format {texture_format:?}, specified was {subresource_range_aspects:?}")] + MissingTextureAspect { + texture_format: wgt::TextureFormat, + subresource_range_aspects: TextureAspect, + }, + #[error("Image subresource level range is outside of the texture's level range. texture range is {texture_level_range:?}, \ +whereas subesource range specified start {subresource_base_mip_level} and count {subresource_mip_level_count:?}")] + InvalidTextureLevelRange { + texture_level_range: Range<u32>, + subresource_base_mip_level: u32, + subresource_mip_level_count: Option<u32>, + }, + #[error("Image subresource layer range is outside of the texture's layer range. texture range is {texture_layer_range:?}, \ +whereas subesource range specified start {subresource_base_array_layer} and count {subresource_array_layer_count:?}")] + InvalidTextureLayerRange { + texture_layer_range: Range<u32>, + subresource_base_array_layer: u32, + subresource_array_layer_count: Option<u32>, + }, + #[error(transparent)] + Device(#[from] DeviceError), +} + +impl Global { + pub fn command_encoder_clear_buffer<A: HalApi>( + &self, + command_encoder_id: CommandEncoderId, + dst: BufferId, + offset: BufferAddress, + size: Option<BufferAddress>, + ) -> Result<(), ClearError> { + profiling::scope!("CommandEncoder::clear_buffer"); + api_log!("CommandEncoder::clear_buffer {dst:?}"); + + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id) + .map_err(|_| ClearError::InvalidCommandEncoder(command_encoder_id))?; + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::ClearBuffer { dst, offset, size }); + } + + let (dst_buffer, dst_pending) = { + let buffer_guard = hub.buffers.read(); + let dst_buffer = buffer_guard + .get(dst) + .map_err(|_| ClearError::InvalidBuffer(dst))?; + cmd_buf_data + .trackers + .buffers + .set_single(dst_buffer, hal::BufferUses::COPY_DST) + .ok_or(ClearError::InvalidBuffer(dst))? + }; + let snatch_guard = dst_buffer.device.snatchable_lock.read(); + let dst_raw = dst_buffer + .raw + .get(&snatch_guard) + .ok_or(ClearError::InvalidBuffer(dst))?; + if !dst_buffer.usage.contains(BufferUsages::COPY_DST) { + return Err(ClearError::MissingCopyDstUsageFlag(Some(dst), None)); + } + + // Check if offset & size are valid. + if offset % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err(ClearError::UnalignedBufferOffset(offset)); + } + if let Some(size) = size { + if size % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err(ClearError::UnalignedFillSize(size)); + } + let destination_end_offset = offset + size; + if destination_end_offset > dst_buffer.size { + return Err(ClearError::BufferOverrun { + start_offset: offset, + end_offset: destination_end_offset, + buffer_size: dst_buffer.size, + }); + } + } + + let end = match size { + Some(size) => offset + size, + None => dst_buffer.size, + }; + if offset == end { + log::trace!("Ignoring fill_buffer of size 0"); + return Ok(()); + } + + // Mark dest as initialized. + cmd_buf_data.buffer_memory_init_actions.extend( + dst_buffer.initialization_status.read().create_action( + &dst_buffer, + offset..end, + MemoryInitKind::ImplicitlyInitialized, + ), + ); + + // actual hal barrier & operation + let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard)); + let cmd_buf_raw = cmd_buf_data.encoder.open()?; + unsafe { + cmd_buf_raw.transition_buffers(dst_barrier.into_iter()); + cmd_buf_raw.clear_buffer(dst_raw, offset..end); + } + Ok(()) + } + + pub fn command_encoder_clear_texture<A: HalApi>( + &self, + command_encoder_id: CommandEncoderId, + dst: TextureId, + subresource_range: &ImageSubresourceRange, + ) -> Result<(), ClearError> { + profiling::scope!("CommandEncoder::clear_texture"); + api_log!("CommandEncoder::clear_texture {dst:?}"); + + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id) + .map_err(|_| ClearError::InvalidCommandEncoder(command_encoder_id))?; + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::ClearTexture { + dst, + subresource_range: *subresource_range, + }); + } + + if !cmd_buf.support_clear_texture { + return Err(ClearError::MissingClearTextureFeature); + } + + let dst_texture = hub + .textures + .get(dst) + .map_err(|_| ClearError::InvalidTexture(dst))?; + + // Check if subresource aspects are valid. + let clear_aspects = + hal::FormatAspects::new(dst_texture.desc.format, subresource_range.aspect); + if clear_aspects.is_empty() { + return Err(ClearError::MissingTextureAspect { + texture_format: dst_texture.desc.format, + subresource_range_aspects: subresource_range.aspect, + }); + }; + + // Check if subresource level range is valid + let subresource_mip_range = subresource_range.mip_range(dst_texture.full_range.mips.end); + if dst_texture.full_range.mips.start > subresource_mip_range.start + || dst_texture.full_range.mips.end < subresource_mip_range.end + { + return Err(ClearError::InvalidTextureLevelRange { + texture_level_range: dst_texture.full_range.mips.clone(), + subresource_base_mip_level: subresource_range.base_mip_level, + subresource_mip_level_count: subresource_range.mip_level_count, + }); + } + // Check if subresource layer range is valid + let subresource_layer_range = + subresource_range.layer_range(dst_texture.full_range.layers.end); + if dst_texture.full_range.layers.start > subresource_layer_range.start + || dst_texture.full_range.layers.end < subresource_layer_range.end + { + return Err(ClearError::InvalidTextureLayerRange { + texture_layer_range: dst_texture.full_range.layers.clone(), + subresource_base_array_layer: subresource_range.base_array_layer, + subresource_array_layer_count: subresource_range.array_layer_count, + }); + } + + let device = &cmd_buf.device; + if !device.is_valid() { + return Err(ClearError::InvalidDevice(cmd_buf.device.as_info().id())); + } + let (encoder, tracker) = cmd_buf_data.open_encoder_and_tracker()?; + + clear_texture( + &dst_texture, + TextureInitRange { + mip_range: subresource_mip_range, + layer_range: subresource_layer_range, + }, + encoder, + &mut tracker.textures, + &device.alignments, + device.zero_buffer.as_ref().unwrap(), + ) + } +} + +pub(crate) fn clear_texture<A: HalApi>( + dst_texture: &Arc<Texture<A>>, + range: TextureInitRange, + encoder: &mut A::CommandEncoder, + texture_tracker: &mut TextureTracker<A>, + alignments: &hal::Alignments, + zero_buffer: &A::Buffer, +) -> Result<(), ClearError> { + let snatch_guard = dst_texture.device.snatchable_lock.read(); + let dst_raw = dst_texture + .raw(&snatch_guard) + .ok_or_else(|| ClearError::InvalidTexture(dst_texture.as_info().id()))?; + + // Issue the right barrier. + let clear_usage = match *dst_texture.clear_mode.read() { + TextureClearMode::BufferCopy => hal::TextureUses::COPY_DST, + TextureClearMode::RenderPass { + is_color: false, .. + } => hal::TextureUses::DEPTH_STENCIL_WRITE, + TextureClearMode::Surface { .. } | TextureClearMode::RenderPass { is_color: true, .. } => { + hal::TextureUses::COLOR_TARGET + } + TextureClearMode::None => { + return Err(ClearError::NoValidTextureClearMode( + dst_texture.as_info().id(), + )); + } + }; + + let selector = TextureSelector { + mips: range.mip_range.clone(), + layers: range.layer_range.clone(), + }; + + // If we're in a texture-init usecase, we know that the texture is already + // tracked since whatever caused the init requirement, will have caused the + // usage tracker to be aware of the texture. Meaning, that it is safe to + // call call change_replace_tracked if the life_guard is already gone (i.e. + // the user no longer holds on to this texture). + // + // On the other hand, when coming via command_encoder_clear_texture, the + // life_guard is still there since in order to call it a texture object is + // needed. + // + // We could in theory distinguish these two scenarios in the internal + // clear_texture api in order to remove this check and call the cheaper + // change_replace_tracked whenever possible. + let dst_barrier = texture_tracker + .set_single(dst_texture, selector, clear_usage) + .unwrap() + .map(|pending| pending.into_hal(dst_raw)); + unsafe { + encoder.transition_textures(dst_barrier.into_iter()); + } + + // Record actual clearing + match *dst_texture.clear_mode.read() { + TextureClearMode::BufferCopy => clear_texture_via_buffer_copies::<A>( + &dst_texture.desc, + alignments, + zero_buffer, + range, + encoder, + dst_raw, + ), + TextureClearMode::Surface { .. } => { + clear_texture_via_render_passes(dst_texture, range, true, encoder)? + } + TextureClearMode::RenderPass { is_color, .. } => { + clear_texture_via_render_passes(dst_texture, range, is_color, encoder)? + } + TextureClearMode::None => { + return Err(ClearError::NoValidTextureClearMode( + dst_texture.as_info().id(), + )); + } + } + Ok(()) +} + +fn clear_texture_via_buffer_copies<A: HalApi>( + texture_desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>, + alignments: &hal::Alignments, + zero_buffer: &A::Buffer, // Buffer of size device::ZERO_BUFFER_SIZE + range: TextureInitRange, + encoder: &mut A::CommandEncoder, + dst_raw: &A::Texture, +) { + assert!(!texture_desc.format.is_depth_stencil_format()); + + if texture_desc.format == wgt::TextureFormat::NV12 { + // TODO: Currently COPY_DST for NV12 textures is unsupported. + return; + } + + // Gather list of zero_buffer copies and issue a single command then to perform them + let mut zero_buffer_copy_regions = Vec::new(); + let buffer_copy_pitch = alignments.buffer_copy_pitch.get() as u32; + let (block_width, block_height) = texture_desc.format.block_dimensions(); + let block_size = texture_desc.format.block_copy_size(None).unwrap(); + + let bytes_per_row_alignment = get_lowest_common_denom(buffer_copy_pitch, block_size); + + for mip_level in range.mip_range { + let mut mip_size = texture_desc.mip_level_size(mip_level).unwrap(); + // Round to multiple of block size + mip_size.width = align_to(mip_size.width, block_width); + mip_size.height = align_to(mip_size.height, block_height); + + let bytes_per_row = align_to( + mip_size.width / block_width * block_size, + bytes_per_row_alignment, + ); + + let max_rows_per_copy = crate::device::ZERO_BUFFER_SIZE as u32 / bytes_per_row; + // round down to a multiple of rows needed by the texture format + let max_rows_per_copy = max_rows_per_copy / block_height * block_height; + assert!( + max_rows_per_copy > 0, + "Zero buffer size is too small to fill a single row \ + of a texture with format {:?} and desc {:?}", + texture_desc.format, + texture_desc.size + ); + + let z_range = 0..(if texture_desc.dimension == wgt::TextureDimension::D3 { + mip_size.depth_or_array_layers + } else { + 1 + }); + + for array_layer in range.layer_range.clone() { + // TODO: Only doing one layer at a time for volume textures right now. + for z in z_range.clone() { + // May need multiple copies for each subresource! However, we + // assume that we never need to split a row. + let mut num_rows_left = mip_size.height; + while num_rows_left > 0 { + let num_rows = num_rows_left.min(max_rows_per_copy); + + zero_buffer_copy_regions.push(hal::BufferTextureCopy { + buffer_layout: wgt::ImageDataLayout { + offset: 0, + bytes_per_row: Some(bytes_per_row), + rows_per_image: None, + }, + texture_base: hal::TextureCopyBase { + mip_level, + array_layer, + origin: wgt::Origin3d { + x: 0, // Always full rows + y: mip_size.height - num_rows_left, + z, + }, + aspect: hal::FormatAspects::COLOR, + }, + size: hal::CopyExtent { + width: mip_size.width, // full row + height: num_rows, + depth: 1, // Only single slice of volume texture at a time right now + }, + }); + + num_rows_left -= num_rows; + } + } + } + } + + unsafe { + encoder.copy_buffer_to_texture(zero_buffer, dst_raw, zero_buffer_copy_regions.into_iter()); + } +} + +fn clear_texture_via_render_passes<A: HalApi>( + dst_texture: &Texture<A>, + range: TextureInitRange, + is_color: bool, + encoder: &mut A::CommandEncoder, +) -> Result<(), ClearError> { + assert_eq!(dst_texture.desc.dimension, wgt::TextureDimension::D2); + + let extent_base = wgt::Extent3d { + width: dst_texture.desc.size.width, + height: dst_texture.desc.size.height, + depth_or_array_layers: 1, // Only one layer is cleared at a time. + }; + let clear_mode = &dst_texture.clear_mode.read(); + + for mip_level in range.mip_range { + let extent = extent_base.mip_level_size(mip_level, dst_texture.desc.dimension); + for depth_or_layer in range.layer_range.clone() { + let color_attachments_tmp; + let (color_attachments, depth_stencil_attachment) = if is_color { + color_attachments_tmp = [Some(hal::ColorAttachment { + target: hal::Attachment { + view: Texture::get_clear_view( + clear_mode, + &dst_texture.desc, + mip_level, + depth_or_layer, + ), + usage: hal::TextureUses::COLOR_TARGET, + }, + resolve_target: None, + ops: hal::AttachmentOps::STORE, + clear_value: wgt::Color::TRANSPARENT, + })]; + (&color_attachments_tmp[..], None) + } else { + ( + &[][..], + Some(hal::DepthStencilAttachment { + target: hal::Attachment { + view: Texture::get_clear_view( + clear_mode, + &dst_texture.desc, + mip_level, + depth_or_layer, + ), + usage: hal::TextureUses::DEPTH_STENCIL_WRITE, + }, + depth_ops: hal::AttachmentOps::STORE, + stencil_ops: hal::AttachmentOps::STORE, + clear_value: (0.0, 0), + }), + ) + }; + unsafe { + encoder.begin_render_pass(&hal::RenderPassDescriptor { + label: Some("(wgpu internal) clear_texture clear pass"), + extent, + sample_count: dst_texture.desc.sample_count, + color_attachments, + depth_stencil_attachment, + multiview: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + encoder.end_render_pass(); + } + } + } + Ok(()) +} diff --git a/third_party/rust/wgpu-core/src/command/compute.rs b/third_party/rust/wgpu-core/src/command/compute.rs new file mode 100644 index 0000000000..804186a01e --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/compute.rs @@ -0,0 +1,1074 @@ +use crate::device::DeviceError; +use crate::resource::Resource; +use crate::snatch::SnatchGuard; +use crate::{ + binding_model::{ + BindError, BindGroup, LateMinBufferBindingSizeMismatch, PushConstantUploadError, + }, + command::{ + bind::Binder, + end_pipeline_statistics_query, + memory_init::{fixup_discarded_surfaces, SurfacesInDiscardState}, + BasePass, BasePassRef, BindGroupStateChange, CommandBuffer, CommandEncoderError, + CommandEncoderStatus, MapPassErr, PassErrorScope, QueryUseError, StateChange, + }, + device::{MissingDownlevelFlags, MissingFeatures}, + error::{ErrorFormatter, PrettyError}, + global::Global, + hal_api::HalApi, + hal_label, id, + id::DeviceId, + init_tracker::MemoryInitKind, + pipeline, + resource::{self}, + storage::Storage, + track::{Tracker, UsageConflict, UsageScope}, + validation::{check_buffer_usage, MissingBufferUsageError}, + Label, +}; + +use hal::CommandEncoder as _; +#[cfg(feature = "serde")] +use serde::Deserialize; +#[cfg(feature = "serde")] +use serde::Serialize; + +use thiserror::Error; + +use std::{fmt, mem, str}; + +#[doc(hidden)] +#[derive(Clone, Copy, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ComputeCommand { + SetBindGroup { + index: u32, + num_dynamic_offsets: usize, + bind_group_id: id::BindGroupId, + }, + SetPipeline(id::ComputePipelineId), + + /// Set a range of push constants to values stored in [`BasePass::push_constant_data`]. + SetPushConstant { + /// The byte offset within the push constant storage to write to. This + /// must be a multiple of four. + offset: u32, + + /// The number of bytes to write. This must be a multiple of four. + size_bytes: u32, + + /// Index in [`BasePass::push_constant_data`] of the start of the data + /// to be written. + /// + /// Note: this is not a byte offset like `offset`. Rather, it is the + /// index of the first `u32` element in `push_constant_data` to read. + values_offset: u32, + }, + + Dispatch([u32; 3]), + DispatchIndirect { + buffer_id: id::BufferId, + offset: wgt::BufferAddress, + }, + PushDebugGroup { + color: u32, + len: usize, + }, + PopDebugGroup, + InsertDebugMarker { + color: u32, + len: usize, + }, + WriteTimestamp { + query_set_id: id::QuerySetId, + query_index: u32, + }, + BeginPipelineStatisticsQuery { + query_set_id: id::QuerySetId, + query_index: u32, + }, + EndPipelineStatisticsQuery, +} + +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct ComputePass { + base: BasePass<ComputeCommand>, + parent_id: id::CommandEncoderId, + timestamp_writes: Option<ComputePassTimestampWrites>, + + // Resource binding dedupe state. + #[cfg_attr(feature = "serde", serde(skip))] + current_bind_groups: BindGroupStateChange, + #[cfg_attr(feature = "serde", serde(skip))] + current_pipeline: StateChange<id::ComputePipelineId>, +} + +impl ComputePass { + pub fn new(parent_id: id::CommandEncoderId, desc: &ComputePassDescriptor) -> Self { + Self { + base: BasePass::new(&desc.label), + parent_id, + timestamp_writes: desc.timestamp_writes.cloned(), + + current_bind_groups: BindGroupStateChange::new(), + current_pipeline: StateChange::new(), + } + } + + pub fn parent_id(&self) -> id::CommandEncoderId { + self.parent_id + } + + #[cfg(feature = "trace")] + pub fn into_command(self) -> crate::device::trace::Command { + crate::device::trace::Command::RunComputePass { + base: self.base, + timestamp_writes: self.timestamp_writes, + } + } +} + +impl fmt::Debug for ComputePass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "ComputePass {{ encoder_id: {:?}, data: {:?} commands and {:?} dynamic offsets }}", + self.parent_id, + self.base.commands.len(), + self.base.dynamic_offsets.len() + ) + } +} + +/// Describes the writing of timestamp values in a compute pass. +#[repr(C)] +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct ComputePassTimestampWrites { + /// The query set to write the timestamps to. + pub query_set: id::QuerySetId, + /// The index of the query set at which a start timestamp of this pass is written, if any. + pub beginning_of_pass_write_index: Option<u32>, + /// The index of the query set at which an end timestamp of this pass is written, if any. + pub end_of_pass_write_index: Option<u32>, +} + +#[derive(Clone, Debug, Default)] +pub struct ComputePassDescriptor<'a> { + pub label: Label<'a>, + /// Defines where and when timestamp values will be written for this pass. + pub timestamp_writes: Option<&'a ComputePassTimestampWrites>, +} + +#[derive(Clone, Debug, Error, Eq, PartialEq)] +#[non_exhaustive] +pub enum DispatchError { + #[error("Compute pipeline must be set")] + MissingPipeline, + #[error("Incompatible bind group at index {index} in the current compute pipeline")] + IncompatibleBindGroup { index: u32, diff: Vec<String> }, + #[error( + "Each current dispatch group size dimension ({current:?}) must be less or equal to {limit}" + )] + InvalidGroupSize { current: [u32; 3], limit: u32 }, + #[error(transparent)] + BindingSizeTooSmall(#[from] LateMinBufferBindingSizeMismatch), +} + +/// Error encountered when performing a compute pass. +#[derive(Clone, Debug, Error)] +pub enum ComputePassErrorInner { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + Encoder(#[from] CommandEncoderError), + #[error("Bind group at index {0:?} is invalid")] + InvalidBindGroup(usize), + #[error("Device {0:?} is invalid")] + InvalidDevice(DeviceId), + #[error("Bind group index {index} is greater than the device's requested `max_bind_group` limit {max}")] + BindGroupIndexOutOfRange { index: u32, max: u32 }, + #[error("Compute pipeline {0:?} is invalid")] + InvalidPipeline(id::ComputePipelineId), + #[error("QuerySet {0:?} is invalid")] + InvalidQuerySet(id::QuerySetId), + #[error("Indirect buffer {0:?} is invalid or destroyed")] + InvalidIndirectBuffer(id::BufferId), + #[error("Indirect buffer uses bytes {offset}..{end_offset} which overruns indirect buffer of size {buffer_size}")] + IndirectBufferOverrun { + offset: u64, + end_offset: u64, + buffer_size: u64, + }, + #[error("Buffer {0:?} is invalid or destroyed")] + InvalidBuffer(id::BufferId), + #[error(transparent)] + ResourceUsageConflict(#[from] UsageConflict), + #[error(transparent)] + MissingBufferUsage(#[from] MissingBufferUsageError), + #[error("Cannot pop debug group, because number of pushed debug groups is zero")] + InvalidPopDebugGroup, + #[error(transparent)] + Dispatch(#[from] DispatchError), + #[error(transparent)] + Bind(#[from] BindError), + #[error(transparent)] + PushConstants(#[from] PushConstantUploadError), + #[error(transparent)] + QueryUse(#[from] QueryUseError), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), +} + +impl PrettyError for ComputePassErrorInner { + fn fmt_pretty(&self, fmt: &mut ErrorFormatter) { + fmt.error(self); + match *self { + Self::InvalidPipeline(id) => { + fmt.compute_pipeline_label(&id); + } + Self::InvalidIndirectBuffer(id) => { + fmt.buffer_label(&id); + } + Self::Dispatch(DispatchError::IncompatibleBindGroup { ref diff, .. }) => { + for d in diff { + fmt.note(&d); + } + } + _ => {} + }; + } +} + +/// Error encountered when performing a compute pass. +#[derive(Clone, Debug, Error)] +#[error("{scope}")] +pub struct ComputePassError { + pub scope: PassErrorScope, + #[source] + inner: ComputePassErrorInner, +} +impl PrettyError for ComputePassError { + fn fmt_pretty(&self, fmt: &mut ErrorFormatter) { + // This error is wrapper for the inner error, + // but the scope has useful labels + fmt.error(self); + self.scope.fmt_pretty(fmt); + } +} + +impl<T, E> MapPassErr<T, ComputePassError> for Result<T, E> +where + E: Into<ComputePassErrorInner>, +{ + fn map_pass_err(self, scope: PassErrorScope) -> Result<T, ComputePassError> { + self.map_err(|inner| ComputePassError { + scope, + inner: inner.into(), + }) + } +} + +struct State<A: HalApi> { + binder: Binder<A>, + pipeline: Option<id::ComputePipelineId>, + scope: UsageScope<A>, + debug_scope_depth: u32, +} + +impl<A: HalApi> State<A> { + fn is_ready(&self) -> Result<(), DispatchError> { + let bind_mask = self.binder.invalid_mask(); + if bind_mask != 0 { + //let (expected, provided) = self.binder.entries[index as usize].info(); + let index = bind_mask.trailing_zeros(); + + return Err(DispatchError::IncompatibleBindGroup { + index, + diff: self.binder.bgl_diff(), + }); + } + if self.pipeline.is_none() { + return Err(DispatchError::MissingPipeline); + } + self.binder.check_late_buffer_bindings()?; + + Ok(()) + } + + // `extra_buffer` is there to represent the indirect buffer that is also + // part of the usage scope. + fn flush_states( + &mut self, + raw_encoder: &mut A::CommandEncoder, + base_trackers: &mut Tracker<A>, + bind_group_guard: &Storage<BindGroup<A>>, + indirect_buffer: Option<id::BufferId>, + snatch_guard: &SnatchGuard, + ) -> Result<(), UsageConflict> { + for id in self.binder.list_active() { + unsafe { self.scope.merge_bind_group(&bind_group_guard[id].used)? }; + // Note: stateless trackers are not merged: the lifetime reference + // is held to the bind group itself. + } + + for id in self.binder.list_active() { + unsafe { + base_trackers.set_and_remove_from_usage_scope_sparse( + &mut self.scope, + &bind_group_guard[id].used, + ) + } + } + + // Add the state of the indirect buffer if it hasn't been hit before. + unsafe { + base_trackers + .buffers + .set_and_remove_from_usage_scope_sparse(&mut self.scope.buffers, indirect_buffer); + } + + log::trace!("Encoding dispatch barriers"); + + CommandBuffer::drain_barriers(raw_encoder, base_trackers, snatch_guard); + Ok(()) + } +} + +// Common routines between render/compute + +impl Global { + pub fn command_encoder_run_compute_pass<A: HalApi>( + &self, + encoder_id: id::CommandEncoderId, + pass: &ComputePass, + ) -> Result<(), ComputePassError> { + self.command_encoder_run_compute_pass_impl::<A>( + encoder_id, + pass.base.as_ref(), + pass.timestamp_writes.as_ref(), + ) + } + + #[doc(hidden)] + pub fn command_encoder_run_compute_pass_impl<A: HalApi>( + &self, + encoder_id: id::CommandEncoderId, + base: BasePassRef<ComputeCommand>, + timestamp_writes: Option<&ComputePassTimestampWrites>, + ) -> Result<(), ComputePassError> { + profiling::scope!("CommandEncoder::run_compute_pass"); + let pass_scope = PassErrorScope::Pass(encoder_id); + + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id).map_pass_err(pass_scope)?; + let device = &cmd_buf.device; + if !device.is_valid() { + return Err(ComputePassErrorInner::InvalidDevice( + cmd_buf.device.as_info().id(), + )) + .map_pass_err(pass_scope); + } + + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(crate::device::trace::Command::RunComputePass { + base: BasePass::from_ref(base), + timestamp_writes: timestamp_writes.cloned(), + }); + } + + let encoder = &mut cmd_buf_data.encoder; + let status = &mut cmd_buf_data.status; + let tracker = &mut cmd_buf_data.trackers; + let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions; + let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions; + + // We automatically keep extending command buffers over time, and because + // we want to insert a command buffer _before_ what we're about to record, + // we need to make sure to close the previous one. + encoder.close().map_pass_err(pass_scope)?; + // will be reset to true if recording is done without errors + *status = CommandEncoderStatus::Error; + let raw = encoder.open().map_pass_err(pass_scope)?; + + let bind_group_guard = hub.bind_groups.read(); + let pipeline_guard = hub.compute_pipelines.read(); + let query_set_guard = hub.query_sets.read(); + let buffer_guard = hub.buffers.read(); + let texture_guard = hub.textures.read(); + + let mut state = State { + binder: Binder::new(), + pipeline: None, + scope: UsageScope::new(&*buffer_guard, &*texture_guard), + debug_scope_depth: 0, + }; + let mut temp_offsets = Vec::new(); + let mut dynamic_offset_count = 0; + let mut string_offset = 0; + let mut active_query = None; + + let timestamp_writes = if let Some(tw) = timestamp_writes { + let query_set: &resource::QuerySet<A> = tracker + .query_sets + .add_single(&*query_set_guard, tw.query_set) + .ok_or(ComputePassErrorInner::InvalidQuerySet(tw.query_set)) + .map_pass_err(pass_scope)?; + + // Unlike in render passes we can't delay resetting the query sets since + // there is no auxiliary pass. + let range = if let (Some(index_a), Some(index_b)) = + (tw.beginning_of_pass_write_index, tw.end_of_pass_write_index) + { + Some(index_a.min(index_b)..index_a.max(index_b) + 1) + } else { + tw.beginning_of_pass_write_index + .or(tw.end_of_pass_write_index) + .map(|i| i..i + 1) + }; + // Range should always be Some, both values being None should lead to a validation error. + // But no point in erroring over that nuance here! + if let Some(range) = range { + unsafe { + raw.reset_queries(query_set.raw.as_ref().unwrap(), range); + } + } + + Some(hal::ComputePassTimestampWrites { + query_set: query_set.raw.as_ref().unwrap(), + beginning_of_pass_write_index: tw.beginning_of_pass_write_index, + end_of_pass_write_index: tw.end_of_pass_write_index, + }) + } else { + None + }; + + let snatch_guard = device.snatchable_lock.read(); + + tracker.set_size( + Some(&*buffer_guard), + Some(&*texture_guard), + None, + None, + Some(&*bind_group_guard), + Some(&*pipeline_guard), + None, + None, + Some(&*query_set_guard), + ); + + let discard_hal_labels = self + .instance + .flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS); + let hal_desc = hal::ComputePassDescriptor { + label: hal_label(base.label, self.instance.flags), + timestamp_writes, + }; + + unsafe { + raw.begin_compute_pass(&hal_desc); + } + + let mut intermediate_trackers = Tracker::<A>::new(); + + // Immediate texture inits required because of prior discards. Need to + // be inserted before texture reads. + let mut pending_discard_init_fixups = SurfacesInDiscardState::new(); + + for command in base.commands { + match *command { + ComputeCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group_id, + } => { + let scope = PassErrorScope::SetBindGroup(bind_group_id); + + let max_bind_groups = cmd_buf.limits.max_bind_groups; + if index >= max_bind_groups { + return Err(ComputePassErrorInner::BindGroupIndexOutOfRange { + index, + max: max_bind_groups, + }) + .map_pass_err(scope); + } + + temp_offsets.clear(); + temp_offsets.extend_from_slice( + &base.dynamic_offsets + [dynamic_offset_count..dynamic_offset_count + num_dynamic_offsets], + ); + dynamic_offset_count += num_dynamic_offsets; + + let bind_group = tracker + .bind_groups + .add_single(&*bind_group_guard, bind_group_id) + .ok_or(ComputePassErrorInner::InvalidBindGroup(index as usize)) + .map_pass_err(scope)?; + bind_group + .validate_dynamic_bindings(index, &temp_offsets, &cmd_buf.limits) + .map_pass_err(scope)?; + + buffer_memory_init_actions.extend( + bind_group.used_buffer_ranges.iter().filter_map(|action| { + action + .buffer + .initialization_status + .read() + .check_action(action) + }), + ); + + for action in bind_group.used_texture_ranges.iter() { + pending_discard_init_fixups + .extend(texture_memory_actions.register_init_action(action)); + } + + let pipeline_layout = state.binder.pipeline_layout.clone(); + let entries = + state + .binder + .assign_group(index as usize, bind_group, &temp_offsets); + if !entries.is_empty() && pipeline_layout.is_some() { + let pipeline_layout = pipeline_layout.as_ref().unwrap().raw(); + for (i, e) in entries.iter().enumerate() { + if let Some(group) = e.group.as_ref() { + let raw_bg = group + .raw(&snatch_guard) + .ok_or(ComputePassErrorInner::InvalidBindGroup(i)) + .map_pass_err(scope)?; + unsafe { + raw.set_bind_group( + pipeline_layout, + index + i as u32, + raw_bg, + &e.dynamic_offsets, + ); + } + } + } + } + } + ComputeCommand::SetPipeline(pipeline_id) => { + let scope = PassErrorScope::SetPipelineCompute(pipeline_id); + + state.pipeline = Some(pipeline_id); + + let pipeline: &pipeline::ComputePipeline<A> = tracker + .compute_pipelines + .add_single(&*pipeline_guard, pipeline_id) + .ok_or(ComputePassErrorInner::InvalidPipeline(pipeline_id)) + .map_pass_err(scope)?; + + unsafe { + raw.set_compute_pipeline(pipeline.raw()); + } + + // Rebind resources + if state.binder.pipeline_layout.is_none() + || !state + .binder + .pipeline_layout + .as_ref() + .unwrap() + .is_equal(&pipeline.layout) + { + let (start_index, entries) = state.binder.change_pipeline_layout( + &pipeline.layout, + &pipeline.late_sized_buffer_groups, + ); + if !entries.is_empty() { + for (i, e) in entries.iter().enumerate() { + if let Some(group) = e.group.as_ref() { + let raw_bg = group + .raw(&snatch_guard) + .ok_or(ComputePassErrorInner::InvalidBindGroup(i)) + .map_pass_err(scope)?; + unsafe { + raw.set_bind_group( + pipeline.layout.raw(), + start_index as u32 + i as u32, + raw_bg, + &e.dynamic_offsets, + ); + } + } + } + } + + // Clear push constant ranges + let non_overlapping = super::bind::compute_nonoverlapping_ranges( + &pipeline.layout.push_constant_ranges, + ); + for range in non_overlapping { + let offset = range.range.start; + let size_bytes = range.range.end - offset; + super::push_constant_clear( + offset, + size_bytes, + |clear_offset, clear_data| unsafe { + raw.set_push_constants( + pipeline.layout.raw(), + wgt::ShaderStages::COMPUTE, + clear_offset, + clear_data, + ); + }, + ); + } + } + } + ComputeCommand::SetPushConstant { + offset, + size_bytes, + values_offset, + } => { + let scope = PassErrorScope::SetPushConstant; + + let end_offset_bytes = offset + size_bytes; + let values_end_offset = + (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; + let data_slice = + &base.push_constant_data[(values_offset as usize)..values_end_offset]; + + let pipeline_layout = state + .binder + .pipeline_layout + .as_ref() + //TODO: don't error here, lazily update the push constants + .ok_or(ComputePassErrorInner::Dispatch( + DispatchError::MissingPipeline, + )) + .map_pass_err(scope)?; + + pipeline_layout + .validate_push_constant_ranges( + wgt::ShaderStages::COMPUTE, + offset, + end_offset_bytes, + ) + .map_pass_err(scope)?; + + unsafe { + raw.set_push_constants( + pipeline_layout.raw(), + wgt::ShaderStages::COMPUTE, + offset, + data_slice, + ); + } + } + ComputeCommand::Dispatch(groups) => { + let scope = PassErrorScope::Dispatch { + indirect: false, + pipeline: state.pipeline, + }; + state.is_ready().map_pass_err(scope)?; + + state + .flush_states( + raw, + &mut intermediate_trackers, + &*bind_group_guard, + None, + &snatch_guard, + ) + .map_pass_err(scope)?; + + let groups_size_limit = cmd_buf.limits.max_compute_workgroups_per_dimension; + + if groups[0] > groups_size_limit + || groups[1] > groups_size_limit + || groups[2] > groups_size_limit + { + return Err(ComputePassErrorInner::Dispatch( + DispatchError::InvalidGroupSize { + current: groups, + limit: groups_size_limit, + }, + )) + .map_pass_err(scope); + } + + unsafe { + raw.dispatch(groups); + } + } + ComputeCommand::DispatchIndirect { buffer_id, offset } => { + let scope = PassErrorScope::Dispatch { + indirect: true, + pipeline: state.pipeline, + }; + + state.is_ready().map_pass_err(scope)?; + + device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) + .map_pass_err(scope)?; + + let indirect_buffer = state + .scope + .buffers + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; + check_buffer_usage(indirect_buffer.usage, wgt::BufferUsages::INDIRECT) + .map_pass_err(scope)?; + + let end_offset = offset + mem::size_of::<wgt::DispatchIndirectArgs>() as u64; + if end_offset > indirect_buffer.size { + return Err(ComputePassErrorInner::IndirectBufferOverrun { + offset, + end_offset, + buffer_size: indirect_buffer.size, + }) + .map_pass_err(scope); + } + + let buf_raw = indirect_buffer + .raw + .get(&snatch_guard) + .ok_or(ComputePassErrorInner::InvalidIndirectBuffer(buffer_id)) + .map_pass_err(scope)?; + + let stride = 3 * 4; // 3 integers, x/y/z group size + + buffer_memory_init_actions.extend( + indirect_buffer.initialization_status.read().create_action( + indirect_buffer, + offset..(offset + stride), + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + state + .flush_states( + raw, + &mut intermediate_trackers, + &*bind_group_guard, + Some(buffer_id), + &snatch_guard, + ) + .map_pass_err(scope)?; + unsafe { + raw.dispatch_indirect(buf_raw, offset); + } + } + ComputeCommand::PushDebugGroup { color: _, len } => { + state.debug_scope_depth += 1; + if !discard_hal_labels { + let label = + str::from_utf8(&base.string_data[string_offset..string_offset + len]) + .unwrap(); + unsafe { + raw.begin_debug_marker(label); + } + } + string_offset += len; + } + ComputeCommand::PopDebugGroup => { + let scope = PassErrorScope::PopDebugGroup; + + if state.debug_scope_depth == 0 { + return Err(ComputePassErrorInner::InvalidPopDebugGroup) + .map_pass_err(scope); + } + state.debug_scope_depth -= 1; + if !discard_hal_labels { + unsafe { + raw.end_debug_marker(); + } + } + } + ComputeCommand::InsertDebugMarker { color: _, len } => { + if !discard_hal_labels { + let label = + str::from_utf8(&base.string_data[string_offset..string_offset + len]) + .unwrap(); + unsafe { raw.insert_debug_marker(label) } + } + string_offset += len; + } + ComputeCommand::WriteTimestamp { + query_set_id, + query_index, + } => { + let scope = PassErrorScope::WriteTimestamp; + + device + .require_features(wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES) + .map_pass_err(scope)?; + + let query_set: &resource::QuerySet<A> = tracker + .query_sets + .add_single(&*query_set_guard, query_set_id) + .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)?; + + query_set + .validate_and_write_timestamp(raw, query_set_id, query_index, None) + .map_pass_err(scope)?; + } + ComputeCommand::BeginPipelineStatisticsQuery { + query_set_id, + query_index, + } => { + let scope = PassErrorScope::BeginPipelineStatisticsQuery; + + let query_set: &resource::QuerySet<A> = tracker + .query_sets + .add_single(&*query_set_guard, query_set_id) + .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)?; + + query_set + .validate_and_begin_pipeline_statistics_query( + raw, + query_set_id, + query_index, + None, + &mut active_query, + ) + .map_pass_err(scope)?; + } + ComputeCommand::EndPipelineStatisticsQuery => { + let scope = PassErrorScope::EndPipelineStatisticsQuery; + + end_pipeline_statistics_query(raw, &*query_set_guard, &mut active_query) + .map_pass_err(scope)?; + } + } + } + + unsafe { + raw.end_compute_pass(); + } + + // We've successfully recorded the compute pass, bring the + // command buffer out of the error state. + *status = CommandEncoderStatus::Recording; + + // Stop the current command buffer. + encoder.close().map_pass_err(pass_scope)?; + + // Create a new command buffer, which we will insert _before_ the body of the compute pass. + // + // Use that buffer to insert barriers and clear discarded images. + let transit = encoder.open().map_pass_err(pass_scope)?; + fixup_discarded_surfaces( + pending_discard_init_fixups.into_iter(), + transit, + &mut tracker.textures, + device, + ); + CommandBuffer::insert_barriers_from_tracker( + transit, + tracker, + &intermediate_trackers, + &snatch_guard, + ); + // Close the command buffer, and swap it with the previous. + encoder.close_and_swap().map_pass_err(pass_scope)?; + + Ok(()) + } +} + +pub mod compute_ffi { + use super::{ComputeCommand, ComputePass}; + use crate::{id, RawString}; + use std::{convert::TryInto, ffi, slice}; + use wgt::{BufferAddress, DynamicOffset}; + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `offset_length` elements. + #[no_mangle] + pub unsafe extern "C" fn wgpu_compute_pass_set_bind_group( + pass: &mut ComputePass, + index: u32, + bind_group_id: id::BindGroupId, + offsets: *const DynamicOffset, + offset_length: usize, + ) { + let redundant = unsafe { + pass.current_bind_groups.set_and_check_redundant( + bind_group_id, + index, + &mut pass.base.dynamic_offsets, + offsets, + offset_length, + ) + }; + + if redundant { + return; + } + + pass.base.commands.push(ComputeCommand::SetBindGroup { + index, + num_dynamic_offsets: offset_length, + bind_group_id, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_set_pipeline( + pass: &mut ComputePass, + pipeline_id: id::ComputePipelineId, + ) { + if pass.current_pipeline.set_and_check_redundant(pipeline_id) { + return; + } + + pass.base + .commands + .push(ComputeCommand::SetPipeline(pipeline_id)); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `size_bytes` bytes. + #[no_mangle] + pub unsafe extern "C" fn wgpu_compute_pass_set_push_constant( + pass: &mut ComputePass, + offset: u32, + size_bytes: u32, + data: *const u8, + ) { + assert_eq!( + offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant offset must be aligned to 4 bytes." + ); + assert_eq!( + size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant size must be aligned to 4 bytes." + ); + let data_slice = unsafe { slice::from_raw_parts(data, size_bytes as usize) }; + let value_offset = pass.base.push_constant_data.len().try_into().expect( + "Ran out of push constant space. Don't set 4gb of push constants per ComputePass.", + ); + + pass.base.push_constant_data.extend( + data_slice + .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize) + .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])), + ); + + pass.base.commands.push(ComputeCommand::SetPushConstant { + offset, + size_bytes, + values_offset: value_offset, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_dispatch_workgroups( + pass: &mut ComputePass, + groups_x: u32, + groups_y: u32, + groups_z: u32, + ) { + pass.base + .commands + .push(ComputeCommand::Dispatch([groups_x, groups_y, groups_z])); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_dispatch_workgroups_indirect( + pass: &mut ComputePass, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + pass.base + .commands + .push(ComputeCommand::DispatchIndirect { buffer_id, offset }); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given `label` + /// is a valid null-terminated string. + #[no_mangle] + pub unsafe extern "C" fn wgpu_compute_pass_push_debug_group( + pass: &mut ComputePass, + label: RawString, + color: u32, + ) { + let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes(); + pass.base.string_data.extend_from_slice(bytes); + + pass.base.commands.push(ComputeCommand::PushDebugGroup { + color, + len: bytes.len(), + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_pop_debug_group(pass: &mut ComputePass) { + pass.base.commands.push(ComputeCommand::PopDebugGroup); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given `label` + /// is a valid null-terminated string. + #[no_mangle] + pub unsafe extern "C" fn wgpu_compute_pass_insert_debug_marker( + pass: &mut ComputePass, + label: RawString, + color: u32, + ) { + let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes(); + pass.base.string_data.extend_from_slice(bytes); + + pass.base.commands.push(ComputeCommand::InsertDebugMarker { + color, + len: bytes.len(), + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_write_timestamp( + pass: &mut ComputePass, + query_set_id: id::QuerySetId, + query_index: u32, + ) { + pass.base.commands.push(ComputeCommand::WriteTimestamp { + query_set_id, + query_index, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_begin_pipeline_statistics_query( + pass: &mut ComputePass, + query_set_id: id::QuerySetId, + query_index: u32, + ) { + pass.base + .commands + .push(ComputeCommand::BeginPipelineStatisticsQuery { + query_set_id, + query_index, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_end_pipeline_statistics_query(pass: &mut ComputePass) { + pass.base + .commands + .push(ComputeCommand::EndPipelineStatisticsQuery); + } +} diff --git a/third_party/rust/wgpu-core/src/command/draw.rs b/third_party/rust/wgpu-core/src/command/draw.rs new file mode 100644 index 0000000000..e03a78ee93 --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/draw.rs @@ -0,0 +1,247 @@ +/*! Draw structures - shared between render passes and bundles. +!*/ + +use crate::{ + binding_model::{LateMinBufferBindingSizeMismatch, PushConstantUploadError}, + error::ErrorFormatter, + id, + track::UsageConflict, + validation::{MissingBufferUsageError, MissingTextureUsageError}, +}; +use wgt::{BufferAddress, BufferSize, Color, VertexStepMode}; + +use std::num::NonZeroU32; +use thiserror::Error; + +/// Error validating a draw call. +#[derive(Clone, Debug, Error, Eq, PartialEq)] +#[non_exhaustive] +pub enum DrawError { + #[error("Blend constant needs to be set")] + MissingBlendConstant, + #[error("Render pipeline must be set")] + MissingPipeline, + #[error("Vertex buffer {index} must be set")] + MissingVertexBuffer { index: u32 }, + #[error("Index buffer must be set")] + MissingIndexBuffer, + #[error("Incompatible bind group at index {index} in the current render pipeline")] + IncompatibleBindGroup { index: u32, diff: Vec<String> }, + #[error("Vertex {last_vertex} extends beyond limit {vertex_limit} imposed by the buffer in slot {slot}. Did you bind the correct `Vertex` step-rate vertex buffer?")] + VertexBeyondLimit { + last_vertex: u64, + vertex_limit: u64, + slot: u32, + }, + #[error("{step_mode:?} buffer out of bounds at slot {slot}. Offset {offset} beyond limit {limit}. Did you bind the correct `Vertex` step-rate vertex buffer?")] + VertexOutOfBounds { + step_mode: VertexStepMode, + offset: u64, + limit: u64, + slot: u32, + }, + #[error("Instance {last_instance} extends beyond limit {instance_limit} imposed by the buffer in slot {slot}. Did you bind the correct `Instance` step-rate vertex buffer?")] + InstanceBeyondLimit { + last_instance: u64, + instance_limit: u64, + slot: u32, + }, + #[error("Index {last_index} extends beyond limit {index_limit}. Did you bind the correct index buffer?")] + IndexBeyondLimit { last_index: u64, index_limit: u64 }, + #[error( + "Pipeline index format ({pipeline:?}) and buffer index format ({buffer:?}) do not match" + )] + UnmatchedIndexFormats { + pipeline: wgt::IndexFormat, + buffer: wgt::IndexFormat, + }, + #[error(transparent)] + BindingSizeTooSmall(#[from] LateMinBufferBindingSizeMismatch), +} + +/// Error encountered when encoding a render command. +/// This is the shared error set between render bundles and passes. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum RenderCommandError { + #[error("Bind group {0:?} is invalid")] + InvalidBindGroup(id::BindGroupId), + #[error("Render bundle {0:?} is invalid")] + InvalidRenderBundle(id::RenderBundleId), + #[error("Bind group index {index} is greater than the device's requested `max_bind_group` limit {max}")] + BindGroupIndexOutOfRange { index: u32, max: u32 }, + #[error("Vertex buffer index {index} is greater than the device's requested `max_vertex_buffers` limit {max}")] + VertexBufferIndexOutOfRange { index: u32, max: u32 }, + #[error("Dynamic buffer offset {0} does not respect device's requested `{1}` limit {2}")] + UnalignedBufferOffset(u64, &'static str, u32), + #[error("Number of buffer offsets ({actual}) does not match the number of dynamic bindings ({expected})")] + InvalidDynamicOffsetCount { actual: usize, expected: usize }, + #[error("Render pipeline {0:?} is invalid")] + InvalidPipeline(id::RenderPipelineId), + #[error("QuerySet {0:?} is invalid")] + InvalidQuerySet(id::QuerySetId), + #[error("Render pipeline targets are incompatible with render pass")] + IncompatiblePipelineTargets(#[from] crate::device::RenderPassCompatibilityError), + #[error("Pipeline writes to depth/stencil, while the pass has read-only depth/stencil")] + IncompatiblePipelineRods, + #[error(transparent)] + UsageConflict(#[from] UsageConflict), + #[error("Buffer {0:?} is destroyed")] + DestroyedBuffer(id::BufferId), + #[error(transparent)] + MissingBufferUsage(#[from] MissingBufferUsageError), + #[error(transparent)] + MissingTextureUsage(#[from] MissingTextureUsageError), + #[error(transparent)] + PushConstants(#[from] PushConstantUploadError), + #[error("Viewport has invalid rect {0:?}; origin and/or size is less than or equal to 0, and/or is not contained in the render target {1:?}")] + InvalidViewportRect(Rect<f32>, wgt::Extent3d), + #[error("Viewport minDepth {0} and/or maxDepth {1} are not in [0, 1]")] + InvalidViewportDepth(f32, f32), + #[error("Scissor {0:?} is not contained in the render target {1:?}")] + InvalidScissorRect(Rect<u32>, wgt::Extent3d), + #[error("Support for {0} is not implemented yet")] + Unimplemented(&'static str), +} +impl crate::error::PrettyError for RenderCommandError { + fn fmt_pretty(&self, fmt: &mut ErrorFormatter) { + fmt.error(self); + match *self { + Self::InvalidBindGroup(id) => { + fmt.bind_group_label(&id); + } + Self::InvalidPipeline(id) => { + fmt.render_pipeline_label(&id); + } + Self::UsageConflict(UsageConflict::TextureInvalid { id }) => { + fmt.texture_label(&id); + } + Self::UsageConflict(UsageConflict::BufferInvalid { id }) + | Self::DestroyedBuffer(id) => { + fmt.buffer_label(&id); + } + _ => {} + }; + } +} + +#[derive(Clone, Copy, Debug, Default)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Rect<T> { + pub x: T, + pub y: T, + pub w: T, + pub h: T, +} + +#[doc(hidden)] +#[derive(Clone, Copy, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum RenderCommand { + SetBindGroup { + index: u32, + num_dynamic_offsets: usize, + bind_group_id: id::BindGroupId, + }, + SetPipeline(id::RenderPipelineId), + SetIndexBuffer { + buffer_id: id::BufferId, + index_format: wgt::IndexFormat, + offset: BufferAddress, + size: Option<BufferSize>, + }, + SetVertexBuffer { + slot: u32, + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option<BufferSize>, + }, + SetBlendConstant(Color), + SetStencilReference(u32), + SetViewport { + rect: Rect<f32>, + //TODO: use half-float to reduce the size? + depth_min: f32, + depth_max: f32, + }, + SetScissor(Rect<u32>), + + /// Set a range of push constants to values stored in [`BasePass::push_constant_data`]. + /// + /// See [`wgpu::RenderPass::set_push_constants`] for a detailed explanation + /// of the restrictions these commands must satisfy. + SetPushConstant { + /// Which stages we are setting push constant values for. + stages: wgt::ShaderStages, + + /// The byte offset within the push constant storage to write to. This + /// must be a multiple of four. + offset: u32, + + /// The number of bytes to write. This must be a multiple of four. + size_bytes: u32, + + /// Index in [`BasePass::push_constant_data`] of the start of the data + /// to be written. + /// + /// Note: this is not a byte offset like `offset`. Rather, it is the + /// index of the first `u32` element in `push_constant_data` to read. + /// + /// `None` means zeros should be written to the destination range, and + /// there is no corresponding data in `push_constant_data`. This is used + /// by render bundles, which explicitly clear out any state that + /// post-bundle code might see. + values_offset: Option<u32>, + }, + Draw { + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, + }, + DrawIndexed { + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, + }, + MultiDrawIndirect { + buffer_id: id::BufferId, + offset: BufferAddress, + /// Count of `None` represents a non-multi call. + count: Option<NonZeroU32>, + indexed: bool, + }, + MultiDrawIndirectCount { + buffer_id: id::BufferId, + offset: BufferAddress, + count_buffer_id: id::BufferId, + count_buffer_offset: BufferAddress, + max_count: u32, + indexed: bool, + }, + PushDebugGroup { + color: u32, + len: usize, + }, + PopDebugGroup, + InsertDebugMarker { + color: u32, + len: usize, + }, + WriteTimestamp { + query_set_id: id::QuerySetId, + query_index: u32, + }, + BeginOcclusionQuery { + query_index: u32, + }, + EndOcclusionQuery, + BeginPipelineStatisticsQuery { + query_set_id: id::QuerySetId, + query_index: u32, + }, + EndPipelineStatisticsQuery, + ExecuteBundle(id::RenderBundleId), +} diff --git a/third_party/rust/wgpu-core/src/command/memory_init.rs b/third_party/rust/wgpu-core/src/command/memory_init.rs new file mode 100644 index 0000000000..3bfc71f4f7 --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/memory_init.rs @@ -0,0 +1,342 @@ +use std::{collections::hash_map::Entry, ops::Range, sync::Arc, vec::Drain}; + +use hal::CommandEncoder; + +use crate::{ + device::Device, + hal_api::HalApi, + init_tracker::*, + resource::{Resource, Texture}, + track::{TextureTracker, Tracker}, + FastHashMap, +}; + +use super::{ + clear::clear_texture, BakedCommands, ClearError, DestroyedBufferError, DestroyedTextureError, +}; + +/// Surface that was discarded by `StoreOp::Discard` of a preceding renderpass. +/// Any read access to this surface needs to be preceded by a texture initialization. +#[derive(Clone)] +pub(crate) struct TextureSurfaceDiscard<A: HalApi> { + pub texture: Arc<Texture<A>>, + pub mip_level: u32, + pub layer: u32, +} + +pub(crate) type SurfacesInDiscardState<A> = Vec<TextureSurfaceDiscard<A>>; + +pub(crate) struct CommandBufferTextureMemoryActions<A: HalApi> { + /// The tracker actions that we need to be executed before the command + /// buffer is executed. + init_actions: Vec<TextureInitTrackerAction<A>>, + /// All the discards that haven't been followed by init again within the + /// command buffer i.e. everything in this list resets the texture init + /// state *after* the command buffer execution + discards: Vec<TextureSurfaceDiscard<A>>, +} + +impl<A: HalApi> Default for CommandBufferTextureMemoryActions<A> { + fn default() -> Self { + Self { + init_actions: Default::default(), + discards: Default::default(), + } + } +} + +impl<A: HalApi> CommandBufferTextureMemoryActions<A> { + pub(crate) fn drain_init_actions(&mut self) -> Drain<TextureInitTrackerAction<A>> { + self.init_actions.drain(..) + } + + pub(crate) fn discard(&mut self, discard: TextureSurfaceDiscard<A>) { + self.discards.push(discard); + } + + // Registers a TextureInitTrackerAction. + // Returns previously discarded surface that need to be initialized *immediately* now. + // Only returns a non-empty list if action is MemoryInitKind::NeedsInitializedMemory. + #[must_use] + pub(crate) fn register_init_action( + &mut self, + action: &TextureInitTrackerAction<A>, + ) -> SurfacesInDiscardState<A> { + let mut immediately_necessary_clears = SurfacesInDiscardState::new(); + + // Note that within a command buffer we may stack arbitrary memory init + // actions on the same texture Since we react to them in sequence, they + // are going to be dropped again at queue submit + // + // We don't need to add MemoryInitKind::NeedsInitializedMemory to + // init_actions if a surface is part of the discard list. But that would + // mean splitting up the action which is more than we'd win here. + self.init_actions.extend( + action + .texture + .initialization_status + .read() + .check_action(action), + ); + + // We expect very few discarded surfaces at any point in time which is + // why a simple linear search is likely best. (i.e. most of the time + // self.discards is empty!) + let init_actions = &mut self.init_actions; + self.discards.retain(|discarded_surface| { + if discarded_surface.texture.as_info().id() == action.texture.as_info().id() + && action.range.layer_range.contains(&discarded_surface.layer) + && action + .range + .mip_range + .contains(&discarded_surface.mip_level) + { + if let MemoryInitKind::NeedsInitializedMemory = action.kind { + immediately_necessary_clears.push(discarded_surface.clone()); + + // Mark surface as implicitly initialized (this is relevant + // because it might have been uninitialized prior to + // discarding + init_actions.push(TextureInitTrackerAction { + texture: discarded_surface.texture.clone(), + range: TextureInitRange { + mip_range: discarded_surface.mip_level + ..(discarded_surface.mip_level + 1), + layer_range: discarded_surface.layer..(discarded_surface.layer + 1), + }, + kind: MemoryInitKind::ImplicitlyInitialized, + }); + } + false + } else { + true + } + }); + + immediately_necessary_clears + } + + // Shortcut for register_init_action when it is known that the action is an + // implicit init, not requiring any immediate resource init. + pub(crate) fn register_implicit_init( + &mut self, + texture: &Arc<Texture<A>>, + range: TextureInitRange, + ) { + let must_be_empty = self.register_init_action(&TextureInitTrackerAction { + texture: texture.clone(), + range, + kind: MemoryInitKind::ImplicitlyInitialized, + }); + assert!(must_be_empty.is_empty()); + } +} + +// Utility function that takes discarded surfaces from (several calls to) +// register_init_action and initializes them on the spot. +// +// Takes care of barriers as well! +pub(crate) fn fixup_discarded_surfaces< + A: HalApi, + InitIter: Iterator<Item = TextureSurfaceDiscard<A>>, +>( + inits: InitIter, + encoder: &mut A::CommandEncoder, + texture_tracker: &mut TextureTracker<A>, + device: &Device<A>, +) { + for init in inits { + clear_texture( + &init.texture, + TextureInitRange { + mip_range: init.mip_level..(init.mip_level + 1), + layer_range: init.layer..(init.layer + 1), + }, + encoder, + texture_tracker, + &device.alignments, + device.zero_buffer.as_ref().unwrap(), + ) + .unwrap(); + } +} + +impl<A: HalApi> BakedCommands<A> { + // inserts all buffer initializations that are going to be needed for + // executing the commands and updates resource init states accordingly + pub(crate) fn initialize_buffer_memory( + &mut self, + device_tracker: &mut Tracker<A>, + ) -> Result<(), DestroyedBufferError> { + // Gather init ranges for each buffer so we can collapse them. + // It is not possible to do this at an earlier point since previously + // executed command buffer change the resource init state. + let mut uninitialized_ranges_per_buffer = FastHashMap::default(); + for buffer_use in self.buffer_memory_init_actions.drain(..) { + let mut initialization_status = buffer_use.buffer.initialization_status.write(); + + // align the end to 4 + let end_remainder = buffer_use.range.end % wgt::COPY_BUFFER_ALIGNMENT; + let end = if end_remainder == 0 { + buffer_use.range.end + } else { + buffer_use.range.end + wgt::COPY_BUFFER_ALIGNMENT - end_remainder + }; + let uninitialized_ranges = initialization_status.drain(buffer_use.range.start..end); + + match buffer_use.kind { + MemoryInitKind::ImplicitlyInitialized => {} + MemoryInitKind::NeedsInitializedMemory => { + match uninitialized_ranges_per_buffer.entry(buffer_use.buffer.as_info().id()) { + Entry::Vacant(e) => { + e.insert(( + buffer_use.buffer.clone(), + uninitialized_ranges.collect::<Vec<Range<wgt::BufferAddress>>>(), + )); + } + Entry::Occupied(mut e) => { + e.get_mut().1.extend(uninitialized_ranges); + } + } + } + } + } + + for (buffer_id, (buffer, mut ranges)) in uninitialized_ranges_per_buffer { + // Collapse touching ranges. + ranges.sort_by_key(|r| r.start); + for i in (1..ranges.len()).rev() { + // The memory init tracker made sure of this! + assert!(ranges[i - 1].end <= ranges[i].start); + if ranges[i].start == ranges[i - 1].end { + ranges[i - 1].end = ranges[i].end; + ranges.swap_remove(i); // Ordering not important at this point + } + } + + // Don't do use_replace since the buffer may already no longer have + // a ref_count. + // + // However, we *know* that it is currently in use, so the tracker + // must already know about it. + let transition = device_tracker + .buffers + .set_single(&buffer, hal::BufferUses::COPY_DST) + .unwrap() + .1; + + let snatch_guard = buffer.device.snatchable_lock.read(); + let raw_buf = buffer + .raw + .get(&snatch_guard) + .ok_or(DestroyedBufferError(buffer_id))?; + + unsafe { + self.encoder.transition_buffers( + transition + .map(|pending| pending.into_hal(&buffer, &snatch_guard)) + .into_iter(), + ); + } + + for range in ranges.iter() { + assert!( + range.start % wgt::COPY_BUFFER_ALIGNMENT == 0, + "Buffer {:?} has an uninitialized range with a start \ + not aligned to 4 (start was {})", + raw_buf, + range.start + ); + assert!( + range.end % wgt::COPY_BUFFER_ALIGNMENT == 0, + "Buffer {:?} has an uninitialized range with an end \ + not aligned to 4 (end was {})", + raw_buf, + range.end + ); + + unsafe { + self.encoder.clear_buffer(raw_buf, range.clone()); + } + } + } + Ok(()) + } + + // inserts all texture initializations that are going to be needed for + // executing the commands and updates resource init states accordingly any + // textures that are left discarded by this command buffer will be marked as + // uninitialized + pub(crate) fn initialize_texture_memory( + &mut self, + device_tracker: &mut Tracker<A>, + device: &Device<A>, + ) -> Result<(), DestroyedTextureError> { + let mut ranges: Vec<TextureInitRange> = Vec::new(); + for texture_use in self.texture_memory_actions.drain_init_actions() { + let mut initialization_status = texture_use.texture.initialization_status.write(); + let use_range = texture_use.range; + let affected_mip_trackers = initialization_status + .mips + .iter_mut() + .enumerate() + .skip(use_range.mip_range.start as usize) + .take((use_range.mip_range.end - use_range.mip_range.start) as usize); + + match texture_use.kind { + MemoryInitKind::ImplicitlyInitialized => { + for (_, mip_tracker) in affected_mip_trackers { + mip_tracker.drain(use_range.layer_range.clone()); + } + } + MemoryInitKind::NeedsInitializedMemory => { + for (mip_level, mip_tracker) in affected_mip_trackers { + for layer_range in mip_tracker.drain(use_range.layer_range.clone()) { + ranges.push(TextureInitRange { + mip_range: (mip_level as u32)..(mip_level as u32 + 1), + layer_range, + }); + } + } + } + } + + // TODO: Could we attempt some range collapsing here? + for range in ranges.drain(..) { + let clear_result = clear_texture( + &texture_use.texture, + range, + &mut self.encoder, + &mut device_tracker.textures, + &device.alignments, + device.zero_buffer.as_ref().unwrap(), + ); + + // A Texture can be destroyed between the command recording + // and now, this is out of our control so we have to handle + // it gracefully. + if let Err(ClearError::InvalidTexture(id)) = clear_result { + return Err(DestroyedTextureError(id)); + } + + // Other errors are unexpected. + if let Err(error) = clear_result { + panic!("{error}"); + } + } + } + + // Now that all buffers/textures have the proper init state for before + // cmdbuf start, we discard init states for textures it left discarded + // after its execution. + for surface_discard in self.texture_memory_actions.discards.iter() { + surface_discard + .texture + .initialization_status + .write() + .discard(surface_discard.mip_level, surface_discard.layer); + } + + Ok(()) + } +} diff --git a/third_party/rust/wgpu-core/src/command/mod.rs b/third_party/rust/wgpu-core/src/command/mod.rs new file mode 100644 index 0000000000..2d5fca200a --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/mod.rs @@ -0,0 +1,732 @@ +mod bind; +mod bundle; +mod clear; +mod compute; +mod draw; +mod memory_init; +mod query; +mod render; +mod transfer; + +use std::slice; +use std::sync::Arc; + +pub(crate) use self::clear::clear_texture; +pub use self::{ + bundle::*, clear::ClearError, compute::*, draw::*, query::*, render::*, transfer::*, +}; + +use self::memory_init::CommandBufferTextureMemoryActions; + +use crate::device::{Device, DeviceError}; +use crate::error::{ErrorFormatter, PrettyError}; +use crate::hub::Hub; +use crate::id::CommandBufferId; +use crate::snatch::SnatchGuard; + +use crate::init_tracker::BufferInitTrackerAction; +use crate::resource::{Resource, ResourceInfo, ResourceType}; +use crate::track::{Tracker, UsageScope}; +use crate::{api_log, global::Global, hal_api::HalApi, id, resource_log, Label}; + +use hal::CommandEncoder as _; +use parking_lot::Mutex; +use thiserror::Error; + +#[cfg(feature = "trace")] +use crate::device::trace::Command as TraceCommand; + +const PUSH_CONSTANT_CLEAR_ARRAY: &[u32] = &[0_u32; 64]; + +#[derive(Debug)] +pub(crate) enum CommandEncoderStatus { + Recording, + Finished, + Error, +} + +pub(crate) struct CommandEncoder<A: HalApi> { + raw: A::CommandEncoder, + list: Vec<A::CommandBuffer>, + is_open: bool, + label: Option<String>, +} + +//TODO: handle errors better +impl<A: HalApi> CommandEncoder<A> { + /// Closes the live encoder + fn close_and_swap(&mut self) -> Result<(), DeviceError> { + if self.is_open { + self.is_open = false; + let new = unsafe { self.raw.end_encoding()? }; + self.list.insert(self.list.len() - 1, new); + } + + Ok(()) + } + + fn close(&mut self) -> Result<(), DeviceError> { + if self.is_open { + self.is_open = false; + let cmd_buf = unsafe { self.raw.end_encoding()? }; + self.list.push(cmd_buf); + } + + Ok(()) + } + + fn discard(&mut self) { + if self.is_open { + self.is_open = false; + unsafe { self.raw.discard_encoding() }; + } + } + + fn open(&mut self) -> Result<&mut A::CommandEncoder, DeviceError> { + if !self.is_open { + self.is_open = true; + let label = self.label.as_deref(); + unsafe { self.raw.begin_encoding(label)? }; + } + + Ok(&mut self.raw) + } + + fn open_pass(&mut self, label: Option<&str>) -> Result<(), DeviceError> { + self.is_open = true; + unsafe { self.raw.begin_encoding(label)? }; + + Ok(()) + } +} + +pub struct BakedCommands<A: HalApi> { + pub(crate) encoder: A::CommandEncoder, + pub(crate) list: Vec<A::CommandBuffer>, + pub(crate) trackers: Tracker<A>, + buffer_memory_init_actions: Vec<BufferInitTrackerAction<A>>, + texture_memory_actions: CommandBufferTextureMemoryActions<A>, +} + +pub(crate) struct DestroyedBufferError(pub id::BufferId); +pub(crate) struct DestroyedTextureError(pub id::TextureId); + +pub struct CommandBufferMutable<A: HalApi> { + encoder: CommandEncoder<A>, + status: CommandEncoderStatus, + pub(crate) trackers: Tracker<A>, + buffer_memory_init_actions: Vec<BufferInitTrackerAction<A>>, + texture_memory_actions: CommandBufferTextureMemoryActions<A>, + pub(crate) pending_query_resets: QueryResetMap<A>, + #[cfg(feature = "trace")] + pub(crate) commands: Option<Vec<TraceCommand>>, +} + +impl<A: HalApi> CommandBufferMutable<A> { + pub(crate) fn open_encoder_and_tracker( + &mut self, + ) -> Result<(&mut A::CommandEncoder, &mut Tracker<A>), DeviceError> { + let encoder = self.encoder.open()?; + let tracker = &mut self.trackers; + + Ok((encoder, tracker)) + } +} + +pub struct CommandBuffer<A: HalApi> { + pub(crate) device: Arc<Device<A>>, + limits: wgt::Limits, + support_clear_texture: bool, + pub(crate) info: ResourceInfo<CommandBuffer<A>>, + pub(crate) data: Mutex<Option<CommandBufferMutable<A>>>, +} + +impl<A: HalApi> Drop for CommandBuffer<A> { + fn drop(&mut self) { + if self.data.lock().is_none() { + return; + } + resource_log!("resource::CommandBuffer::drop {:?}", self.info.label()); + let mut baked = self.extract_baked_commands(); + unsafe { + baked.encoder.reset_all(baked.list.into_iter()); + } + unsafe { + use hal::Device; + self.device.raw().destroy_command_encoder(baked.encoder); + } + } +} + +impl<A: HalApi> CommandBuffer<A> { + pub(crate) fn new( + encoder: A::CommandEncoder, + device: &Arc<Device<A>>, + #[cfg(feature = "trace")] enable_tracing: bool, + label: Option<String>, + ) -> Self { + CommandBuffer { + device: device.clone(), + limits: device.limits.clone(), + support_clear_texture: device.features.contains(wgt::Features::CLEAR_TEXTURE), + info: ResourceInfo::new( + label + .as_ref() + .unwrap_or(&String::from("<CommandBuffer>")) + .as_str(), + ), + data: Mutex::new(Some(CommandBufferMutable { + encoder: CommandEncoder { + raw: encoder, + is_open: false, + list: Vec::new(), + label, + }, + status: CommandEncoderStatus::Recording, + trackers: Tracker::new(), + buffer_memory_init_actions: Default::default(), + texture_memory_actions: Default::default(), + pending_query_resets: QueryResetMap::new(), + #[cfg(feature = "trace")] + commands: if enable_tracing { + Some(Vec::new()) + } else { + None + }, + })), + } + } + + pub(crate) fn insert_barriers_from_tracker( + raw: &mut A::CommandEncoder, + base: &mut Tracker<A>, + head: &Tracker<A>, + snatch_guard: &SnatchGuard, + ) { + profiling::scope!("insert_barriers"); + + base.buffers.set_from_tracker(&head.buffers); + base.textures.set_from_tracker(&head.textures); + + Self::drain_barriers(raw, base, snatch_guard); + } + + pub(crate) fn insert_barriers_from_scope( + raw: &mut A::CommandEncoder, + base: &mut Tracker<A>, + head: &UsageScope<A>, + snatch_guard: &SnatchGuard, + ) { + profiling::scope!("insert_barriers"); + + base.buffers.set_from_usage_scope(&head.buffers); + base.textures.set_from_usage_scope(&head.textures); + + Self::drain_barriers(raw, base, snatch_guard); + } + + pub(crate) fn drain_barriers( + raw: &mut A::CommandEncoder, + base: &mut Tracker<A>, + snatch_guard: &SnatchGuard, + ) { + profiling::scope!("drain_barriers"); + + let buffer_barriers = base.buffers.drain_transitions(snatch_guard); + let (transitions, textures) = base.textures.drain_transitions(snatch_guard); + let texture_barriers = transitions + .into_iter() + .enumerate() + .map(|(i, p)| p.into_hal(textures[i].unwrap().raw().unwrap())); + + unsafe { + raw.transition_buffers(buffer_barriers); + raw.transition_textures(texture_barriers); + } + } +} + +impl<A: HalApi> CommandBuffer<A> { + fn get_encoder( + hub: &Hub<A>, + id: id::CommandEncoderId, + ) -> Result<Arc<Self>, CommandEncoderError> { + let storage = hub.command_buffers.read(); + match storage.get(id.transmute()) { + Ok(cmd_buf) => match cmd_buf.data.lock().as_ref().unwrap().status { + CommandEncoderStatus::Recording => Ok(cmd_buf.clone()), + CommandEncoderStatus::Finished => Err(CommandEncoderError::NotRecording), + CommandEncoderStatus::Error => Err(CommandEncoderError::Invalid), + }, + Err(_) => Err(CommandEncoderError::Invalid), + } + } + + pub fn is_finished(&self) -> bool { + match self.data.lock().as_ref().unwrap().status { + CommandEncoderStatus::Finished => true, + _ => false, + } + } + + pub(crate) fn extract_baked_commands(&mut self) -> BakedCommands<A> { + log::trace!( + "Extracting BakedCommands from CommandBuffer {:?}", + self.info.label() + ); + let data = self.data.lock().take().unwrap(); + BakedCommands { + encoder: data.encoder.raw, + list: data.encoder.list, + trackers: data.trackers, + buffer_memory_init_actions: data.buffer_memory_init_actions, + texture_memory_actions: data.texture_memory_actions, + } + } + + pub(crate) fn from_arc_into_baked(self: Arc<Self>) -> BakedCommands<A> { + if let Some(mut command_buffer) = Arc::into_inner(self) { + command_buffer.extract_baked_commands() + } else { + panic!("CommandBuffer cannot be destroyed because is still in use"); + } + } +} + +impl<A: HalApi> Resource for CommandBuffer<A> { + const TYPE: ResourceType = "CommandBuffer"; + + type Marker = crate::id::markers::CommandBuffer; + + fn as_info(&self) -> &ResourceInfo<Self> { + &self.info + } + + fn as_info_mut(&mut self) -> &mut ResourceInfo<Self> { + &mut self.info + } + + fn label(&self) -> String { + let str = match self.data.lock().as_ref().unwrap().encoder.label.as_ref() { + Some(label) => label.clone(), + _ => String::new(), + }; + str + } +} + +#[derive(Copy, Clone, Debug)] +pub struct BasePassRef<'a, C> { + pub label: Option<&'a str>, + pub commands: &'a [C], + pub dynamic_offsets: &'a [wgt::DynamicOffset], + pub string_data: &'a [u8], + pub push_constant_data: &'a [u32], +} + +/// A stream of commands for a render pass or compute pass. +/// +/// This also contains side tables referred to by certain commands, +/// like dynamic offsets for [`SetBindGroup`] or string data for +/// [`InsertDebugMarker`]. +/// +/// Render passes use `BasePass<RenderCommand>`, whereas compute +/// passes use `BasePass<ComputeCommand>`. +/// +/// [`SetBindGroup`]: RenderCommand::SetBindGroup +/// [`InsertDebugMarker`]: RenderCommand::InsertDebugMarker +#[doc(hidden)] +#[derive(Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct BasePass<C> { + pub label: Option<String>, + + /// The stream of commands. + pub commands: Vec<C>, + + /// Dynamic offsets consumed by [`SetBindGroup`] commands in `commands`. + /// + /// Each successive `SetBindGroup` consumes the next + /// [`num_dynamic_offsets`] values from this list. + pub dynamic_offsets: Vec<wgt::DynamicOffset>, + + /// Strings used by debug instructions. + /// + /// Each successive [`PushDebugGroup`] or [`InsertDebugMarker`] + /// instruction consumes the next `len` bytes from this vector. + pub string_data: Vec<u8>, + + /// Data used by `SetPushConstant` instructions. + /// + /// See the documentation for [`RenderCommand::SetPushConstant`] + /// and [`ComputeCommand::SetPushConstant`] for details. + pub push_constant_data: Vec<u32>, +} + +impl<C: Clone> BasePass<C> { + fn new(label: &Label) -> Self { + Self { + label: label.as_ref().map(|cow| cow.to_string()), + commands: Vec::new(), + dynamic_offsets: Vec::new(), + string_data: Vec::new(), + push_constant_data: Vec::new(), + } + } + + #[cfg(feature = "trace")] + fn from_ref(base: BasePassRef<C>) -> Self { + Self { + label: base.label.map(str::to_string), + commands: base.commands.to_vec(), + dynamic_offsets: base.dynamic_offsets.to_vec(), + string_data: base.string_data.to_vec(), + push_constant_data: base.push_constant_data.to_vec(), + } + } + + pub fn as_ref(&self) -> BasePassRef<C> { + BasePassRef { + label: self.label.as_deref(), + commands: &self.commands, + dynamic_offsets: &self.dynamic_offsets, + string_data: &self.string_data, + push_constant_data: &self.push_constant_data, + } + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CommandEncoderError { + #[error("Command encoder is invalid")] + Invalid, + #[error("Command encoder must be active")] + NotRecording, + #[error(transparent)] + Device(#[from] DeviceError), +} + +impl Global { + pub fn command_encoder_finish<A: HalApi>( + &self, + encoder_id: id::CommandEncoderId, + _desc: &wgt::CommandBufferDescriptor<Label>, + ) -> (CommandBufferId, Option<CommandEncoderError>) { + profiling::scope!("CommandEncoder::finish"); + + let hub = A::hub(self); + + let error = match hub.command_buffers.get(encoder_id.transmute()) { + Ok(cmd_buf) => { + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + match cmd_buf_data.status { + CommandEncoderStatus::Recording => { + if let Err(e) = cmd_buf_data.encoder.close() { + Some(e.into()) + } else { + cmd_buf_data.status = CommandEncoderStatus::Finished; + //Note: if we want to stop tracking the swapchain texture view, + // this is the place to do it. + log::trace!("Command buffer {:?}", encoder_id); + None + } + } + CommandEncoderStatus::Finished => Some(CommandEncoderError::NotRecording), + CommandEncoderStatus::Error => { + cmd_buf_data.encoder.discard(); + Some(CommandEncoderError::Invalid) + } + } + } + Err(_) => Some(CommandEncoderError::Invalid), + }; + + (encoder_id.transmute(), error) + } + + pub fn command_encoder_push_debug_group<A: HalApi>( + &self, + encoder_id: id::CommandEncoderId, + label: &str, + ) -> Result<(), CommandEncoderError> { + profiling::scope!("CommandEncoder::push_debug_group"); + api_log!("CommandEncoder::push_debug_group {label}"); + + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id)?; + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::PushDebugGroup(label.to_string())); + } + + let cmd_buf_raw = cmd_buf_data.encoder.open()?; + if !self + .instance + .flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) + { + unsafe { + cmd_buf_raw.begin_debug_marker(label); + } + } + Ok(()) + } + + pub fn command_encoder_insert_debug_marker<A: HalApi>( + &self, + encoder_id: id::CommandEncoderId, + label: &str, + ) -> Result<(), CommandEncoderError> { + profiling::scope!("CommandEncoder::insert_debug_marker"); + api_log!("CommandEncoder::insert_debug_marker {label}"); + + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id)?; + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::InsertDebugMarker(label.to_string())); + } + + if !self + .instance + .flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) + { + let cmd_buf_raw = cmd_buf_data.encoder.open()?; + unsafe { + cmd_buf_raw.insert_debug_marker(label); + } + } + Ok(()) + } + + pub fn command_encoder_pop_debug_group<A: HalApi>( + &self, + encoder_id: id::CommandEncoderId, + ) -> Result<(), CommandEncoderError> { + profiling::scope!("CommandEncoder::pop_debug_marker"); + api_log!("CommandEncoder::pop_debug_group"); + + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id)?; + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::PopDebugGroup); + } + + let cmd_buf_raw = cmd_buf_data.encoder.open()?; + if !self + .instance + .flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) + { + unsafe { + cmd_buf_raw.end_debug_marker(); + } + } + Ok(()) + } +} + +fn push_constant_clear<PushFn>(offset: u32, size_bytes: u32, mut push_fn: PushFn) +where + PushFn: FnMut(u32, &[u32]), +{ + let mut count_words = 0_u32; + let size_words = size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT; + while count_words < size_words { + let count_bytes = count_words * wgt::PUSH_CONSTANT_ALIGNMENT; + let size_to_write_words = + (size_words - count_words).min(PUSH_CONSTANT_CLEAR_ARRAY.len() as u32); + + push_fn( + offset + count_bytes, + &PUSH_CONSTANT_CLEAR_ARRAY[0..size_to_write_words as usize], + ); + + count_words += size_to_write_words; + } +} + +#[derive(Debug, Copy, Clone)] +struct StateChange<T> { + last_state: Option<T>, +} + +impl<T: Copy + PartialEq> StateChange<T> { + fn new() -> Self { + Self { last_state: None } + } + fn set_and_check_redundant(&mut self, new_state: T) -> bool { + let already_set = self.last_state == Some(new_state); + self.last_state = Some(new_state); + already_set + } + fn reset(&mut self) { + self.last_state = None; + } +} + +impl<T: Copy + PartialEq> Default for StateChange<T> { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug)] +struct BindGroupStateChange { + last_states: [StateChange<id::BindGroupId>; hal::MAX_BIND_GROUPS], +} + +impl BindGroupStateChange { + fn new() -> Self { + Self { + last_states: [StateChange::new(); hal::MAX_BIND_GROUPS], + } + } + + unsafe fn set_and_check_redundant( + &mut self, + bind_group_id: id::BindGroupId, + index: u32, + dynamic_offsets: &mut Vec<u32>, + offsets: *const wgt::DynamicOffset, + offset_length: usize, + ) -> bool { + // For now never deduplicate bind groups with dynamic offsets. + if offset_length == 0 { + // If this get returns None, that means we're well over the limit, + // so let the call through to get a proper error + if let Some(current_bind_group) = self.last_states.get_mut(index as usize) { + // Bail out if we're binding the same bind group. + if current_bind_group.set_and_check_redundant(bind_group_id) { + return true; + } + } + } else { + // We intentionally remove the memory of this bind group if we have dynamic offsets, + // such that if you try to bind this bind group later with _no_ dynamic offsets it + // tries to bind it again and gives a proper validation error. + if let Some(current_bind_group) = self.last_states.get_mut(index as usize) { + current_bind_group.reset(); + } + dynamic_offsets + .extend_from_slice(unsafe { slice::from_raw_parts(offsets, offset_length) }); + } + false + } + fn reset(&mut self) { + self.last_states = [StateChange::new(); hal::MAX_BIND_GROUPS]; + } +} + +impl Default for BindGroupStateChange { + fn default() -> Self { + Self::new() + } +} + +trait MapPassErr<T, O> { + fn map_pass_err(self, scope: PassErrorScope) -> Result<T, O>; +} + +#[derive(Clone, Copy, Debug, Error)] +pub enum PassErrorScope { + #[error("In a bundle parameter")] + Bundle, + #[error("In a pass parameter")] + Pass(id::CommandEncoderId), + #[error("In a set_bind_group command")] + SetBindGroup(id::BindGroupId), + #[error("In a set_pipeline command")] + SetPipelineRender(id::RenderPipelineId), + #[error("In a set_pipeline command")] + SetPipelineCompute(id::ComputePipelineId), + #[error("In a set_push_constant command")] + SetPushConstant, + #[error("In a set_vertex_buffer command")] + SetVertexBuffer(id::BufferId), + #[error("In a set_index_buffer command")] + SetIndexBuffer(id::BufferId), + #[error("In a set_viewport command")] + SetViewport, + #[error("In a set_scissor_rect command")] + SetScissorRect, + #[error("In a draw command, indexed:{indexed} indirect:{indirect}")] + Draw { + indexed: bool, + indirect: bool, + pipeline: Option<id::RenderPipelineId>, + }, + #[error("While resetting queries after the renderpass was ran")] + QueryReset, + #[error("In a write_timestamp command")] + WriteTimestamp, + #[error("In a begin_occlusion_query command")] + BeginOcclusionQuery, + #[error("In a end_occlusion_query command")] + EndOcclusionQuery, + #[error("In a begin_pipeline_statistics_query command")] + BeginPipelineStatisticsQuery, + #[error("In a end_pipeline_statistics_query command")] + EndPipelineStatisticsQuery, + #[error("In a execute_bundle command")] + ExecuteBundle, + #[error("In a dispatch command, indirect:{indirect}")] + Dispatch { + indirect: bool, + pipeline: Option<id::ComputePipelineId>, + }, + #[error("In a pop_debug_group command")] + PopDebugGroup, +} + +impl PrettyError for PassErrorScope { + fn fmt_pretty(&self, fmt: &mut ErrorFormatter) { + // This error is not in the error chain, only notes are needed + match *self { + Self::Pass(id) => { + fmt.command_buffer_label(&id.transmute()); + } + Self::SetBindGroup(id) => { + fmt.bind_group_label(&id); + } + Self::SetPipelineRender(id) => { + fmt.render_pipeline_label(&id); + } + Self::SetPipelineCompute(id) => { + fmt.compute_pipeline_label(&id); + } + Self::SetVertexBuffer(id) => { + fmt.buffer_label(&id); + } + Self::SetIndexBuffer(id) => { + fmt.buffer_label(&id); + } + Self::Draw { + pipeline: Some(id), .. + } => { + fmt.render_pipeline_label(&id); + } + Self::Dispatch { + pipeline: Some(id), .. + } => { + fmt.compute_pipeline_label(&id); + } + _ => {} + } + } +} diff --git a/third_party/rust/wgpu-core/src/command/query.rs b/third_party/rust/wgpu-core/src/command/query.rs new file mode 100644 index 0000000000..39d7a9cc93 --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/query.rs @@ -0,0 +1,501 @@ +use hal::CommandEncoder as _; + +#[cfg(feature = "trace")] +use crate::device::trace::Command as TraceCommand; +use crate::{ + command::{CommandBuffer, CommandEncoderError}, + device::DeviceError, + global::Global, + hal_api::HalApi, + id::{self, Id}, + init_tracker::MemoryInitKind, + resource::QuerySet, + storage::Storage, + Epoch, FastHashMap, Index, +}; +use std::{iter, marker::PhantomData}; +use thiserror::Error; +use wgt::BufferAddress; + +#[derive(Debug)] +pub(crate) struct QueryResetMap<A: HalApi> { + map: FastHashMap<Index, (Vec<bool>, Epoch)>, + _phantom: PhantomData<A>, +} +impl<A: HalApi> QueryResetMap<A> { + pub fn new() -> Self { + Self { + map: FastHashMap::default(), + _phantom: PhantomData, + } + } + + pub fn use_query_set( + &mut self, + id: id::QuerySetId, + query_set: &QuerySet<A>, + query: u32, + ) -> bool { + let (index, epoch, _) = id.unzip(); + let vec_pair = self + .map + .entry(index) + .or_insert_with(|| (vec![false; query_set.desc.count as usize], epoch)); + + std::mem::replace(&mut vec_pair.0[query as usize], true) + } + + pub fn reset_queries( + &mut self, + raw_encoder: &mut A::CommandEncoder, + query_set_storage: &Storage<QuerySet<A>>, + backend: wgt::Backend, + ) -> Result<(), id::QuerySetId> { + for (query_set_id, (state, epoch)) in self.map.drain() { + let id = Id::zip(query_set_id, epoch, backend); + let query_set = query_set_storage.get(id).map_err(|_| id)?; + + debug_assert_eq!(state.len(), query_set.desc.count as usize); + + // Need to find all "runs" of values which need resets. If the state vector is: + // [false, true, true, false, true], we want to reset [1..3, 4..5]. This minimizes + // the amount of resets needed. + let mut run_start: Option<u32> = None; + for (idx, value) in state.into_iter().chain(iter::once(false)).enumerate() { + match (run_start, value) { + // We're inside of a run, do nothing + (Some(..), true) => {} + // We've hit the end of a run, dispatch a reset + (Some(start), false) => { + run_start = None; + unsafe { raw_encoder.reset_queries(query_set.raw(), start..idx as u32) }; + } + // We're starting a run + (None, true) => { + run_start = Some(idx as u32); + } + // We're in a run of falses, do nothing. + (None, false) => {} + } + } + } + + Ok(()) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum SimplifiedQueryType { + Occlusion, + Timestamp, + PipelineStatistics, +} +impl From<wgt::QueryType> for SimplifiedQueryType { + fn from(q: wgt::QueryType) -> Self { + match q { + wgt::QueryType::Occlusion => SimplifiedQueryType::Occlusion, + wgt::QueryType::Timestamp => SimplifiedQueryType::Timestamp, + wgt::QueryType::PipelineStatistics(..) => SimplifiedQueryType::PipelineStatistics, + } + } +} + +/// Error encountered when dealing with queries +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum QueryError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + Encoder(#[from] CommandEncoderError), + #[error("Error encountered while trying to use queries")] + Use(#[from] QueryUseError), + #[error("Error encountered while trying to resolve a query")] + Resolve(#[from] ResolveError), + #[error("Buffer {0:?} is invalid or destroyed")] + InvalidBuffer(id::BufferId), + #[error("QuerySet {0:?} is invalid or destroyed")] + InvalidQuerySet(id::QuerySetId), +} + +impl crate::error::PrettyError for QueryError { + fn fmt_pretty(&self, fmt: &mut crate::error::ErrorFormatter) { + fmt.error(self); + match *self { + Self::InvalidBuffer(id) => fmt.buffer_label(&id), + Self::InvalidQuerySet(id) => fmt.query_set_label(&id), + + _ => {} + } + } +} + +/// Error encountered while trying to use queries +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum QueryUseError { + #[error("Query {query_index} is out of bounds for a query set of size {query_set_size}")] + OutOfBounds { + query_index: u32, + query_set_size: u32, + }, + #[error("Query {query_index} has already been used within the same renderpass. Queries must only be used once per renderpass")] + UsedTwiceInsideRenderpass { query_index: u32 }, + #[error("Query {new_query_index} was started while query {active_query_index} was already active. No more than one statistic or occlusion query may be active at once")] + AlreadyStarted { + active_query_index: u32, + new_query_index: u32, + }, + #[error("Query was stopped while there was no active query")] + AlreadyStopped, + #[error("A query of type {query_type:?} was started using a query set of type {set_type:?}")] + IncompatibleType { + set_type: SimplifiedQueryType, + query_type: SimplifiedQueryType, + }, +} + +/// Error encountered while trying to resolve a query. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum ResolveError { + #[error("Queries can only be resolved to buffers that contain the QUERY_RESOLVE usage")] + MissingBufferUsage, + #[error("Resolve buffer offset has to be aligned to `QUERY_RESOLVE_BUFFER_ALIGNMENT")] + BufferOffsetAlignment, + #[error("Resolving queries {start_query}..{end_query} would overrun the query set of size {query_set_size}")] + QueryOverrun { + start_query: u32, + end_query: u32, + query_set_size: u32, + }, + #[error("Resolving queries {start_query}..{end_query} ({stride} byte queries) will end up overrunning the bounds of the destination buffer of size {buffer_size} using offsets {buffer_start_offset}..{buffer_end_offset}")] + BufferOverrun { + start_query: u32, + end_query: u32, + stride: u32, + buffer_size: BufferAddress, + buffer_start_offset: BufferAddress, + buffer_end_offset: BufferAddress, + }, +} + +impl<A: HalApi> QuerySet<A> { + fn validate_query( + &self, + query_set_id: id::QuerySetId, + query_type: SimplifiedQueryType, + query_index: u32, + reset_state: Option<&mut QueryResetMap<A>>, + ) -> Result<&A::QuerySet, QueryUseError> { + // We need to defer our resets because we are in a renderpass, + // add the usage to the reset map. + if let Some(reset) = reset_state { + let used = reset.use_query_set(query_set_id, self, query_index); + if used { + return Err(QueryUseError::UsedTwiceInsideRenderpass { query_index }); + } + } + + let simple_set_type = SimplifiedQueryType::from(self.desc.ty); + if simple_set_type != query_type { + return Err(QueryUseError::IncompatibleType { + query_type, + set_type: simple_set_type, + }); + } + + if query_index >= self.desc.count { + return Err(QueryUseError::OutOfBounds { + query_index, + query_set_size: self.desc.count, + }); + } + + Ok(self.raw()) + } + + pub(super) fn validate_and_write_timestamp( + &self, + raw_encoder: &mut A::CommandEncoder, + query_set_id: id::QuerySetId, + query_index: u32, + reset_state: Option<&mut QueryResetMap<A>>, + ) -> Result<(), QueryUseError> { + let needs_reset = reset_state.is_none(); + let query_set = self.validate_query( + query_set_id, + SimplifiedQueryType::Timestamp, + query_index, + reset_state, + )?; + + unsafe { + // If we don't have a reset state tracker which can defer resets, we must reset now. + if needs_reset { + raw_encoder.reset_queries(self.raw(), query_index..(query_index + 1)); + } + raw_encoder.write_timestamp(query_set, query_index); + } + + Ok(()) + } + + pub(super) fn validate_and_begin_occlusion_query( + &self, + raw_encoder: &mut A::CommandEncoder, + query_set_id: id::QuerySetId, + query_index: u32, + reset_state: Option<&mut QueryResetMap<A>>, + active_query: &mut Option<(id::QuerySetId, u32)>, + ) -> Result<(), QueryUseError> { + let needs_reset = reset_state.is_none(); + let query_set = self.validate_query( + query_set_id, + SimplifiedQueryType::Occlusion, + query_index, + reset_state, + )?; + + if let Some((_old_id, old_idx)) = active_query.replace((query_set_id, query_index)) { + return Err(QueryUseError::AlreadyStarted { + active_query_index: old_idx, + new_query_index: query_index, + }); + } + + unsafe { + // If we don't have a reset state tracker which can defer resets, we must reset now. + if needs_reset { + raw_encoder + .reset_queries(self.raw.as_ref().unwrap(), query_index..(query_index + 1)); + } + raw_encoder.begin_query(query_set, query_index); + } + + Ok(()) + } + + pub(super) fn validate_and_begin_pipeline_statistics_query( + &self, + raw_encoder: &mut A::CommandEncoder, + query_set_id: id::QuerySetId, + query_index: u32, + reset_state: Option<&mut QueryResetMap<A>>, + active_query: &mut Option<(id::QuerySetId, u32)>, + ) -> Result<(), QueryUseError> { + let needs_reset = reset_state.is_none(); + let query_set = self.validate_query( + query_set_id, + SimplifiedQueryType::PipelineStatistics, + query_index, + reset_state, + )?; + + if let Some((_old_id, old_idx)) = active_query.replace((query_set_id, query_index)) { + return Err(QueryUseError::AlreadyStarted { + active_query_index: old_idx, + new_query_index: query_index, + }); + } + + unsafe { + // If we don't have a reset state tracker which can defer resets, we must reset now. + if needs_reset { + raw_encoder.reset_queries(self.raw(), query_index..(query_index + 1)); + } + raw_encoder.begin_query(query_set, query_index); + } + + Ok(()) + } +} + +pub(super) fn end_occlusion_query<A: HalApi>( + raw_encoder: &mut A::CommandEncoder, + storage: &Storage<QuerySet<A>>, + active_query: &mut Option<(id::QuerySetId, u32)>, +) -> Result<(), QueryUseError> { + if let Some((query_set_id, query_index)) = active_query.take() { + // We can unwrap here as the validity was validated when the active query was set + let query_set = storage.get(query_set_id).unwrap(); + + unsafe { raw_encoder.end_query(query_set.raw.as_ref().unwrap(), query_index) }; + + Ok(()) + } else { + Err(QueryUseError::AlreadyStopped) + } +} + +pub(super) fn end_pipeline_statistics_query<A: HalApi>( + raw_encoder: &mut A::CommandEncoder, + storage: &Storage<QuerySet<A>>, + active_query: &mut Option<(id::QuerySetId, u32)>, +) -> Result<(), QueryUseError> { + if let Some((query_set_id, query_index)) = active_query.take() { + // We can unwrap here as the validity was validated when the active query was set + let query_set = storage.get(query_set_id).unwrap(); + + unsafe { raw_encoder.end_query(query_set.raw(), query_index) }; + + Ok(()) + } else { + Err(QueryUseError::AlreadyStopped) + } +} + +impl Global { + pub fn command_encoder_write_timestamp<A: HalApi>( + &self, + command_encoder_id: id::CommandEncoderId, + query_set_id: id::QuerySetId, + query_index: u32, + ) -> Result<(), QueryError> { + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?; + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::WriteTimestamp { + query_set_id, + query_index, + }); + } + + let encoder = &mut cmd_buf_data.encoder; + let tracker = &mut cmd_buf_data.trackers; + + let raw_encoder = encoder.open()?; + + let query_set_guard = hub.query_sets.read(); + let query_set = tracker + .query_sets + .add_single(&*query_set_guard, query_set_id) + .ok_or(QueryError::InvalidQuerySet(query_set_id))?; + + query_set.validate_and_write_timestamp(raw_encoder, query_set_id, query_index, None)?; + + Ok(()) + } + + pub fn command_encoder_resolve_query_set<A: HalApi>( + &self, + command_encoder_id: id::CommandEncoderId, + query_set_id: id::QuerySetId, + start_query: u32, + query_count: u32, + destination: id::BufferId, + destination_offset: BufferAddress, + ) -> Result<(), QueryError> { + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?; + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::ResolveQuerySet { + query_set_id, + start_query, + query_count, + destination, + destination_offset, + }); + } + + let encoder = &mut cmd_buf_data.encoder; + let tracker = &mut cmd_buf_data.trackers; + let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions; + let raw_encoder = encoder.open()?; + + if destination_offset % wgt::QUERY_RESOLVE_BUFFER_ALIGNMENT != 0 { + return Err(QueryError::Resolve(ResolveError::BufferOffsetAlignment)); + } + let query_set_guard = hub.query_sets.read(); + let query_set = tracker + .query_sets + .add_single(&*query_set_guard, query_set_id) + .ok_or(QueryError::InvalidQuerySet(query_set_id))?; + + let (dst_buffer, dst_pending) = { + let buffer_guard = hub.buffers.read(); + let dst_buffer = buffer_guard + .get(destination) + .map_err(|_| QueryError::InvalidBuffer(destination))?; + tracker + .buffers + .set_single(dst_buffer, hal::BufferUses::COPY_DST) + .ok_or(QueryError::InvalidBuffer(destination))? + }; + + let snatch_guard = dst_buffer.device.snatchable_lock.read(); + + let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard)); + + if !dst_buffer.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) { + return Err(ResolveError::MissingBufferUsage.into()); + } + + let end_query = start_query + query_count; + if end_query > query_set.desc.count { + return Err(ResolveError::QueryOverrun { + start_query, + end_query, + query_set_size: query_set.desc.count, + } + .into()); + } + + let elements_per_query = match query_set.desc.ty { + wgt::QueryType::Occlusion => 1, + wgt::QueryType::PipelineStatistics(ps) => ps.bits().count_ones(), + wgt::QueryType::Timestamp => 1, + }; + let stride = elements_per_query * wgt::QUERY_SIZE; + let bytes_used = (stride * query_count) as BufferAddress; + + let buffer_start_offset = destination_offset; + let buffer_end_offset = buffer_start_offset + bytes_used; + + if buffer_end_offset > dst_buffer.size { + return Err(ResolveError::BufferOverrun { + start_query, + end_query, + stride, + buffer_size: dst_buffer.size, + buffer_start_offset, + buffer_end_offset, + } + .into()); + } + + // TODO(https://github.com/gfx-rs/wgpu/issues/3993): Need to track initialization state. + buffer_memory_init_actions.extend(dst_buffer.initialization_status.read().create_action( + &dst_buffer, + buffer_start_offset..buffer_end_offset, + MemoryInitKind::ImplicitlyInitialized, + )); + + let raw_dst_buffer = dst_buffer + .raw(&snatch_guard) + .ok_or(QueryError::InvalidBuffer(destination))?; + + unsafe { + raw_encoder.transition_buffers(dst_barrier.into_iter()); + raw_encoder.copy_query_results( + query_set.raw(), + start_query..end_query, + raw_dst_buffer, + destination_offset, + wgt::BufferSize::new_unchecked(stride as u64), + ); + } + + Ok(()) + } +} diff --git a/third_party/rust/wgpu-core/src/command/render.rs b/third_party/rust/wgpu-core/src/command/render.rs new file mode 100644 index 0000000000..d3de3e26e1 --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/render.rs @@ -0,0 +1,2866 @@ +use crate::resource::Resource; +use crate::snatch::SnatchGuard; +use crate::{ + api_log, + binding_model::BindError, + command::{ + self, + bind::Binder, + end_occlusion_query, end_pipeline_statistics_query, + memory_init::{fixup_discarded_surfaces, SurfacesInDiscardState}, + BasePass, BasePassRef, BindGroupStateChange, CommandBuffer, CommandEncoderError, + CommandEncoderStatus, DrawError, ExecutionError, MapPassErr, PassErrorScope, QueryUseError, + RenderCommand, RenderCommandError, StateChange, + }, + device::{ + AttachmentData, Device, DeviceError, MissingDownlevelFlags, MissingFeatures, + RenderPassCompatibilityCheckType, RenderPassCompatibilityError, RenderPassContext, + }, + error::{ErrorFormatter, PrettyError}, + global::Global, + hal_api::HalApi, + hal_label, id, + init_tracker::{MemoryInitKind, TextureInitRange, TextureInitTrackerAction}, + pipeline::{self, PipelineFlags}, + resource::{Buffer, QuerySet, Texture, TextureView, TextureViewNotRenderableReason}, + storage::Storage, + track::{TextureSelector, Tracker, UsageConflict, UsageScope}, + validation::{ + check_buffer_usage, check_texture_usage, MissingBufferUsageError, MissingTextureUsageError, + }, + Label, +}; + +use arrayvec::ArrayVec; +use hal::CommandEncoder as _; +use thiserror::Error; +use wgt::{ + BufferAddress, BufferSize, BufferUsages, Color, IndexFormat, TextureUsages, + TextureViewDimension, VertexStepMode, +}; + +#[cfg(feature = "serde")] +use serde::Deserialize; +#[cfg(feature = "serde")] +use serde::Serialize; + +use std::sync::Arc; +use std::{borrow::Cow, fmt, iter, marker::PhantomData, mem, num::NonZeroU32, ops::Range, str}; + +use super::{ + memory_init::TextureSurfaceDiscard, CommandBufferTextureMemoryActions, CommandEncoder, + QueryResetMap, +}; + +/// Operation to perform to the output attachment at the start of a renderpass. +#[repr(C)] +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "kebab-case"))] +pub enum LoadOp { + /// Clear the output attachment with the clear color. Clearing is faster than loading. + Clear = 0, + /// Do not clear output attachment. + Load = 1, +} + +/// Operation to perform to the output attachment at the end of a renderpass. +#[repr(C)] +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "kebab-case"))] +pub enum StoreOp { + /// Discards the content of the render target. + /// + /// If you don't care about the contents of the target, this can be faster. + Discard = 0, + /// Store the result of the renderpass. + Store = 1, +} + +/// Describes an individual channel within a render pass, such as color, depth, or stencil. +#[repr(C)] +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct PassChannel<V> { + /// Operation to perform to the output attachment at the start of a + /// renderpass. + /// + /// This must be clear if it is the first renderpass rendering to a swap + /// chain image. + pub load_op: LoadOp, + /// Operation to perform to the output attachment at the end of a renderpass. + pub store_op: StoreOp, + /// If load_op is [`LoadOp::Clear`], the attachment will be cleared to this + /// color. + pub clear_value: V, + /// If true, the relevant channel is not changed by a renderpass, and the + /// corresponding attachment can be used inside the pass by other read-only + /// usages. + pub read_only: bool, +} + +impl<V> PassChannel<V> { + fn hal_ops(&self) -> hal::AttachmentOps { + let mut ops = hal::AttachmentOps::empty(); + match self.load_op { + LoadOp::Load => ops |= hal::AttachmentOps::LOAD, + LoadOp::Clear => (), + }; + match self.store_op { + StoreOp::Store => ops |= hal::AttachmentOps::STORE, + StoreOp::Discard => (), + }; + ops + } +} + +/// Describes a color attachment to a render pass. +#[repr(C)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct RenderPassColorAttachment { + /// The view to use as an attachment. + pub view: id::TextureViewId, + /// The view that will receive the resolved output if multisampling is used. + pub resolve_target: Option<id::TextureViewId>, + /// What operations will be performed on this color attachment. + pub channel: PassChannel<Color>, +} + +/// Describes a depth/stencil attachment to a render pass. +#[repr(C)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct RenderPassDepthStencilAttachment { + /// The view to use as an attachment. + pub view: id::TextureViewId, + /// What operations will be performed on the depth part of the attachment. + pub depth: PassChannel<f32>, + /// What operations will be performed on the stencil part of the attachment. + pub stencil: PassChannel<u32>, +} + +impl RenderPassDepthStencilAttachment { + /// Validate the given aspects' read-only flags against their load + /// and store ops. + /// + /// When an aspect is read-only, its load and store ops must be + /// `LoadOp::Load` and `StoreOp::Store`. + /// + /// On success, return a pair `(depth, stencil)` indicating + /// whether the depth and stencil passes are read-only. + fn depth_stencil_read_only( + &self, + aspects: hal::FormatAspects, + ) -> Result<(bool, bool), RenderPassErrorInner> { + let mut depth_read_only = true; + let mut stencil_read_only = true; + + if aspects.contains(hal::FormatAspects::DEPTH) { + if self.depth.read_only + && (self.depth.load_op, self.depth.store_op) != (LoadOp::Load, StoreOp::Store) + { + return Err(RenderPassErrorInner::InvalidDepthOps); + } + depth_read_only = self.depth.read_only; + } + + if aspects.contains(hal::FormatAspects::STENCIL) { + if self.stencil.read_only + && (self.stencil.load_op, self.stencil.store_op) != (LoadOp::Load, StoreOp::Store) + { + return Err(RenderPassErrorInner::InvalidStencilOps); + } + stencil_read_only = self.stencil.read_only; + } + + Ok((depth_read_only, stencil_read_only)) + } +} + +/// Location to write a timestamp to (beginning or end of the pass). +#[repr(C)] +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "kebab-case"))] +pub enum RenderPassTimestampLocation { + Beginning = 0, + End = 1, +} + +/// Describes the writing of timestamp values in a render pass. +#[repr(C)] +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct RenderPassTimestampWrites { + /// The query set to write the timestamp to. + pub query_set: id::QuerySetId, + /// The index of the query set at which a start timestamp of this pass is written, if any. + pub beginning_of_pass_write_index: Option<u32>, + /// The index of the query set at which an end timestamp of this pass is written, if any. + pub end_of_pass_write_index: Option<u32>, +} + +/// Describes the attachments of a render pass. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct RenderPassDescriptor<'a> { + pub label: Label<'a>, + /// The color attachments of the render pass. + pub color_attachments: Cow<'a, [Option<RenderPassColorAttachment>]>, + /// The depth and stencil attachment of the render pass, if any. + pub depth_stencil_attachment: Option<&'a RenderPassDepthStencilAttachment>, + /// Defines where and when timestamp values will be written for this pass. + pub timestamp_writes: Option<&'a RenderPassTimestampWrites>, + /// Defines where the occlusion query results will be stored for this pass. + pub occlusion_query_set: Option<id::QuerySetId>, +} + +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct RenderPass { + base: BasePass<RenderCommand>, + parent_id: id::CommandEncoderId, + color_targets: ArrayVec<Option<RenderPassColorAttachment>, { hal::MAX_COLOR_ATTACHMENTS }>, + depth_stencil_target: Option<RenderPassDepthStencilAttachment>, + timestamp_writes: Option<RenderPassTimestampWrites>, + occlusion_query_set_id: Option<id::QuerySetId>, + + // Resource binding dedupe state. + #[cfg_attr(feature = "serde", serde(skip))] + current_bind_groups: BindGroupStateChange, + #[cfg_attr(feature = "serde", serde(skip))] + current_pipeline: StateChange<id::RenderPipelineId>, +} + +impl RenderPass { + pub fn new(parent_id: id::CommandEncoderId, desc: &RenderPassDescriptor) -> Self { + Self { + base: BasePass::new(&desc.label), + parent_id, + color_targets: desc.color_attachments.iter().cloned().collect(), + depth_stencil_target: desc.depth_stencil_attachment.cloned(), + timestamp_writes: desc.timestamp_writes.cloned(), + occlusion_query_set_id: desc.occlusion_query_set, + + current_bind_groups: BindGroupStateChange::new(), + current_pipeline: StateChange::new(), + } + } + + pub fn parent_id(&self) -> id::CommandEncoderId { + self.parent_id + } + + #[cfg(feature = "trace")] + pub fn into_command(self) -> crate::device::trace::Command { + crate::device::trace::Command::RunRenderPass { + base: self.base, + target_colors: self.color_targets.into_iter().collect(), + target_depth_stencil: self.depth_stencil_target, + timestamp_writes: self.timestamp_writes, + occlusion_query_set_id: self.occlusion_query_set_id, + } + } + + pub fn set_index_buffer( + &mut self, + buffer_id: id::BufferId, + index_format: IndexFormat, + offset: BufferAddress, + size: Option<BufferSize>, + ) { + self.base.commands.push(RenderCommand::SetIndexBuffer { + buffer_id, + index_format, + offset, + size, + }); + } +} + +impl fmt::Debug for RenderPass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RenderPass") + .field("encoder_id", &self.parent_id) + .field("color_targets", &self.color_targets) + .field("depth_stencil_target", &self.depth_stencil_target) + .field("command count", &self.base.commands.len()) + .field("dynamic offset count", &self.base.dynamic_offsets.len()) + .field( + "push constant u32 count", + &self.base.push_constant_data.len(), + ) + .finish() + } +} + +#[derive(Debug, PartialEq)] +enum OptionalState { + Unused, + Required, + Set, +} + +impl OptionalState { + fn require(&mut self, require: bool) { + if require && *self == Self::Unused { + *self = Self::Required; + } + } +} + +#[derive(Debug, Default)] +struct IndexState { + bound_buffer_view: Option<(id::BufferId, Range<BufferAddress>)>, + format: Option<IndexFormat>, + pipeline_format: Option<IndexFormat>, + limit: u64, +} + +impl IndexState { + fn update_limit(&mut self) { + self.limit = match self.bound_buffer_view { + Some((_, ref range)) => { + let format = self + .format + .expect("IndexState::update_limit must be called after a index buffer is set"); + let shift = match format { + IndexFormat::Uint16 => 1, + IndexFormat::Uint32 => 2, + }; + + (range.end - range.start) >> shift + } + None => 0, + } + } + + fn reset(&mut self) { + self.bound_buffer_view = None; + self.limit = 0; + } +} + +#[derive(Clone, Copy, Debug)] +struct VertexBufferState { + total_size: BufferAddress, + step: pipeline::VertexStep, + bound: bool, +} + +impl VertexBufferState { + const EMPTY: Self = Self { + total_size: 0, + step: pipeline::VertexStep { + stride: 0, + last_stride: 0, + mode: VertexStepMode::Vertex, + }, + bound: false, + }; +} + +#[derive(Debug, Default)] +struct VertexState { + inputs: ArrayVec<VertexBufferState, { hal::MAX_VERTEX_BUFFERS }>, + /// Length of the shortest vertex rate vertex buffer + vertex_limit: u64, + /// Buffer slot which the shortest vertex rate vertex buffer is bound to + vertex_limit_slot: u32, + /// Length of the shortest instance rate vertex buffer + instance_limit: u64, + /// Buffer slot which the shortest instance rate vertex buffer is bound to + instance_limit_slot: u32, + /// Total amount of buffers required by the pipeline. + buffers_required: u32, +} + +impl VertexState { + fn update_limits(&mut self) { + // Implements the validation from https://gpuweb.github.io/gpuweb/#dom-gpurendercommandsmixin-draw + // Except that the formula is shuffled to extract the number of vertices in order + // to carry the bulk of the computation when changing states instead of when producing + // draws. Draw calls tend to happen at a higher frequency. Here we determine vertex + // limits that can be cheaply checked for each draw call. + self.vertex_limit = u32::MAX as u64; + self.instance_limit = u32::MAX as u64; + for (idx, vbs) in self.inputs.iter().enumerate() { + if !vbs.bound { + continue; + } + + let limit = if vbs.total_size < vbs.step.last_stride { + // The buffer cannot fit the last vertex. + 0 + } else { + if vbs.step.stride == 0 { + // We already checked that the last stride fits, the same + // vertex will be repeated so this slot can accommodate any number of + // vertices. + continue; + } + + // The general case. + (vbs.total_size - vbs.step.last_stride) / vbs.step.stride + 1 + }; + + match vbs.step.mode { + VertexStepMode::Vertex => { + if limit < self.vertex_limit { + self.vertex_limit = limit; + self.vertex_limit_slot = idx as _; + } + } + VertexStepMode::Instance => { + if limit < self.instance_limit { + self.instance_limit = limit; + self.instance_limit_slot = idx as _; + } + } + } + } + } + + fn reset(&mut self) { + self.inputs.clear(); + self.vertex_limit = 0; + self.instance_limit = 0; + } +} + +#[derive(Debug)] +struct State<A: HalApi> { + pipeline_flags: PipelineFlags, + binder: Binder<A>, + blend_constant: OptionalState, + stencil_reference: u32, + pipeline: Option<id::RenderPipelineId>, + index: IndexState, + vertex: VertexState, + debug_scope_depth: u32, +} + +impl<A: HalApi> State<A> { + fn is_ready(&self, indexed: bool) -> Result<(), DrawError> { + // Determine how many vertex buffers have already been bound + let vertex_buffer_count = self.vertex.inputs.iter().take_while(|v| v.bound).count() as u32; + // Compare with the needed quantity + if vertex_buffer_count < self.vertex.buffers_required { + return Err(DrawError::MissingVertexBuffer { + index: vertex_buffer_count, + }); + } + + let bind_mask = self.binder.invalid_mask(); + if bind_mask != 0 { + //let (expected, provided) = self.binder.entries[index as usize].info(); + return Err(DrawError::IncompatibleBindGroup { + index: bind_mask.trailing_zeros(), + diff: self.binder.bgl_diff(), + }); + } + if self.pipeline.is_none() { + return Err(DrawError::MissingPipeline); + } + if self.blend_constant == OptionalState::Required { + return Err(DrawError::MissingBlendConstant); + } + + if indexed { + // Pipeline expects an index buffer + if let Some(pipeline_index_format) = self.index.pipeline_format { + // We have a buffer bound + let buffer_index_format = self.index.format.ok_or(DrawError::MissingIndexBuffer)?; + + // The buffers are different formats + if pipeline_index_format != buffer_index_format { + return Err(DrawError::UnmatchedIndexFormats { + pipeline: pipeline_index_format, + buffer: buffer_index_format, + }); + } + } + } + + self.binder.check_late_buffer_bindings()?; + + Ok(()) + } + + /// Reset the `RenderBundle`-related states. + fn reset_bundle(&mut self) { + self.binder.reset(); + self.pipeline = None; + self.index.reset(); + self.vertex.reset(); + } +} + +/// Describes an attachment location in words. +/// +/// Can be used as "the {loc} has..." or "{loc} has..." +#[derive(Debug, Copy, Clone)] +pub enum AttachmentErrorLocation { + Color { index: usize, resolve: bool }, + Depth, +} + +impl fmt::Display for AttachmentErrorLocation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + AttachmentErrorLocation::Color { + index, + resolve: false, + } => write!(f, "color attachment at index {index}'s texture view"), + AttachmentErrorLocation::Color { + index, + resolve: true, + } => write!( + f, + "color attachment at index {index}'s resolve texture view" + ), + AttachmentErrorLocation::Depth => write!(f, "depth attachment's texture view"), + } + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum ColorAttachmentError { + #[error("Attachment format {0:?} is not a color format")] + InvalidFormat(wgt::TextureFormat), + #[error("The number of color attachments {given} exceeds the limit {limit}")] + TooMany { given: usize, limit: usize }, +} + +/// Error encountered when performing a render pass. +#[derive(Clone, Debug, Error)] +pub enum RenderPassErrorInner { + #[error(transparent)] + Device(DeviceError), + #[error(transparent)] + ColorAttachment(#[from] ColorAttachmentError), + #[error(transparent)] + Encoder(#[from] CommandEncoderError), + #[error("Attachment texture view {0:?} is invalid")] + InvalidAttachment(id::TextureViewId), + #[error("Attachment texture view {0:?} is invalid")] + InvalidResolveTarget(id::TextureViewId), + #[error("The format of the depth-stencil attachment ({0:?}) is not a depth-stencil format")] + InvalidDepthStencilAttachmentFormat(wgt::TextureFormat), + #[error("The format of the {location} ({format:?}) is not resolvable")] + UnsupportedResolveTargetFormat { + location: AttachmentErrorLocation, + format: wgt::TextureFormat, + }, + #[error("No color attachments or depth attachments were provided, at least one attachment of any kind must be provided")] + MissingAttachments, + #[error("The {location} is not renderable:")] + TextureViewIsNotRenderable { + location: AttachmentErrorLocation, + #[source] + reason: TextureViewNotRenderableReason, + }, + #[error("Attachments have differing sizes: the {expected_location} has extent {expected_extent:?} but is followed by the {actual_location} which has {actual_extent:?}")] + AttachmentsDimensionMismatch { + expected_location: AttachmentErrorLocation, + expected_extent: wgt::Extent3d, + actual_location: AttachmentErrorLocation, + actual_extent: wgt::Extent3d, + }, + #[error("Attachments have differing sample counts: the {expected_location} has count {expected_samples:?} but is followed by the {actual_location} which has count {actual_samples:?}")] + AttachmentSampleCountMismatch { + expected_location: AttachmentErrorLocation, + expected_samples: u32, + actual_location: AttachmentErrorLocation, + actual_samples: u32, + }, + #[error("The resolve source, {location}, must be multi-sampled (has {src} samples) while the resolve destination must not be multisampled (has {dst} samples)")] + InvalidResolveSampleCounts { + location: AttachmentErrorLocation, + src: u32, + dst: u32, + }, + #[error( + "Resource source, {location}, format ({src:?}) must match the resolve destination format ({dst:?})" + )] + MismatchedResolveTextureFormat { + location: AttachmentErrorLocation, + src: wgt::TextureFormat, + dst: wgt::TextureFormat, + }, + #[error("Surface texture is dropped before the render pass is finished")] + SurfaceTextureDropped, + #[error("Not enough memory left for render pass")] + OutOfMemory, + #[error("The bind group at index {0:?} is invalid")] + InvalidBindGroup(usize), + #[error("Unable to clear non-present/read-only depth")] + InvalidDepthOps, + #[error("Unable to clear non-present/read-only stencil")] + InvalidStencilOps, + #[error("Setting `values_offset` to be `None` is only for internal use in render bundles")] + InvalidValuesOffset, + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), + #[error("Indirect draw uses bytes {offset}..{end_offset} {} which overruns indirect buffer of size {buffer_size}", + count.map_or_else(String::new, |v| format!("(using count {v})")))] + IndirectBufferOverrun { + count: Option<NonZeroU32>, + offset: u64, + end_offset: u64, + buffer_size: u64, + }, + #[error("Indirect draw uses bytes {begin_count_offset}..{end_count_offset} which overruns indirect buffer of size {count_buffer_size}")] + IndirectCountBufferOverrun { + begin_count_offset: u64, + end_count_offset: u64, + count_buffer_size: u64, + }, + #[error("Cannot pop debug group, because number of pushed debug groups is zero")] + InvalidPopDebugGroup, + #[error(transparent)] + ResourceUsageConflict(#[from] UsageConflict), + #[error("Render bundle has incompatible targets, {0}")] + IncompatibleBundleTargets(#[from] RenderPassCompatibilityError), + #[error( + "Render bundle has incompatible read-only flags: \ + bundle has flags depth = {bundle_depth} and stencil = {bundle_stencil}, \ + while the pass has flags depth = {pass_depth} and stencil = {pass_stencil}. \ + Read-only renderpasses are only compatible with read-only bundles for that aspect." + )] + IncompatibleBundleReadOnlyDepthStencil { + pass_depth: bool, + pass_stencil: bool, + bundle_depth: bool, + bundle_stencil: bool, + }, + #[error(transparent)] + RenderCommand(#[from] RenderCommandError), + #[error(transparent)] + Draw(#[from] DrawError), + #[error(transparent)] + Bind(#[from] BindError), + #[error(transparent)] + QueryUse(#[from] QueryUseError), + #[error("Multiview layer count must match")] + MultiViewMismatch, + #[error( + "Multiview pass texture views with more than one array layer must have D2Array dimension" + )] + MultiViewDimensionMismatch, + #[error("QuerySet {0:?} is invalid")] + InvalidQuerySet(id::QuerySetId), + #[error("missing occlusion query set")] + MissingOcclusionQuerySet, +} + +impl PrettyError for RenderPassErrorInner { + fn fmt_pretty(&self, fmt: &mut ErrorFormatter) { + fmt.error(self); + if let Self::InvalidAttachment(id) = *self { + fmt.texture_view_label_with_key(&id, "attachment"); + }; + if let Self::Draw(DrawError::IncompatibleBindGroup { diff, .. }) = self { + for d in diff { + fmt.note(&d); + } + }; + } +} + +impl From<MissingBufferUsageError> for RenderPassErrorInner { + fn from(error: MissingBufferUsageError) -> Self { + Self::RenderCommand(error.into()) + } +} + +impl From<MissingTextureUsageError> for RenderPassErrorInner { + fn from(error: MissingTextureUsageError) -> Self { + Self::RenderCommand(error.into()) + } +} + +impl From<DeviceError> for RenderPassErrorInner { + fn from(error: DeviceError) -> Self { + Self::Device(error) + } +} + +/// Error encountered when performing a render pass. +#[derive(Clone, Debug, Error)] +#[error("{scope}")] +pub struct RenderPassError { + pub scope: PassErrorScope, + #[source] + inner: RenderPassErrorInner, +} +impl PrettyError for RenderPassError { + fn fmt_pretty(&self, fmt: &mut ErrorFormatter) { + // This error is wrapper for the inner error, + // but the scope has useful labels + fmt.error(self); + self.scope.fmt_pretty(fmt); + } +} + +impl<T, E> MapPassErr<T, RenderPassError> for Result<T, E> +where + E: Into<RenderPassErrorInner>, +{ + fn map_pass_err(self, scope: PassErrorScope) -> Result<T, RenderPassError> { + self.map_err(|inner| RenderPassError { + scope, + inner: inner.into(), + }) + } +} + +struct RenderAttachment<'a, A: HalApi> { + texture: Arc<Texture<A>>, + selector: &'a TextureSelector, + usage: hal::TextureUses, +} + +impl<A: HalApi> TextureView<A> { + fn to_render_attachment(&self, usage: hal::TextureUses) -> RenderAttachment<A> { + RenderAttachment { + texture: self.parent.clone(), + selector: &self.selector, + usage, + } + } +} + +const MAX_TOTAL_ATTACHMENTS: usize = hal::MAX_COLOR_ATTACHMENTS + hal::MAX_COLOR_ATTACHMENTS + 1; +type AttachmentDataVec<T> = ArrayVec<T, MAX_TOTAL_ATTACHMENTS>; + +struct RenderPassInfo<'a, A: HalApi> { + context: RenderPassContext, + usage_scope: UsageScope<A>, + /// All render attachments, including depth/stencil + render_attachments: AttachmentDataVec<RenderAttachment<'a, A>>, + is_depth_read_only: bool, + is_stencil_read_only: bool, + extent: wgt::Extent3d, + _phantom: PhantomData<A>, + + pending_discard_init_fixups: SurfacesInDiscardState<A>, + divergent_discarded_depth_stencil_aspect: Option<(wgt::TextureAspect, &'a TextureView<A>)>, + multiview: Option<NonZeroU32>, +} + +impl<'a, A: HalApi> RenderPassInfo<'a, A> { + fn add_pass_texture_init_actions<V>( + channel: &PassChannel<V>, + texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>, + view: &TextureView<A>, + pending_discard_init_fixups: &mut SurfacesInDiscardState<A>, + ) { + if channel.load_op == LoadOp::Load { + pending_discard_init_fixups.extend(texture_memory_actions.register_init_action( + &TextureInitTrackerAction { + texture: view.parent.clone(), + range: TextureInitRange::from(view.selector.clone()), + // Note that this is needed even if the target is discarded, + kind: MemoryInitKind::NeedsInitializedMemory, + }, + )); + } else if channel.store_op == StoreOp::Store { + // Clear + Store + texture_memory_actions.register_implicit_init( + &view.parent, + TextureInitRange::from(view.selector.clone()), + ); + } + if channel.store_op == StoreOp::Discard { + // the discard happens at the *end* of a pass, but recording the + // discard right away be alright since the texture can't be used + // during the pass anyways + texture_memory_actions.discard(TextureSurfaceDiscard { + texture: view.parent.clone(), + mip_level: view.selector.mips.start, + layer: view.selector.layers.start, + }); + } + } + + fn start( + device: &Device<A>, + label: Option<&str>, + color_attachments: &[Option<RenderPassColorAttachment>], + depth_stencil_attachment: Option<&RenderPassDepthStencilAttachment>, + timestamp_writes: Option<&RenderPassTimestampWrites>, + occlusion_query_set: Option<id::QuerySetId>, + encoder: &mut CommandEncoder<A>, + trackers: &mut Tracker<A>, + texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>, + pending_query_resets: &mut QueryResetMap<A>, + view_guard: &'a Storage<TextureView<A>>, + buffer_guard: &'a Storage<Buffer<A>>, + texture_guard: &'a Storage<Texture<A>>, + query_set_guard: &'a Storage<QuerySet<A>>, + snatch_guard: &SnatchGuard<'a>, + ) -> Result<Self, RenderPassErrorInner> { + profiling::scope!("RenderPassInfo::start"); + + // We default to false intentionally, even if depth-stencil isn't used at all. + // This allows us to use the primary raw pipeline in `RenderPipeline`, + // instead of the special read-only one, which would be `None`. + let mut is_depth_read_only = false; + let mut is_stencil_read_only = false; + + let mut render_attachments = AttachmentDataVec::<RenderAttachment<A>>::new(); + let mut discarded_surfaces = AttachmentDataVec::new(); + let mut pending_discard_init_fixups = SurfacesInDiscardState::new(); + let mut divergent_discarded_depth_stencil_aspect = None; + + let mut attachment_location = AttachmentErrorLocation::Color { + index: usize::MAX, + resolve: false, + }; + let mut extent = None; + let mut sample_count = 0; + + let mut detected_multiview: Option<Option<NonZeroU32>> = None; + + let mut check_multiview = |view: &TextureView<A>| { + // Get the multiview configuration for this texture view + let layers = view.selector.layers.end - view.selector.layers.start; + let this_multiview = if layers >= 2 { + // Trivially proven by the if above + Some(unsafe { NonZeroU32::new_unchecked(layers) }) + } else { + None + }; + + // Make sure that if this view is a multiview, it is set to be an array + if this_multiview.is_some() && view.desc.dimension != TextureViewDimension::D2Array { + return Err(RenderPassErrorInner::MultiViewDimensionMismatch); + } + + // Validate matching first, or store the first one + if let Some(multiview) = detected_multiview { + if multiview != this_multiview { + return Err(RenderPassErrorInner::MultiViewMismatch); + } + } else { + // Multiview is only supported if the feature is enabled + if this_multiview.is_some() { + device.require_features(wgt::Features::MULTIVIEW)?; + } + + detected_multiview = Some(this_multiview); + } + + Ok(()) + }; + let mut add_view = |view: &TextureView<A>, location| { + let render_extent = view.render_extent.map_err(|reason| { + RenderPassErrorInner::TextureViewIsNotRenderable { location, reason } + })?; + if let Some(ex) = extent { + if ex != render_extent { + return Err(RenderPassErrorInner::AttachmentsDimensionMismatch { + expected_location: attachment_location, + expected_extent: ex, + actual_location: location, + actual_extent: render_extent, + }); + } + } else { + extent = Some(render_extent); + } + if sample_count == 0 { + sample_count = view.samples; + } else if sample_count != view.samples { + return Err(RenderPassErrorInner::AttachmentSampleCountMismatch { + expected_location: attachment_location, + expected_samples: sample_count, + actual_location: location, + actual_samples: view.samples, + }); + } + attachment_location = location; + Ok(()) + }; + + let mut colors = + ArrayVec::<Option<hal::ColorAttachment<A>>, { hal::MAX_COLOR_ATTACHMENTS }>::new(); + let mut depth_stencil = None; + + if let Some(at) = depth_stencil_attachment { + let view: &TextureView<A> = trackers + .views + .add_single(view_guard, at.view) + .ok_or(RenderPassErrorInner::InvalidAttachment(at.view))?; + check_multiview(view)?; + add_view(view, AttachmentErrorLocation::Depth)?; + + let ds_aspects = view.desc.aspects(); + if ds_aspects.contains(hal::FormatAspects::COLOR) { + return Err(RenderPassErrorInner::InvalidDepthStencilAttachmentFormat( + view.desc.format, + )); + } + + if !ds_aspects.contains(hal::FormatAspects::STENCIL) + || (at.stencil.load_op == at.depth.load_op + && at.stencil.store_op == at.depth.store_op) + { + Self::add_pass_texture_init_actions( + &at.depth, + texture_memory_actions, + view, + &mut pending_discard_init_fixups, + ); + } else if !ds_aspects.contains(hal::FormatAspects::DEPTH) { + Self::add_pass_texture_init_actions( + &at.stencil, + texture_memory_actions, + view, + &mut pending_discard_init_fixups, + ); + } else { + // This is the only place (anywhere in wgpu) where Stencil & + // Depth init state can diverge. + // + // To safe us the overhead of tracking init state of texture + // aspects everywhere, we're going to cheat a little bit in + // order to keep the init state of both Stencil and Depth + // aspects in sync. The expectation is that we hit this path + // extremely rarely! + // + // Diverging LoadOp, i.e. Load + Clear: + // + // Record MemoryInitKind::NeedsInitializedMemory for the entire + // surface, a bit wasteful on unit but no negative effect! + // + // Rationale: If the loaded channel is uninitialized it needs + // clearing, the cleared channel doesn't care. (If everything is + // already initialized nothing special happens) + // + // (possible minor optimization: Clear caused by + // NeedsInitializedMemory should know that it doesn't need to + // clear the aspect that was set to C) + let need_init_beforehand = + at.depth.load_op == LoadOp::Load || at.stencil.load_op == LoadOp::Load; + if need_init_beforehand { + pending_discard_init_fixups.extend( + texture_memory_actions.register_init_action(&TextureInitTrackerAction { + texture: view.parent.clone(), + range: TextureInitRange::from(view.selector.clone()), + kind: MemoryInitKind::NeedsInitializedMemory, + }), + ); + } + + // Diverging Store, i.e. Discard + Store: + // + // Immediately zero out channel that is set to discard after + // we're done with the render pass. This allows us to set the + // entire surface to MemoryInitKind::ImplicitlyInitialized (if + // it isn't already set to NeedsInitializedMemory). + // + // (possible optimization: Delay and potentially drop this zeroing) + if at.depth.store_op != at.stencil.store_op { + if !need_init_beforehand { + texture_memory_actions.register_implicit_init( + &view.parent, + TextureInitRange::from(view.selector.clone()), + ); + } + divergent_discarded_depth_stencil_aspect = Some(( + if at.depth.store_op == StoreOp::Discard { + wgt::TextureAspect::DepthOnly + } else { + wgt::TextureAspect::StencilOnly + }, + view, + )); + } else if at.depth.store_op == StoreOp::Discard { + // Both are discarded using the regular path. + discarded_surfaces.push(TextureSurfaceDiscard { + texture: view.parent.clone(), + mip_level: view.selector.mips.start, + layer: view.selector.layers.start, + }); + } + } + + (is_depth_read_only, is_stencil_read_only) = at.depth_stencil_read_only(ds_aspects)?; + + let usage = if is_depth_read_only + && is_stencil_read_only + && device + .downlevel + .flags + .contains(wgt::DownlevelFlags::READ_ONLY_DEPTH_STENCIL) + { + hal::TextureUses::DEPTH_STENCIL_READ | hal::TextureUses::RESOURCE + } else { + hal::TextureUses::DEPTH_STENCIL_WRITE + }; + render_attachments.push(view.to_render_attachment(usage)); + + depth_stencil = Some(hal::DepthStencilAttachment { + target: hal::Attachment { + view: view + .raw(snatch_guard) + .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(view.info.id()))?, + usage, + }, + depth_ops: at.depth.hal_ops(), + stencil_ops: at.stencil.hal_ops(), + clear_value: (at.depth.clear_value, at.stencil.clear_value), + }); + } + + for (index, attachment) in color_attachments.iter().enumerate() { + let at = if let Some(attachment) = attachment.as_ref() { + attachment + } else { + colors.push(None); + continue; + }; + let color_view: &TextureView<A> = trackers + .views + .add_single(view_guard, at.view) + .ok_or(RenderPassErrorInner::InvalidAttachment(at.view))?; + check_multiview(color_view)?; + add_view( + color_view, + AttachmentErrorLocation::Color { + index, + resolve: false, + }, + )?; + + if !color_view + .desc + .aspects() + .contains(hal::FormatAspects::COLOR) + { + return Err(RenderPassErrorInner::ColorAttachment( + ColorAttachmentError::InvalidFormat(color_view.desc.format), + )); + } + + Self::add_pass_texture_init_actions( + &at.channel, + texture_memory_actions, + color_view, + &mut pending_discard_init_fixups, + ); + render_attachments + .push(color_view.to_render_attachment(hal::TextureUses::COLOR_TARGET)); + + let mut hal_resolve_target = None; + if let Some(resolve_target) = at.resolve_target { + let resolve_view: &TextureView<A> = trackers + .views + .add_single(view_guard, resolve_target) + .ok_or(RenderPassErrorInner::InvalidAttachment(resolve_target))?; + + check_multiview(resolve_view)?; + + let resolve_location = AttachmentErrorLocation::Color { + index, + resolve: true, + }; + + let render_extent = resolve_view.render_extent.map_err(|reason| { + RenderPassErrorInner::TextureViewIsNotRenderable { + location: resolve_location, + reason, + } + })?; + if color_view.render_extent.unwrap() != render_extent { + return Err(RenderPassErrorInner::AttachmentsDimensionMismatch { + expected_location: attachment_location, + expected_extent: extent.unwrap_or_default(), + actual_location: resolve_location, + actual_extent: render_extent, + }); + } + if color_view.samples == 1 || resolve_view.samples != 1 { + return Err(RenderPassErrorInner::InvalidResolveSampleCounts { + location: resolve_location, + src: color_view.samples, + dst: resolve_view.samples, + }); + } + if color_view.desc.format != resolve_view.desc.format { + return Err(RenderPassErrorInner::MismatchedResolveTextureFormat { + location: resolve_location, + src: color_view.desc.format, + dst: resolve_view.desc.format, + }); + } + if !resolve_view + .format_features + .flags + .contains(wgt::TextureFormatFeatureFlags::MULTISAMPLE_RESOLVE) + { + return Err(RenderPassErrorInner::UnsupportedResolveTargetFormat { + location: resolve_location, + format: resolve_view.desc.format, + }); + } + + texture_memory_actions.register_implicit_init( + &resolve_view.parent, + TextureInitRange::from(resolve_view.selector.clone()), + ); + render_attachments + .push(resolve_view.to_render_attachment(hal::TextureUses::COLOR_TARGET)); + + hal_resolve_target = Some(hal::Attachment { + view: resolve_view.raw(snatch_guard).ok_or_else(|| { + RenderPassErrorInner::InvalidResolveTarget(resolve_view.info.id()) + })?, + usage: hal::TextureUses::COLOR_TARGET, + }); + } + + colors.push(Some(hal::ColorAttachment { + target: hal::Attachment { + view: color_view.raw(snatch_guard).ok_or_else(|| { + RenderPassErrorInner::InvalidAttachment(color_view.info.id()) + })?, + usage: hal::TextureUses::COLOR_TARGET, + }, + resolve_target: hal_resolve_target, + ops: at.channel.hal_ops(), + clear_value: at.channel.clear_value, + })); + } + + let extent = extent.ok_or(RenderPassErrorInner::MissingAttachments)?; + let multiview = detected_multiview.expect("Multiview was not detected, no attachments"); + + let view_data = AttachmentData { + colors: color_attachments + .iter() + .map(|at| at.as_ref().map(|at| view_guard.get(at.view).unwrap())) + .collect(), + resolves: color_attachments + .iter() + .filter_map(|at| match *at { + Some(RenderPassColorAttachment { + resolve_target: Some(resolve), + .. + }) => Some(view_guard.get(resolve).unwrap()), + _ => None, + }) + .collect(), + depth_stencil: depth_stencil_attachment.map(|at| view_guard.get(at.view).unwrap()), + }; + + let context = RenderPassContext { + attachments: view_data.map(|view| view.desc.format), + sample_count, + multiview, + }; + + let timestamp_writes = if let Some(tw) = timestamp_writes { + let query_set = trackers + .query_sets + .add_single(query_set_guard, tw.query_set) + .ok_or(RenderPassErrorInner::InvalidQuerySet(tw.query_set))?; + + if let Some(index) = tw.beginning_of_pass_write_index { + pending_query_resets.use_query_set(tw.query_set, query_set, index); + } + if let Some(index) = tw.end_of_pass_write_index { + pending_query_resets.use_query_set(tw.query_set, query_set, index); + } + + Some(hal::RenderPassTimestampWrites { + query_set: query_set.raw.as_ref().unwrap(), + beginning_of_pass_write_index: tw.beginning_of_pass_write_index, + end_of_pass_write_index: tw.end_of_pass_write_index, + }) + } else { + None + }; + + let occlusion_query_set = if let Some(occlusion_query_set) = occlusion_query_set { + let query_set = trackers + .query_sets + .add_single(query_set_guard, occlusion_query_set) + .ok_or(RenderPassErrorInner::InvalidQuerySet(occlusion_query_set))?; + + Some(query_set.raw.as_ref().unwrap()) + } else { + None + }; + + let hal_desc = hal::RenderPassDescriptor { + label: hal_label(label, device.instance_flags), + extent, + sample_count, + color_attachments: &colors, + depth_stencil_attachment: depth_stencil, + multiview, + timestamp_writes, + occlusion_query_set, + }; + unsafe { + encoder.raw.begin_render_pass(&hal_desc); + }; + + Ok(Self { + context, + usage_scope: UsageScope::new(buffer_guard, texture_guard), + render_attachments, + is_depth_read_only, + is_stencil_read_only, + extent, + _phantom: PhantomData, + pending_discard_init_fixups, + divergent_discarded_depth_stencil_aspect, + multiview, + }) + } + + fn finish( + mut self, + raw: &mut A::CommandEncoder, + snatch_guard: &SnatchGuard, + ) -> Result<(UsageScope<A>, SurfacesInDiscardState<A>), RenderPassErrorInner> { + profiling::scope!("RenderPassInfo::finish"); + unsafe { + raw.end_render_pass(); + } + + for ra in self.render_attachments { + let texture = &ra.texture; + check_texture_usage(texture.desc.usage, TextureUsages::RENDER_ATTACHMENT)?; + + // the tracker set of the pass is always in "extend" mode + unsafe { + self.usage_scope + .textures + .merge_single(texture, Some(ra.selector.clone()), ra.usage) + .map_err(UsageConflict::from)? + }; + } + + // If either only stencil or depth was discarded, we put in a special + // clear pass to keep the init status of the aspects in sync. We do this + // so we don't need to track init state for depth/stencil aspects + // individually. + // + // Note that we don't go the usual route of "brute force" initializing + // the texture when need arises here, since this path is actually + // something a user may genuinely want (where as the other cases are + // more seen along the lines as gracefully handling a user error). + if let Some((aspect, view)) = self.divergent_discarded_depth_stencil_aspect { + let (depth_ops, stencil_ops) = if aspect == wgt::TextureAspect::DepthOnly { + ( + hal::AttachmentOps::STORE, // clear depth + hal::AttachmentOps::LOAD | hal::AttachmentOps::STORE, // unchanged stencil + ) + } else { + ( + hal::AttachmentOps::LOAD | hal::AttachmentOps::STORE, // unchanged stencil + hal::AttachmentOps::STORE, // clear depth + ) + }; + let desc = hal::RenderPassDescriptor { + label: Some("(wgpu internal) Zero init discarded depth/stencil aspect"), + extent: view.render_extent.unwrap(), + sample_count: view.samples, + color_attachments: &[], + depth_stencil_attachment: Some(hal::DepthStencilAttachment { + target: hal::Attachment { + view: view.raw(snatch_guard).ok_or_else(|| { + RenderPassErrorInner::InvalidAttachment(view.info.id()) + })?, + usage: hal::TextureUses::DEPTH_STENCIL_WRITE, + }, + depth_ops, + stencil_ops, + clear_value: (0.0, 0), + }), + multiview: self.multiview, + timestamp_writes: None, + occlusion_query_set: None, + }; + unsafe { + raw.begin_render_pass(&desc); + raw.end_render_pass(); + } + } + + Ok((self.usage_scope, self.pending_discard_init_fixups)) + } +} + +// Common routines between render/compute + +impl Global { + pub fn command_encoder_run_render_pass<A: HalApi>( + &self, + encoder_id: id::CommandEncoderId, + pass: &RenderPass, + ) -> Result<(), RenderPassError> { + self.command_encoder_run_render_pass_impl::<A>( + encoder_id, + pass.base.as_ref(), + &pass.color_targets, + pass.depth_stencil_target.as_ref(), + pass.timestamp_writes.as_ref(), + pass.occlusion_query_set_id, + ) + } + + #[doc(hidden)] + pub fn command_encoder_run_render_pass_impl<A: HalApi>( + &self, + encoder_id: id::CommandEncoderId, + base: BasePassRef<RenderCommand>, + color_attachments: &[Option<RenderPassColorAttachment>], + depth_stencil_attachment: Option<&RenderPassDepthStencilAttachment>, + timestamp_writes: Option<&RenderPassTimestampWrites>, + occlusion_query_set_id: Option<id::QuerySetId>, + ) -> Result<(), RenderPassError> { + profiling::scope!( + "CommandEncoder::run_render_pass {}", + base.label.unwrap_or("") + ); + + let discard_hal_labels = self + .instance + .flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS); + let label = hal_label(base.label, self.instance.flags); + + let pass_scope = PassErrorScope::Pass(encoder_id); + + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id).map_pass_err(pass_scope)?; + let device = &cmd_buf.device; + let snatch_guard = device.snatchable_lock.read(); + + let (scope, pending_discard_init_fixups) = { + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(crate::device::trace::Command::RunRenderPass { + base: BasePass::from_ref(base), + target_colors: color_attachments.to_vec(), + target_depth_stencil: depth_stencil_attachment.cloned(), + timestamp_writes: timestamp_writes.cloned(), + occlusion_query_set_id, + }); + } + + if !device.is_valid() { + return Err(DeviceError::Lost).map_pass_err(pass_scope); + } + + let encoder = &mut cmd_buf_data.encoder; + let status = &mut cmd_buf_data.status; + let tracker = &mut cmd_buf_data.trackers; + let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions; + let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions; + let pending_query_resets = &mut cmd_buf_data.pending_query_resets; + + // We automatically keep extending command buffers over time, and because + // we want to insert a command buffer _before_ what we're about to record, + // we need to make sure to close the previous one. + encoder.close().map_pass_err(pass_scope)?; + // We will reset this to `Recording` if we succeed, acts as a fail-safe. + *status = CommandEncoderStatus::Error; + encoder.open_pass(label).map_pass_err(pass_scope)?; + + let bundle_guard = hub.render_bundles.read(); + let bind_group_guard = hub.bind_groups.read(); + let render_pipeline_guard = hub.render_pipelines.read(); + let query_set_guard = hub.query_sets.read(); + let buffer_guard = hub.buffers.read(); + let texture_guard = hub.textures.read(); + let view_guard = hub.texture_views.read(); + + log::trace!( + "Encoding render pass begin in command buffer {:?}", + encoder_id + ); + + let mut info = RenderPassInfo::start( + device, + label, + color_attachments, + depth_stencil_attachment, + timestamp_writes, + occlusion_query_set_id, + encoder, + tracker, + texture_memory_actions, + pending_query_resets, + &*view_guard, + &*buffer_guard, + &*texture_guard, + &*query_set_guard, + &snatch_guard, + ) + .map_pass_err(pass_scope)?; + + tracker.set_size( + Some(&*buffer_guard), + Some(&*texture_guard), + Some(&*view_guard), + None, + Some(&*bind_group_guard), + None, + Some(&*render_pipeline_guard), + Some(&*bundle_guard), + Some(&*query_set_guard), + ); + + let raw = &mut encoder.raw; + + let mut state = State { + pipeline_flags: PipelineFlags::empty(), + binder: Binder::new(), + blend_constant: OptionalState::Unused, + stencil_reference: 0, + pipeline: None, + index: IndexState::default(), + vertex: VertexState::default(), + debug_scope_depth: 0, + }; + let mut temp_offsets = Vec::new(); + let mut dynamic_offset_count = 0; + let mut string_offset = 0; + let mut active_query = None; + + for command in base.commands { + match *command { + RenderCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group_id, + } => { + api_log!("RenderPass::set_bind_group {index} {bind_group_id:?}"); + + let scope = PassErrorScope::SetBindGroup(bind_group_id); + let max_bind_groups = device.limits.max_bind_groups; + if index >= max_bind_groups { + return Err(RenderCommandError::BindGroupIndexOutOfRange { + index, + max: max_bind_groups, + }) + .map_pass_err(scope); + } + + temp_offsets.clear(); + temp_offsets.extend_from_slice( + &base.dynamic_offsets + [dynamic_offset_count..dynamic_offset_count + num_dynamic_offsets], + ); + dynamic_offset_count += num_dynamic_offsets; + + let bind_group = tracker + .bind_groups + .add_single(&*bind_group_guard, bind_group_id) + .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id)) + .map_pass_err(scope)?; + + if bind_group.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice).map_pass_err(scope); + } + + bind_group + .validate_dynamic_bindings(index, &temp_offsets, &cmd_buf.limits) + .map_pass_err(scope)?; + + // merge the resource tracker in + unsafe { + info.usage_scope + .merge_bind_group(&bind_group.used) + .map_pass_err(scope)?; + } + //Note: stateless trackers are not merged: the lifetime reference + // is held to the bind group itself. + + buffer_memory_init_actions.extend( + bind_group.used_buffer_ranges.iter().filter_map(|action| { + action + .buffer + .initialization_status + .read() + .check_action(action) + }), + ); + for action in bind_group.used_texture_ranges.iter() { + info.pending_discard_init_fixups + .extend(texture_memory_actions.register_init_action(action)); + } + + let pipeline_layout = state.binder.pipeline_layout.clone(); + let entries = + state + .binder + .assign_group(index as usize, bind_group, &temp_offsets); + if !entries.is_empty() && pipeline_layout.is_some() { + let pipeline_layout = pipeline_layout.as_ref().unwrap().raw(); + for (i, e) in entries.iter().enumerate() { + if let Some(group) = e.group.as_ref() { + let raw_bg = group + .raw(&snatch_guard) + .ok_or(RenderPassErrorInner::InvalidBindGroup(i)) + .map_pass_err(scope)?; + unsafe { + raw.set_bind_group( + pipeline_layout, + index + i as u32, + raw_bg, + &e.dynamic_offsets, + ); + } + } + } + } + } + RenderCommand::SetPipeline(pipeline_id) => { + api_log!("RenderPass::set_pipeline {pipeline_id:?}"); + + let scope = PassErrorScope::SetPipelineRender(pipeline_id); + state.pipeline = Some(pipeline_id); + + let pipeline: &pipeline::RenderPipeline<A> = tracker + .render_pipelines + .add_single(&*render_pipeline_guard, pipeline_id) + .ok_or(RenderCommandError::InvalidPipeline(pipeline_id)) + .map_pass_err(scope)?; + + if pipeline.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice).map_pass_err(scope); + } + + info.context + .check_compatible( + &pipeline.pass_context, + RenderPassCompatibilityCheckType::RenderPipeline, + ) + .map_err(RenderCommandError::IncompatiblePipelineTargets) + .map_pass_err(scope)?; + + state.pipeline_flags = pipeline.flags; + + if (pipeline.flags.contains(PipelineFlags::WRITES_DEPTH) + && info.is_depth_read_only) + || (pipeline.flags.contains(PipelineFlags::WRITES_STENCIL) + && info.is_stencil_read_only) + { + return Err(RenderCommandError::IncompatiblePipelineRods) + .map_pass_err(scope); + } + + state + .blend_constant + .require(pipeline.flags.contains(PipelineFlags::BLEND_CONSTANT)); + + unsafe { + raw.set_render_pipeline(pipeline.raw()); + } + + if pipeline.flags.contains(PipelineFlags::STENCIL_REFERENCE) { + unsafe { + raw.set_stencil_reference(state.stencil_reference); + } + } + + // Rebind resource + if state.binder.pipeline_layout.is_none() + || !state + .binder + .pipeline_layout + .as_ref() + .unwrap() + .is_equal(&pipeline.layout) + { + let (start_index, entries) = state.binder.change_pipeline_layout( + &pipeline.layout, + &pipeline.late_sized_buffer_groups, + ); + if !entries.is_empty() { + for (i, e) in entries.iter().enumerate() { + if let Some(group) = e.group.as_ref() { + let raw_bg = group + .raw(&snatch_guard) + .ok_or(RenderPassErrorInner::InvalidBindGroup(i)) + .map_pass_err(scope)?; + unsafe { + raw.set_bind_group( + pipeline.layout.raw(), + start_index as u32 + i as u32, + raw_bg, + &e.dynamic_offsets, + ); + } + } + } + } + + // Clear push constant ranges + let non_overlapping = super::bind::compute_nonoverlapping_ranges( + &pipeline.layout.push_constant_ranges, + ); + for range in non_overlapping { + let offset = range.range.start; + let size_bytes = range.range.end - offset; + super::push_constant_clear( + offset, + size_bytes, + |clear_offset, clear_data| unsafe { + raw.set_push_constants( + pipeline.layout.raw(), + range.stages, + clear_offset, + clear_data, + ); + }, + ); + } + } + + state.index.pipeline_format = pipeline.strip_index_format; + + let vertex_steps_len = pipeline.vertex_steps.len(); + state.vertex.buffers_required = vertex_steps_len as u32; + + // Initialize each `vertex.inputs[i].step` from + // `pipeline.vertex_steps[i]`. Enlarge `vertex.inputs` + // as necessary to accommodate all slots in the + // pipeline. If `vertex.inputs` is longer, fill the + // extra entries with default `VertexStep`s. + while state.vertex.inputs.len() < vertex_steps_len { + state.vertex.inputs.push(VertexBufferState::EMPTY); + } + + // This is worse as a `zip`, but it's close. + let mut steps = pipeline.vertex_steps.iter(); + for input in state.vertex.inputs.iter_mut() { + input.step = steps.next().cloned().unwrap_or_default(); + } + + // Update vertex buffer limits. + state.vertex.update_limits(); + } + RenderCommand::SetIndexBuffer { + buffer_id, + index_format, + offset, + size, + } => { + api_log!("RenderPass::set_index_buffer {buffer_id:?}"); + + let scope = PassErrorScope::SetIndexBuffer(buffer_id); + let buffer = info + .usage_scope + .buffers + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) + .map_pass_err(scope)?; + + if buffer.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice).map_pass_err(scope); + } + + check_buffer_usage(buffer.usage, BufferUsages::INDEX) + .map_pass_err(scope)?; + let buf_raw = buffer + .raw + .get(&snatch_guard) + .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) + .map_pass_err(scope)?; + + let end = match size { + Some(s) => offset + s.get(), + None => buffer.size, + }; + state.index.bound_buffer_view = Some((buffer_id, offset..end)); + + state.index.format = Some(index_format); + state.index.update_limit(); + + buffer_memory_init_actions.extend( + buffer.initialization_status.read().create_action( + buffer, + offset..end, + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + let bb = hal::BufferBinding { + buffer: buf_raw, + offset, + size, + }; + unsafe { + raw.set_index_buffer(bb, index_format); + } + } + RenderCommand::SetVertexBuffer { + slot, + buffer_id, + offset, + size, + } => { + api_log!("RenderPass::set_vertex_buffer {slot} {buffer_id:?}"); + + let scope = PassErrorScope::SetVertexBuffer(buffer_id); + let buffer = info + .usage_scope + .buffers + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) + .map_pass_err(scope)?; + + if buffer.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice).map_pass_err(scope); + } + + let max_vertex_buffers = device.limits.max_vertex_buffers; + if slot >= max_vertex_buffers { + return Err(RenderCommandError::VertexBufferIndexOutOfRange { + index: slot, + max: max_vertex_buffers, + }) + .map_pass_err(scope); + } + + check_buffer_usage(buffer.usage, BufferUsages::VERTEX) + .map_pass_err(scope)?; + let buf_raw = buffer + .raw + .get(&snatch_guard) + .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) + .map_pass_err(scope)?; + + let empty_slots = + (1 + slot as usize).saturating_sub(state.vertex.inputs.len()); + state + .vertex + .inputs + .extend(iter::repeat(VertexBufferState::EMPTY).take(empty_slots)); + let vertex_state = &mut state.vertex.inputs[slot as usize]; + //TODO: where are we checking that the offset is in bound? + vertex_state.total_size = match size { + Some(s) => s.get(), + None => buffer.size - offset, + }; + vertex_state.bound = true; + + buffer_memory_init_actions.extend( + buffer.initialization_status.read().create_action( + buffer, + offset..(offset + vertex_state.total_size), + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + let bb = hal::BufferBinding { + buffer: buf_raw, + offset, + size, + }; + unsafe { + raw.set_vertex_buffer(slot, bb); + } + state.vertex.update_limits(); + } + RenderCommand::SetBlendConstant(ref color) => { + api_log!("RenderPass::set_blend_constant"); + + state.blend_constant = OptionalState::Set; + let array = [ + color.r as f32, + color.g as f32, + color.b as f32, + color.a as f32, + ]; + unsafe { + raw.set_blend_constants(&array); + } + } + RenderCommand::SetStencilReference(value) => { + api_log!("RenderPass::set_stencil_reference {value}"); + + state.stencil_reference = value; + if state + .pipeline_flags + .contains(PipelineFlags::STENCIL_REFERENCE) + { + unsafe { + raw.set_stencil_reference(value); + } + } + } + RenderCommand::SetViewport { + ref rect, + depth_min, + depth_max, + } => { + api_log!("RenderPass::set_viewport {rect:?}"); + + let scope = PassErrorScope::SetViewport; + if rect.x < 0.0 + || rect.y < 0.0 + || rect.w <= 0.0 + || rect.h <= 0.0 + || rect.x + rect.w > info.extent.width as f32 + || rect.y + rect.h > info.extent.height as f32 + { + return Err(RenderCommandError::InvalidViewportRect( + *rect, + info.extent, + )) + .map_pass_err(scope); + } + if !(0.0..=1.0).contains(&depth_min) || !(0.0..=1.0).contains(&depth_max) { + return Err(RenderCommandError::InvalidViewportDepth( + depth_min, depth_max, + )) + .map_pass_err(scope); + } + let r = hal::Rect { + x: rect.x, + y: rect.y, + w: rect.w, + h: rect.h, + }; + unsafe { + raw.set_viewport(&r, depth_min..depth_max); + } + } + RenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset, + } => { + api_log!("RenderPass::set_push_constants"); + + let scope = PassErrorScope::SetPushConstant; + let values_offset = values_offset + .ok_or(RenderPassErrorInner::InvalidValuesOffset) + .map_pass_err(scope)?; + + let end_offset_bytes = offset + size_bytes; + let values_end_offset = + (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; + let data_slice = + &base.push_constant_data[(values_offset as usize)..values_end_offset]; + + let pipeline_layout = state + .binder + .pipeline_layout + .as_ref() + .ok_or(DrawError::MissingPipeline) + .map_pass_err(scope)?; + + pipeline_layout + .validate_push_constant_ranges(stages, offset, end_offset_bytes) + .map_err(RenderCommandError::from) + .map_pass_err(scope)?; + + unsafe { + raw.set_push_constants( + pipeline_layout.raw(), + stages, + offset, + data_slice, + ) + } + } + RenderCommand::SetScissor(ref rect) => { + api_log!("RenderPass::set_scissor_rect {rect:?}"); + + let scope = PassErrorScope::SetScissorRect; + if rect.x + rect.w > info.extent.width + || rect.y + rect.h > info.extent.height + { + return Err(RenderCommandError::InvalidScissorRect(*rect, info.extent)) + .map_pass_err(scope); + } + let r = hal::Rect { + x: rect.x, + y: rect.y, + w: rect.w, + h: rect.h, + }; + unsafe { + raw.set_scissor_rect(&r); + } + } + RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + } => { + api_log!( + "RenderPass::draw {vertex_count} {instance_count} {first_vertex} {first_instance}" + ); + + let indexed = false; + let scope = PassErrorScope::Draw { + indexed, + indirect: false, + pipeline: state.pipeline, + }; + state.is_ready(indexed).map_pass_err(scope)?; + + let last_vertex = first_vertex as u64 + vertex_count as u64; + let vertex_limit = state.vertex.vertex_limit; + if last_vertex > vertex_limit { + return Err(DrawError::VertexBeyondLimit { + last_vertex, + vertex_limit, + slot: state.vertex.vertex_limit_slot, + }) + .map_pass_err(scope); + } + let last_instance = first_instance as u64 + instance_count as u64; + let instance_limit = state.vertex.instance_limit; + if last_instance > instance_limit { + return Err(DrawError::InstanceBeyondLimit { + last_instance, + instance_limit, + slot: state.vertex.instance_limit_slot, + }) + .map_pass_err(scope); + } + + unsafe { + if instance_count > 0 && vertex_count > 0 { + raw.draw( + first_vertex, + vertex_count, + first_instance, + instance_count, + ); + } + } + } + RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + } => { + api_log!("RenderPass::draw_indexed {index_count} {instance_count} {first_index} {base_vertex} {first_instance}"); + + let indexed = true; + let scope = PassErrorScope::Draw { + indexed, + indirect: false, + pipeline: state.pipeline, + }; + state.is_ready(indexed).map_pass_err(scope)?; + + let last_index = first_index as u64 + index_count as u64; + let index_limit = state.index.limit; + if last_index > index_limit { + return Err(DrawError::IndexBeyondLimit { + last_index, + index_limit, + }) + .map_pass_err(scope); + } + let last_instance = first_instance as u64 + instance_count as u64; + let instance_limit = state.vertex.instance_limit; + if last_instance > instance_limit { + return Err(DrawError::InstanceBeyondLimit { + last_instance, + instance_limit, + slot: state.vertex.instance_limit_slot, + }) + .map_pass_err(scope); + } + + unsafe { + if instance_count > 0 && index_count > 0 { + raw.draw_indexed( + first_index, + index_count, + base_vertex, + first_instance, + instance_count, + ); + } + } + } + RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count, + indexed, + } => { + api_log!("RenderPass::draw_indirect (indexed:{indexed}) {buffer_id:?} {offset} {count:?}"); + + let scope = PassErrorScope::Draw { + indexed, + indirect: true, + pipeline: state.pipeline, + }; + state.is_ready(indexed).map_pass_err(scope)?; + + let stride = match indexed { + false => mem::size_of::<wgt::DrawIndirectArgs>(), + true => mem::size_of::<wgt::DrawIndexedIndirectArgs>(), + }; + + if count.is_some() { + device + .require_features(wgt::Features::MULTI_DRAW_INDIRECT) + .map_pass_err(scope)?; + } + device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) + .map_pass_err(scope)?; + + let indirect_buffer = info + .usage_scope + .buffers + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; + check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT) + .map_pass_err(scope)?; + let indirect_raw = indirect_buffer + .raw + .get(&snatch_guard) + .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) + .map_pass_err(scope)?; + + let actual_count = count.map_or(1, |c| c.get()); + + let end_offset = offset + stride as u64 * actual_count as u64; + if end_offset > indirect_buffer.size { + return Err(RenderPassErrorInner::IndirectBufferOverrun { + count, + offset, + end_offset, + buffer_size: indirect_buffer.size, + }) + .map_pass_err(scope); + } + + buffer_memory_init_actions.extend( + indirect_buffer.initialization_status.read().create_action( + indirect_buffer, + offset..end_offset, + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + match indexed { + false => unsafe { + raw.draw_indirect(indirect_raw, offset, actual_count); + }, + true => unsafe { + raw.draw_indexed_indirect(indirect_raw, offset, actual_count); + }, + } + } + RenderCommand::MultiDrawIndirectCount { + buffer_id, + offset, + count_buffer_id, + count_buffer_offset, + max_count, + indexed, + } => { + api_log!("RenderPass::multi_draw_indirect_count (indexed:{indexed}) {buffer_id:?} {offset} {count_buffer_id:?} {count_buffer_offset:?} {max_count:?}"); + + let scope = PassErrorScope::Draw { + indexed, + indirect: true, + pipeline: state.pipeline, + }; + state.is_ready(indexed).map_pass_err(scope)?; + + let stride = match indexed { + false => mem::size_of::<wgt::DrawIndirectArgs>(), + true => mem::size_of::<wgt::DrawIndexedIndirectArgs>(), + } as u64; + + device + .require_features(wgt::Features::MULTI_DRAW_INDIRECT_COUNT) + .map_pass_err(scope)?; + device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) + .map_pass_err(scope)?; + + let indirect_buffer = info + .usage_scope + .buffers + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; + check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT) + .map_pass_err(scope)?; + let indirect_raw = indirect_buffer + .raw + .get(&snatch_guard) + .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) + .map_pass_err(scope)?; + + let count_buffer = info + .usage_scope + .buffers + .merge_single( + &*buffer_guard, + count_buffer_id, + hal::BufferUses::INDIRECT, + ) + .map_pass_err(scope)?; + check_buffer_usage(count_buffer.usage, BufferUsages::INDIRECT) + .map_pass_err(scope)?; + let count_raw = count_buffer + .raw + .get(&snatch_guard) + .ok_or(RenderCommandError::DestroyedBuffer(count_buffer_id)) + .map_pass_err(scope)?; + + let end_offset = offset + stride * max_count as u64; + if end_offset > indirect_buffer.size { + return Err(RenderPassErrorInner::IndirectBufferOverrun { + count: None, + offset, + end_offset, + buffer_size: indirect_buffer.size, + }) + .map_pass_err(scope); + } + buffer_memory_init_actions.extend( + indirect_buffer.initialization_status.read().create_action( + indirect_buffer, + offset..end_offset, + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + let begin_count_offset = count_buffer_offset; + let end_count_offset = count_buffer_offset + 4; + if end_count_offset > count_buffer.size { + return Err(RenderPassErrorInner::IndirectCountBufferOverrun { + begin_count_offset, + end_count_offset, + count_buffer_size: count_buffer.size, + }) + .map_pass_err(scope); + } + buffer_memory_init_actions.extend( + count_buffer.initialization_status.read().create_action( + count_buffer, + count_buffer_offset..end_count_offset, + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + match indexed { + false => unsafe { + raw.draw_indirect_count( + indirect_raw, + offset, + count_raw, + count_buffer_offset, + max_count, + ); + }, + true => unsafe { + raw.draw_indexed_indirect_count( + indirect_raw, + offset, + count_raw, + count_buffer_offset, + max_count, + ); + }, + } + } + RenderCommand::PushDebugGroup { color: _, len } => { + state.debug_scope_depth += 1; + if !discard_hal_labels { + let label = str::from_utf8( + &base.string_data[string_offset..string_offset + len], + ) + .unwrap(); + + api_log!("RenderPass::push_debug_group {label:?}"); + unsafe { + raw.begin_debug_marker(label); + } + } + string_offset += len; + } + RenderCommand::PopDebugGroup => { + api_log!("RenderPass::pop_debug_group"); + + let scope = PassErrorScope::PopDebugGroup; + if state.debug_scope_depth == 0 { + return Err(RenderPassErrorInner::InvalidPopDebugGroup) + .map_pass_err(scope); + } + state.debug_scope_depth -= 1; + if !discard_hal_labels { + unsafe { + raw.end_debug_marker(); + } + } + } + RenderCommand::InsertDebugMarker { color: _, len } => { + if !discard_hal_labels { + let label = str::from_utf8( + &base.string_data[string_offset..string_offset + len], + ) + .unwrap(); + api_log!("RenderPass::insert_debug_marker {label:?}"); + unsafe { + raw.insert_debug_marker(label); + } + } + string_offset += len; + } + RenderCommand::WriteTimestamp { + query_set_id, + query_index, + } => { + api_log!("RenderPass::write_timestamps {query_set_id:?} {query_index}"); + let scope = PassErrorScope::WriteTimestamp; + + device + .require_features(wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES) + .map_pass_err(scope)?; + + let query_set = tracker + .query_sets + .add_single(&*query_set_guard, query_set_id) + .ok_or(RenderCommandError::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)?; + + query_set + .validate_and_write_timestamp( + raw, + query_set_id, + query_index, + Some(&mut cmd_buf_data.pending_query_resets), + ) + .map_pass_err(scope)?; + } + RenderCommand::BeginOcclusionQuery { query_index } => { + api_log!("RenderPass::begin_occlusion_query {query_index}"); + let scope = PassErrorScope::BeginOcclusionQuery; + + let query_set_id = occlusion_query_set_id + .ok_or(RenderPassErrorInner::MissingOcclusionQuerySet) + .map_pass_err(scope)?; + + let query_set = tracker + .query_sets + .add_single(&*query_set_guard, query_set_id) + .ok_or(RenderCommandError::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)?; + + query_set + .validate_and_begin_occlusion_query( + raw, + query_set_id, + query_index, + Some(&mut cmd_buf_data.pending_query_resets), + &mut active_query, + ) + .map_pass_err(scope)?; + } + RenderCommand::EndOcclusionQuery => { + api_log!("RenderPass::end_occlusion_query"); + let scope = PassErrorScope::EndOcclusionQuery; + + end_occlusion_query(raw, &*query_set_guard, &mut active_query) + .map_pass_err(scope)?; + } + RenderCommand::BeginPipelineStatisticsQuery { + query_set_id, + query_index, + } => { + api_log!("RenderPass::begin_pipeline_statistics_query {query_set_id:?} {query_index}"); + let scope = PassErrorScope::BeginPipelineStatisticsQuery; + + let query_set = tracker + .query_sets + .add_single(&*query_set_guard, query_set_id) + .ok_or(RenderCommandError::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)?; + + query_set + .validate_and_begin_pipeline_statistics_query( + raw, + query_set_id, + query_index, + Some(&mut cmd_buf_data.pending_query_resets), + &mut active_query, + ) + .map_pass_err(scope)?; + } + RenderCommand::EndPipelineStatisticsQuery => { + api_log!("RenderPass::end_pipeline_statistics_query"); + let scope = PassErrorScope::EndPipelineStatisticsQuery; + + end_pipeline_statistics_query(raw, &*query_set_guard, &mut active_query) + .map_pass_err(scope)?; + } + RenderCommand::ExecuteBundle(bundle_id) => { + api_log!("RenderPass::execute_bundle {bundle_id:?}"); + let scope = PassErrorScope::ExecuteBundle; + let bundle: &command::RenderBundle<A> = tracker + .bundles + .add_single(&*bundle_guard, bundle_id) + .ok_or(RenderCommandError::InvalidRenderBundle(bundle_id)) + .map_pass_err(scope)?; + + if bundle.device.as_info().id() != device.as_info().id() { + return Err(DeviceError::WrongDevice).map_pass_err(scope); + } + + info.context + .check_compatible( + &bundle.context, + RenderPassCompatibilityCheckType::RenderBundle, + ) + .map_err(RenderPassErrorInner::IncompatibleBundleTargets) + .map_pass_err(scope)?; + + if (info.is_depth_read_only && !bundle.is_depth_read_only) + || (info.is_stencil_read_only && !bundle.is_stencil_read_only) + { + return Err( + RenderPassErrorInner::IncompatibleBundleReadOnlyDepthStencil { + pass_depth: info.is_depth_read_only, + pass_stencil: info.is_stencil_read_only, + bundle_depth: bundle.is_depth_read_only, + bundle_stencil: bundle.is_stencil_read_only, + }, + ) + .map_pass_err(scope); + } + + buffer_memory_init_actions.extend( + bundle + .buffer_memory_init_actions + .iter() + .filter_map(|action| { + action + .buffer + .initialization_status + .read() + .check_action(action) + }), + ); + for action in bundle.texture_memory_init_actions.iter() { + info.pending_discard_init_fixups + .extend(texture_memory_actions.register_init_action(action)); + } + + unsafe { bundle.execute(raw) } + .map_err(|e| match e { + ExecutionError::DestroyedBuffer(id) => { + RenderCommandError::DestroyedBuffer(id) + } + ExecutionError::InvalidBindGroup(id) => { + RenderCommandError::InvalidBindGroup(id) + } + ExecutionError::Unimplemented(what) => { + RenderCommandError::Unimplemented(what) + } + }) + .map_pass_err(scope)?; + + unsafe { + info.usage_scope + .merge_render_bundle(&bundle.used) + .map_pass_err(scope)?; + tracker + .add_from_render_bundle(&bundle.used) + .map_pass_err(scope)?; + }; + state.reset_bundle(); + } + } + } + + log::trace!("Merging renderpass into cmd_buf {:?}", encoder_id); + let (trackers, pending_discard_init_fixups) = + info.finish(raw, &snatch_guard).map_pass_err(pass_scope)?; + + encoder.close().map_pass_err(pass_scope)?; + (trackers, pending_discard_init_fixups) + }; + + let cmd_buf = hub.command_buffers.get(encoder_id.transmute()).unwrap(); + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + let query_set_guard = hub.query_sets.read(); + + let encoder = &mut cmd_buf_data.encoder; + let status = &mut cmd_buf_data.status; + let tracker = &mut cmd_buf_data.trackers; + + { + let transit = encoder.open().map_pass_err(pass_scope)?; + + fixup_discarded_surfaces( + pending_discard_init_fixups.into_iter(), + transit, + &mut tracker.textures, + &cmd_buf.device, + ); + + cmd_buf_data + .pending_query_resets + .reset_queries( + transit, + &query_set_guard, + cmd_buf.device.info.id().backend(), + ) + .map_err(RenderCommandError::InvalidQuerySet) + .map_pass_err(PassErrorScope::QueryReset)?; + + super::CommandBuffer::insert_barriers_from_scope( + transit, + tracker, + &scope, + &snatch_guard, + ); + } + + *status = CommandEncoderStatus::Recording; + encoder.close_and_swap().map_pass_err(pass_scope)?; + + Ok(()) + } +} + +pub mod render_ffi { + use super::{ + super::{Rect, RenderCommand}, + RenderPass, + }; + use crate::{id, RawString}; + use std::{convert::TryInto, ffi, num::NonZeroU32, slice}; + use wgt::{BufferAddress, BufferSize, Color, DynamicOffset, IndexFormat}; + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `offset_length` elements. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_pass_set_bind_group( + pass: &mut RenderPass, + index: u32, + bind_group_id: id::BindGroupId, + offsets: *const DynamicOffset, + offset_length: usize, + ) { + let redundant = unsafe { + pass.current_bind_groups.set_and_check_redundant( + bind_group_id, + index, + &mut pass.base.dynamic_offsets, + offsets, + offset_length, + ) + }; + + if redundant { + return; + } + + pass.base.commands.push(RenderCommand::SetBindGroup { + index, + num_dynamic_offsets: offset_length, + bind_group_id, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_pipeline( + pass: &mut RenderPass, + pipeline_id: id::RenderPipelineId, + ) { + if pass.current_pipeline.set_and_check_redundant(pipeline_id) { + return; + } + + pass.base + .commands + .push(RenderCommand::SetPipeline(pipeline_id)); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_vertex_buffer( + pass: &mut RenderPass, + slot: u32, + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option<BufferSize>, + ) { + pass.base.commands.push(RenderCommand::SetVertexBuffer { + slot, + buffer_id, + offset, + size, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_index_buffer( + pass: &mut RenderPass, + buffer: id::BufferId, + index_format: IndexFormat, + offset: BufferAddress, + size: Option<BufferSize>, + ) { + pass.set_index_buffer(buffer, index_format, offset, size); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_blend_constant(pass: &mut RenderPass, color: &Color) { + pass.base + .commands + .push(RenderCommand::SetBlendConstant(*color)); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_stencil_reference(pass: &mut RenderPass, value: u32) { + pass.base + .commands + .push(RenderCommand::SetStencilReference(value)); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_viewport( + pass: &mut RenderPass, + x: f32, + y: f32, + w: f32, + h: f32, + depth_min: f32, + depth_max: f32, + ) { + pass.base.commands.push(RenderCommand::SetViewport { + rect: Rect { x, y, w, h }, + depth_min, + depth_max, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_scissor_rect( + pass: &mut RenderPass, + x: u32, + y: u32, + w: u32, + h: u32, + ) { + pass.base + .commands + .push(RenderCommand::SetScissor(Rect { x, y, w, h })); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `size_bytes` bytes. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_pass_set_push_constants( + pass: &mut RenderPass, + stages: wgt::ShaderStages, + offset: u32, + size_bytes: u32, + data: *const u8, + ) { + assert_eq!( + offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant offset must be aligned to 4 bytes." + ); + assert_eq!( + size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant size must be aligned to 4 bytes." + ); + let data_slice = unsafe { slice::from_raw_parts(data, size_bytes as usize) }; + let value_offset = pass.base.push_constant_data.len().try_into().expect( + "Ran out of push constant space. Don't set 4gb of push constants per RenderPass.", + ); + + pass.base.push_constant_data.extend( + data_slice + .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize) + .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])), + ); + + pass.base.commands.push(RenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset: Some(value_offset), + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_draw( + pass: &mut RenderPass, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, + ) { + pass.base.commands.push(RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_draw_indexed( + pass: &mut RenderPass, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, + ) { + pass.base.commands.push(RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_draw_indirect( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + pass.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: false, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_draw_indexed_indirect( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + pass.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: true, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_multi_draw_indirect( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + count: u32, + ) { + pass.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: NonZeroU32::new(count), + indexed: false, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_multi_draw_indexed_indirect( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + count: u32, + ) { + pass.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: NonZeroU32::new(count), + indexed: true, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_multi_draw_indirect_count( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + count_buffer_id: id::BufferId, + count_buffer_offset: BufferAddress, + max_count: u32, + ) { + pass.base + .commands + .push(RenderCommand::MultiDrawIndirectCount { + buffer_id, + offset, + count_buffer_id, + count_buffer_offset, + max_count, + indexed: false, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_multi_draw_indexed_indirect_count( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + count_buffer_id: id::BufferId, + count_buffer_offset: BufferAddress, + max_count: u32, + ) { + pass.base + .commands + .push(RenderCommand::MultiDrawIndirectCount { + buffer_id, + offset, + count_buffer_id, + count_buffer_offset, + max_count, + indexed: true, + }); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given `label` + /// is a valid null-terminated string. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_pass_push_debug_group( + pass: &mut RenderPass, + label: RawString, + color: u32, + ) { + let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes(); + pass.base.string_data.extend_from_slice(bytes); + + pass.base.commands.push(RenderCommand::PushDebugGroup { + color, + len: bytes.len(), + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_pop_debug_group(pass: &mut RenderPass) { + pass.base.commands.push(RenderCommand::PopDebugGroup); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given `label` + /// is a valid null-terminated string. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_pass_insert_debug_marker( + pass: &mut RenderPass, + label: RawString, + color: u32, + ) { + let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes(); + pass.base.string_data.extend_from_slice(bytes); + + pass.base.commands.push(RenderCommand::InsertDebugMarker { + color, + len: bytes.len(), + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_write_timestamp( + pass: &mut RenderPass, + query_set_id: id::QuerySetId, + query_index: u32, + ) { + pass.base.commands.push(RenderCommand::WriteTimestamp { + query_set_id, + query_index, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_begin_occlusion_query( + pass: &mut RenderPass, + query_index: u32, + ) { + pass.base + .commands + .push(RenderCommand::BeginOcclusionQuery { query_index }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_end_occlusion_query(pass: &mut RenderPass) { + pass.base.commands.push(RenderCommand::EndOcclusionQuery); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_begin_pipeline_statistics_query( + pass: &mut RenderPass, + query_set_id: id::QuerySetId, + query_index: u32, + ) { + pass.base + .commands + .push(RenderCommand::BeginPipelineStatisticsQuery { + query_set_id, + query_index, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_end_pipeline_statistics_query(pass: &mut RenderPass) { + pass.base + .commands + .push(RenderCommand::EndPipelineStatisticsQuery); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `render_bundle_ids_length` elements. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_pass_execute_bundles( + pass: &mut RenderPass, + render_bundle_ids: *const id::RenderBundleId, + render_bundle_ids_length: usize, + ) { + for &bundle_id in + unsafe { slice::from_raw_parts(render_bundle_ids, render_bundle_ids_length) } + { + pass.base + .commands + .push(RenderCommand::ExecuteBundle(bundle_id)); + } + pass.current_pipeline.reset(); + pass.current_bind_groups.reset(); + } +} diff --git a/third_party/rust/wgpu-core/src/command/transfer.rs b/third_party/rust/wgpu-core/src/command/transfer.rs new file mode 100644 index 0000000000..0a952dfc84 --- /dev/null +++ b/third_party/rust/wgpu-core/src/command/transfer.rs @@ -0,0 +1,1229 @@ +#[cfg(feature = "trace")] +use crate::device::trace::Command as TraceCommand; +use crate::{ + api_log, + command::{clear_texture, CommandBuffer, CommandEncoderError}, + conv, + device::{Device, DeviceError, MissingDownlevelFlags}, + error::{ErrorFormatter, PrettyError}, + global::Global, + hal_api::HalApi, + id::{BufferId, CommandEncoderId, DeviceId, TextureId}, + init_tracker::{ + has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange, + TextureInitTrackerAction, + }, + resource::{Resource, Texture, TextureErrorDimension}, + track::{TextureSelector, Tracker}, +}; + +use arrayvec::ArrayVec; +use hal::CommandEncoder as _; +use thiserror::Error; +use wgt::{BufferAddress, BufferUsages, Extent3d, TextureUsages}; + +use std::{iter, sync::Arc}; + +use super::{memory_init::CommandBufferTextureMemoryActions, ClearError, CommandEncoder}; + +pub type ImageCopyBuffer = wgt::ImageCopyBuffer<BufferId>; +pub type ImageCopyTexture = wgt::ImageCopyTexture<TextureId>; +pub type ImageCopyTextureTagged = wgt::ImageCopyTextureTagged<TextureId>; + +#[derive(Clone, Copy, Debug)] +pub enum CopySide { + Source, + Destination, +} + +/// Error encountered while attempting a data transfer. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum TransferError { + #[error("Device {0:?} is invalid")] + InvalidDevice(DeviceId), + #[error("Buffer {0:?} is invalid or destroyed")] + InvalidBuffer(BufferId), + #[error("Texture {0:?} is invalid or destroyed")] + InvalidTexture(TextureId), + #[error("Source and destination cannot be the same buffer")] + SameSourceDestinationBuffer, + #[error("Source buffer/texture is missing the `COPY_SRC` usage flag")] + MissingCopySrcUsageFlag, + #[error("Destination buffer/texture is missing the `COPY_DST` usage flag")] + MissingCopyDstUsageFlag(Option<BufferId>, Option<TextureId>), + #[error("Destination texture is missing the `RENDER_ATTACHMENT` usage flag")] + MissingRenderAttachmentUsageFlag(TextureId), + #[error("Copy of {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}")] + BufferOverrun { + start_offset: BufferAddress, + end_offset: BufferAddress, + buffer_size: BufferAddress, + side: CopySide, + }, + #[error("Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")] + TextureOverrun { + start_offset: u32, + end_offset: u32, + texture_size: u32, + dimension: TextureErrorDimension, + side: CopySide, + }, + #[error("Unable to select texture aspect {aspect:?} from format {format:?}")] + InvalidTextureAspect { + format: wgt::TextureFormat, + aspect: wgt::TextureAspect, + }, + #[error("Unable to select texture mip level {level} out of {total}")] + InvalidTextureMipLevel { level: u32, total: u32 }, + #[error("Texture dimension must be 2D when copying from an external texture")] + InvalidDimensionExternal(TextureId), + #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")] + UnalignedBufferOffset(BufferAddress), + #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")] + UnalignedCopySize(BufferAddress), + #[error("Copy width is not a multiple of block width")] + UnalignedCopyWidth, + #[error("Copy height is not a multiple of block height")] + UnalignedCopyHeight, + #[error("Copy origin's x component is not a multiple of block width")] + UnalignedCopyOriginX, + #[error("Copy origin's y component is not a multiple of block height")] + UnalignedCopyOriginY, + #[error("Bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")] + UnalignedBytesPerRow, + #[error("Number of bytes per row needs to be specified since more than one row is copied")] + UnspecifiedBytesPerRow, + #[error("Number of rows per image needs to be specified since more than one image is copied")] + UnspecifiedRowsPerImage, + #[error("Number of bytes per row is less than the number of bytes in a complete row")] + InvalidBytesPerRow, + #[error("Image is 1D and the copy height and depth are not both set to 1")] + InvalidCopySize, + #[error("Number of rows per image is invalid")] + InvalidRowsPerImage, + #[error("Copy source aspects must refer to all aspects of the source texture format")] + CopySrcMissingAspects, + #[error( + "Copy destination aspects must refer to all aspects of the destination texture format" + )] + CopyDstMissingAspects, + #[error("Copy aspect must refer to a single aspect of texture format")] + CopyAspectNotOne, + #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")] + CopyFromForbiddenTextureFormat { + format: wgt::TextureFormat, + aspect: wgt::TextureAspect, + }, + #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")] + CopyToForbiddenTextureFormat { + format: wgt::TextureFormat, + aspect: wgt::TextureAspect, + }, + #[error( + "Copying to textures with format {0:?} is forbidden when copying from external texture" + )] + ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat), + #[error("The entire texture must be copied when copying from depth texture")] + InvalidDepthTextureExtent, + #[error( + "Source format ({src_format:?}) and destination format ({dst_format:?}) are not copy-compatible (they may only differ in srgb-ness)" + )] + TextureFormatsNotCopyCompatible { + src_format: wgt::TextureFormat, + dst_format: wgt::TextureFormat, + }, + #[error(transparent)] + MemoryInitFailure(#[from] ClearError), + #[error("Cannot encode this copy because of a missing downelevel flag")] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), + #[error("Source texture sample count must be 1, got {sample_count}")] + InvalidSampleCount { sample_count: u32 }, + #[error("Requested mip level {requested} does no exist (count: {count})")] + InvalidMipLevel { requested: u32, count: u32 }, +} + +impl PrettyError for TransferError { + fn fmt_pretty(&self, fmt: &mut ErrorFormatter) { + fmt.error(self); + match *self { + Self::InvalidBuffer(id) => { + fmt.buffer_label(&id); + } + Self::InvalidTexture(id) => { + fmt.texture_label(&id); + } + // Self::MissingCopySrcUsageFlag(buf_opt, tex_opt) => { + // if let Some(buf) = buf_opt { + // let name = crate::gfx_select!(buf => global.buffer_label(buf)); + // ret.push_str(&format_label_line("source", &name)); + // } + // if let Some(tex) = tex_opt { + // let name = crate::gfx_select!(tex => global.texture_label(tex)); + // ret.push_str(&format_label_line("source", &name)); + // } + // } + Self::MissingCopyDstUsageFlag(buf_opt, tex_opt) => { + if let Some(buf) = buf_opt { + fmt.buffer_label_with_key(&buf, "destination"); + } + if let Some(tex) = tex_opt { + fmt.texture_label_with_key(&tex, "destination"); + } + } + _ => {} + }; + } +} +/// Error encountered while attempting to do a copy on a command encoder. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CopyError { + #[error(transparent)] + Encoder(#[from] CommandEncoderError), + #[error("Copy error")] + Transfer(#[from] TransferError), +} + +impl From<DeviceError> for CopyError { + fn from(err: DeviceError) -> Self { + CopyError::Encoder(CommandEncoderError::Device(err)) + } +} + +pub(crate) fn extract_texture_selector<A: HalApi>( + copy_texture: &ImageCopyTexture, + copy_size: &Extent3d, + texture: &Texture<A>, +) -> Result<(TextureSelector, hal::TextureCopyBase), TransferError> { + let format = texture.desc.format; + let copy_aspect = hal::FormatAspects::new(format, copy_texture.aspect); + if copy_aspect.is_empty() { + return Err(TransferError::InvalidTextureAspect { + format, + aspect: copy_texture.aspect, + }); + } + + let (layers, origin_z) = match texture.desc.dimension { + wgt::TextureDimension::D1 => (0..1, 0), + wgt::TextureDimension::D2 => ( + copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers, + 0, + ), + wgt::TextureDimension::D3 => (0..1, copy_texture.origin.z), + }; + let base = hal::TextureCopyBase { + origin: wgt::Origin3d { + x: copy_texture.origin.x, + y: copy_texture.origin.y, + z: origin_z, + }, + // this value will be incremented per copied layer + array_layer: layers.start, + mip_level: copy_texture.mip_level, + aspect: copy_aspect, + }; + let selector = TextureSelector { + mips: copy_texture.mip_level..copy_texture.mip_level + 1, + layers, + }; + + Ok((selector, base)) +} + +/// WebGPU's [validating linear texture data][vltd] algorithm. +/// +/// Copied with some modifications from WebGPU standard. +/// +/// If successful, returns a pair `(bytes, stride)`, where: +/// - `bytes` is the number of buffer bytes required for this copy, and +/// - `stride` number of bytes between array layers. +/// +/// [vltd]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-linear-texture-data +pub(crate) fn validate_linear_texture_data( + layout: &wgt::ImageDataLayout, + format: wgt::TextureFormat, + aspect: wgt::TextureAspect, + buffer_size: BufferAddress, + buffer_side: CopySide, + copy_size: &Extent3d, + need_copy_aligned_rows: bool, +) -> Result<(BufferAddress, BufferAddress), TransferError> { + // Convert all inputs to BufferAddress (u64) to avoid some of the overflow issues + // Note: u64 is not always enough to prevent overflow, especially when multiplying + // something with a potentially large depth value, so it is preferable to validate + // the copy size before calling this function (for example via `validate_texture_copy_range`). + let copy_width = copy_size.width as BufferAddress; + let copy_height = copy_size.height as BufferAddress; + let copy_depth = copy_size.depth_or_array_layers as BufferAddress; + + let offset = layout.offset; + + let block_size = format.block_copy_size(Some(aspect)).unwrap() as BufferAddress; + let (block_width, block_height) = format.block_dimensions(); + let block_width = block_width as BufferAddress; + let block_height = block_height as BufferAddress; + + if copy_width % block_width != 0 { + return Err(TransferError::UnalignedCopyWidth); + } + if copy_height % block_height != 0 { + return Err(TransferError::UnalignedCopyHeight); + } + + let width_in_blocks = copy_width / block_width; + let height_in_blocks = copy_height / block_height; + + let bytes_in_last_row = width_in_blocks * block_size; + + let bytes_per_row = if let Some(bytes_per_row) = layout.bytes_per_row { + let bytes_per_row = bytes_per_row as BufferAddress; + if bytes_per_row < bytes_in_last_row { + return Err(TransferError::InvalidBytesPerRow); + } + bytes_per_row + } else { + if copy_depth > 1 || height_in_blocks > 1 { + return Err(TransferError::UnspecifiedBytesPerRow); + } + 0 + }; + let block_rows_per_image = if let Some(rows_per_image) = layout.rows_per_image { + let rows_per_image = rows_per_image as BufferAddress; + if rows_per_image < height_in_blocks { + return Err(TransferError::InvalidRowsPerImage); + } + rows_per_image + } else { + if copy_depth > 1 { + return Err(TransferError::UnspecifiedRowsPerImage); + } + 0 + }; + + if need_copy_aligned_rows { + let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT as BufferAddress; + + let mut offset_alignment = block_size; + if format.is_depth_stencil_format() { + offset_alignment = 4 + } + if offset % offset_alignment != 0 { + return Err(TransferError::UnalignedBufferOffset(offset)); + } + + if bytes_per_row % bytes_per_row_alignment != 0 { + return Err(TransferError::UnalignedBytesPerRow); + } + } + + let bytes_per_image = bytes_per_row * block_rows_per_image; + + let required_bytes_in_copy = if copy_depth == 0 { + 0 + } else { + let mut required_bytes_in_copy = bytes_per_image * (copy_depth - 1); + if height_in_blocks > 0 { + required_bytes_in_copy += bytes_per_row * (height_in_blocks - 1) + bytes_in_last_row; + } + required_bytes_in_copy + }; + + if offset + required_bytes_in_copy > buffer_size { + return Err(TransferError::BufferOverrun { + start_offset: offset, + end_offset: offset + required_bytes_in_copy, + buffer_size, + side: buffer_side, + }); + } + + Ok((required_bytes_in_copy, bytes_per_image)) +} + +/// WebGPU's [validating texture copy range][vtcr] algorithm. +/// +/// Copied with minor modifications from WebGPU standard. +/// +/// Returns the HAL copy extent and the layer count. +/// +/// [vtcr]: https://gpuweb.github.io/gpuweb/#validating-texture-copy-range +pub(crate) fn validate_texture_copy_range( + texture_copy_view: &ImageCopyTexture, + desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>, + texture_side: CopySide, + copy_size: &Extent3d, +) -> Result<(hal::CopyExtent, u32), TransferError> { + let (block_width, block_height) = desc.format.block_dimensions(); + + let extent_virtual = desc.mip_level_size(texture_copy_view.mip_level).ok_or( + TransferError::InvalidTextureMipLevel { + level: texture_copy_view.mip_level, + total: desc.mip_level_count, + }, + )?; + // physical size can be larger than the virtual + let extent = extent_virtual.physical_size(desc.format); + + if desc.format.is_depth_stencil_format() && *copy_size != extent { + return Err(TransferError::InvalidDepthTextureExtent); + } + + /// Return `Ok` if a run `size` texels long starting at `start_offset` falls + /// entirely within `texture_size`. Otherwise, return an appropriate a`Err`. + fn check_dimension( + dimension: TextureErrorDimension, + side: CopySide, + start_offset: u32, + size: u32, + texture_size: u32, + ) -> Result<(), TransferError> { + // Avoid underflow in the subtraction by checking start_offset against + // texture_size first. + if start_offset <= texture_size && size <= texture_size - start_offset { + Ok(()) + } else { + Err(TransferError::TextureOverrun { + start_offset, + end_offset: start_offset.wrapping_add(size), + texture_size, + dimension, + side, + }) + } + } + + check_dimension( + TextureErrorDimension::X, + texture_side, + texture_copy_view.origin.x, + copy_size.width, + extent.width, + )?; + check_dimension( + TextureErrorDimension::Y, + texture_side, + texture_copy_view.origin.y, + copy_size.height, + extent.height, + )?; + check_dimension( + TextureErrorDimension::Z, + texture_side, + texture_copy_view.origin.z, + copy_size.depth_or_array_layers, + extent.depth_or_array_layers, + )?; + + if texture_copy_view.origin.x % block_width != 0 { + return Err(TransferError::UnalignedCopyOriginX); + } + if texture_copy_view.origin.y % block_height != 0 { + return Err(TransferError::UnalignedCopyOriginY); + } + if copy_size.width % block_width != 0 { + return Err(TransferError::UnalignedCopyWidth); + } + if copy_size.height % block_height != 0 { + return Err(TransferError::UnalignedCopyHeight); + } + + let (depth, array_layer_count) = match desc.dimension { + wgt::TextureDimension::D1 => (1, 1), + wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers), + wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1), + }; + + let copy_extent = hal::CopyExtent { + width: copy_size.width, + height: copy_size.height, + depth, + }; + Ok((copy_extent, array_layer_count)) +} + +fn handle_texture_init<A: HalApi>( + init_kind: MemoryInitKind, + encoder: &mut CommandEncoder<A>, + trackers: &mut Tracker<A>, + texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>, + device: &Device<A>, + copy_texture: &ImageCopyTexture, + copy_size: &Extent3d, + texture: &Arc<Texture<A>>, +) -> Result<(), ClearError> { + let init_action = TextureInitTrackerAction { + texture: texture.clone(), + range: TextureInitRange { + mip_range: copy_texture.mip_level..copy_texture.mip_level + 1, + layer_range: copy_texture.origin.z + ..(copy_texture.origin.z + copy_size.depth_or_array_layers), + }, + kind: init_kind, + }; + + // Register the init action. + let immediate_inits = texture_memory_actions.register_init_action(&{ init_action }); + + // In rare cases we may need to insert an init operation immediately onto the command buffer. + if !immediate_inits.is_empty() { + let cmd_buf_raw = encoder.open()?; + for init in immediate_inits { + clear_texture( + &init.texture, + TextureInitRange { + mip_range: init.mip_level..(init.mip_level + 1), + layer_range: init.layer..(init.layer + 1), + }, + cmd_buf_raw, + &mut trackers.textures, + &device.alignments, + device.zero_buffer.as_ref().unwrap(), + )?; + } + } + + Ok(()) +} + +/// Prepare a transfer's source texture. +/// +/// Ensure the source texture of a transfer is in the right initialization +/// state, and record the state for after the transfer operation. +fn handle_src_texture_init<A: HalApi>( + encoder: &mut CommandEncoder<A>, + trackers: &mut Tracker<A>, + texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>, + device: &Device<A>, + source: &ImageCopyTexture, + copy_size: &Extent3d, + texture: &Arc<Texture<A>>, +) -> Result<(), TransferError> { + handle_texture_init( + MemoryInitKind::NeedsInitializedMemory, + encoder, + trackers, + texture_memory_actions, + device, + source, + copy_size, + texture, + )?; + Ok(()) +} + +/// Prepare a transfer's destination texture. +/// +/// Ensure the destination texture of a transfer is in the right initialization +/// state, and record the state for after the transfer operation. +fn handle_dst_texture_init<A: HalApi>( + encoder: &mut CommandEncoder<A>, + trackers: &mut Tracker<A>, + texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>, + device: &Device<A>, + destination: &ImageCopyTexture, + copy_size: &Extent3d, + texture: &Arc<Texture<A>>, +) -> Result<(), TransferError> { + // Attention: If we don't write full texture subresources, we need to a full + // clear first since we don't track subrects. This means that in rare cases + // even a *destination* texture of a transfer may need an immediate texture + // init. + let dst_init_kind = if has_copy_partial_init_tracker_coverage( + copy_size, + destination.mip_level, + &texture.desc, + ) { + MemoryInitKind::NeedsInitializedMemory + } else { + MemoryInitKind::ImplicitlyInitialized + }; + + handle_texture_init( + dst_init_kind, + encoder, + trackers, + texture_memory_actions, + device, + destination, + copy_size, + texture, + )?; + Ok(()) +} + +impl Global { + pub fn command_encoder_copy_buffer_to_buffer<A: HalApi>( + &self, + command_encoder_id: CommandEncoderId, + source: BufferId, + source_offset: BufferAddress, + destination: BufferId, + destination_offset: BufferAddress, + size: BufferAddress, + ) -> Result<(), CopyError> { + profiling::scope!("CommandEncoder::copy_buffer_to_buffer"); + api_log!( + "CommandEncoder::copy_buffer_to_buffer {source:?} -> {destination:?} {size:?}bytes" + ); + + if source == destination { + return Err(TransferError::SameSourceDestinationBuffer.into()); + } + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?; + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + let device = &cmd_buf.device; + if !device.is_valid() { + return Err(TransferError::InvalidDevice(cmd_buf.device.as_info().id()).into()); + } + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::CopyBufferToBuffer { + src: source, + src_offset: source_offset, + dst: destination, + dst_offset: destination_offset, + size, + }); + } + + let snatch_guard = device.snatchable_lock.read(); + + let (src_buffer, src_pending) = { + let buffer_guard = hub.buffers.read(); + let src_buffer = buffer_guard + .get(source) + .map_err(|_| TransferError::InvalidBuffer(source))?; + cmd_buf_data + .trackers + .buffers + .set_single(src_buffer, hal::BufferUses::COPY_SRC) + .ok_or(TransferError::InvalidBuffer(source))? + }; + let src_raw = src_buffer + .raw + .get(&snatch_guard) + .ok_or(TransferError::InvalidBuffer(source))?; + if !src_buffer.usage.contains(BufferUsages::COPY_SRC) { + return Err(TransferError::MissingCopySrcUsageFlag.into()); + } + // expecting only a single barrier + let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard)); + + let (dst_buffer, dst_pending) = { + let buffer_guard = hub.buffers.read(); + let dst_buffer = buffer_guard + .get(destination) + .map_err(|_| TransferError::InvalidBuffer(destination))?; + cmd_buf_data + .trackers + .buffers + .set_single(dst_buffer, hal::BufferUses::COPY_DST) + .ok_or(TransferError::InvalidBuffer(destination))? + }; + let dst_raw = dst_buffer + .raw + .get(&snatch_guard) + .ok_or(TransferError::InvalidBuffer(destination))?; + if !dst_buffer.usage.contains(BufferUsages::COPY_DST) { + return Err(TransferError::MissingCopyDstUsageFlag(Some(destination), None).into()); + } + let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard)); + + if size % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err(TransferError::UnalignedCopySize(size).into()); + } + if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err(TransferError::UnalignedBufferOffset(source_offset).into()); + } + if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 { + return Err(TransferError::UnalignedBufferOffset(destination_offset).into()); + } + if !device + .downlevel + .flags + .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER) + && (src_buffer.usage.contains(wgt::BufferUsages::INDEX) + || dst_buffer.usage.contains(wgt::BufferUsages::INDEX)) + { + let forbidden_usages = wgt::BufferUsages::VERTEX + | wgt::BufferUsages::UNIFORM + | wgt::BufferUsages::INDIRECT + | wgt::BufferUsages::STORAGE; + if src_buffer.usage.intersects(forbidden_usages) + || dst_buffer.usage.intersects(forbidden_usages) + { + return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags( + wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER, + )) + .into()); + } + } + + let source_end_offset = source_offset + size; + let destination_end_offset = destination_offset + size; + if source_end_offset > src_buffer.size { + return Err(TransferError::BufferOverrun { + start_offset: source_offset, + end_offset: source_end_offset, + buffer_size: src_buffer.size, + side: CopySide::Source, + } + .into()); + } + if destination_end_offset > dst_buffer.size { + return Err(TransferError::BufferOverrun { + start_offset: destination_offset, + end_offset: destination_end_offset, + buffer_size: dst_buffer.size, + side: CopySide::Destination, + } + .into()); + } + + if size == 0 { + log::trace!("Ignoring copy_buffer_to_buffer of size 0"); + return Ok(()); + } + + // Make sure source is initialized memory and mark dest as initialized. + cmd_buf_data.buffer_memory_init_actions.extend( + dst_buffer.initialization_status.read().create_action( + &dst_buffer, + destination_offset..(destination_offset + size), + MemoryInitKind::ImplicitlyInitialized, + ), + ); + cmd_buf_data.buffer_memory_init_actions.extend( + src_buffer.initialization_status.read().create_action( + &src_buffer, + source_offset..(source_offset + size), + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + let region = hal::BufferCopy { + src_offset: source_offset, + dst_offset: destination_offset, + size: wgt::BufferSize::new(size).unwrap(), + }; + let cmd_buf_raw = cmd_buf_data.encoder.open()?; + unsafe { + cmd_buf_raw.transition_buffers(src_barrier.into_iter().chain(dst_barrier)); + cmd_buf_raw.copy_buffer_to_buffer(src_raw, dst_raw, iter::once(region)); + } + Ok(()) + } + + pub fn command_encoder_copy_buffer_to_texture<A: HalApi>( + &self, + command_encoder_id: CommandEncoderId, + source: &ImageCopyBuffer, + destination: &ImageCopyTexture, + copy_size: &Extent3d, + ) -> Result<(), CopyError> { + profiling::scope!("CommandEncoder::copy_buffer_to_texture"); + api_log!( + "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}", + source.buffer, + destination.texture + ); + + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?; + let device = &cmd_buf.device; + if !device.is_valid() { + return Err(TransferError::InvalidDevice(cmd_buf.device.as_info().id()).into()); + } + + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::CopyBufferToTexture { + src: *source, + dst: *destination, + size: *copy_size, + }); + } + + let encoder = &mut cmd_buf_data.encoder; + let tracker = &mut cmd_buf_data.trackers; + let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions; + let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions; + + if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 { + log::trace!("Ignoring copy_buffer_to_texture of size 0"); + return Ok(()); + } + + let dst_texture = hub + .textures + .get(destination.texture) + .map_err(|_| TransferError::InvalidTexture(destination.texture))?; + + let (hal_copy_size, array_layer_count) = validate_texture_copy_range( + destination, + &dst_texture.desc, + CopySide::Destination, + copy_size, + )?; + + let (dst_range, dst_base) = extract_texture_selector(destination, copy_size, &dst_texture)?; + + // Handle texture init *before* dealing with barrier transitions so we + // have an easier time inserting "immediate-inits" that may be required + // by prior discards in rare cases. + handle_dst_texture_init( + encoder, + tracker, + texture_memory_actions, + device, + destination, + copy_size, + &dst_texture, + )?; + + let snatch_guard = device.snatchable_lock.read(); + + let (src_buffer, src_pending) = { + let buffer_guard = hub.buffers.read(); + let src_buffer = buffer_guard + .get(source.buffer) + .map_err(|_| TransferError::InvalidBuffer(source.buffer))?; + tracker + .buffers + .set_single(src_buffer, hal::BufferUses::COPY_SRC) + .ok_or(TransferError::InvalidBuffer(source.buffer))? + }; + let src_raw = src_buffer + .raw + .get(&snatch_guard) + .ok_or(TransferError::InvalidBuffer(source.buffer))?; + if !src_buffer.usage.contains(BufferUsages::COPY_SRC) { + return Err(TransferError::MissingCopySrcUsageFlag.into()); + } + let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard)); + + let dst_pending = tracker + .textures + .set_single(&dst_texture, dst_range, hal::TextureUses::COPY_DST) + .ok_or(TransferError::InvalidTexture(destination.texture))?; + let dst_raw = dst_texture + .raw(&snatch_guard) + .ok_or(TransferError::InvalidTexture(destination.texture))?; + if !dst_texture.desc.usage.contains(TextureUsages::COPY_DST) { + return Err( + TransferError::MissingCopyDstUsageFlag(None, Some(destination.texture)).into(), + ); + } + let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_raw)); + + if !dst_base.aspect.is_one() { + return Err(TransferError::CopyAspectNotOne.into()); + } + + if !conv::is_valid_copy_dst_texture_format(dst_texture.desc.format, destination.aspect) { + return Err(TransferError::CopyToForbiddenTextureFormat { + format: dst_texture.desc.format, + aspect: destination.aspect, + } + .into()); + } + + let (required_buffer_bytes_in_copy, bytes_per_array_layer) = validate_linear_texture_data( + &source.layout, + dst_texture.desc.format, + destination.aspect, + src_buffer.size, + CopySide::Source, + copy_size, + true, + )?; + + if dst_texture.desc.format.is_depth_stencil_format() { + device + .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES) + .map_err(TransferError::from)?; + } + + buffer_memory_init_actions.extend(src_buffer.initialization_status.read().create_action( + &src_buffer, + source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy), + MemoryInitKind::NeedsInitializedMemory, + )); + + let regions = (0..array_layer_count).map(|rel_array_layer| { + let mut texture_base = dst_base.clone(); + texture_base.array_layer += rel_array_layer; + let mut buffer_layout = source.layout; + buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer; + hal::BufferTextureCopy { + buffer_layout, + texture_base, + size: hal_copy_size, + } + }); + + let cmd_buf_raw = encoder.open()?; + unsafe { + cmd_buf_raw.transition_textures(dst_barrier.into_iter()); + cmd_buf_raw.transition_buffers(src_barrier.into_iter()); + cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, regions); + } + Ok(()) + } + + pub fn command_encoder_copy_texture_to_buffer<A: HalApi>( + &self, + command_encoder_id: CommandEncoderId, + source: &ImageCopyTexture, + destination: &ImageCopyBuffer, + copy_size: &Extent3d, + ) -> Result<(), CopyError> { + profiling::scope!("CommandEncoder::copy_texture_to_buffer"); + api_log!( + "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}", + source.texture, + destination.buffer + ); + + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?; + let device = &cmd_buf.device; + if !device.is_valid() { + return Err(TransferError::InvalidDevice(cmd_buf.device.as_info().id()).into()); + } + + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::CopyTextureToBuffer { + src: *source, + dst: *destination, + size: *copy_size, + }); + } + let encoder = &mut cmd_buf_data.encoder; + let tracker = &mut cmd_buf_data.trackers; + let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions; + let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions; + + if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 { + log::trace!("Ignoring copy_texture_to_buffer of size 0"); + return Ok(()); + } + + let src_texture = hub + .textures + .get(source.texture) + .map_err(|_| TransferError::InvalidTexture(source.texture))?; + + let (hal_copy_size, array_layer_count) = + validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?; + + let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?; + + // Handle texture init *before* dealing with barrier transitions so we + // have an easier time inserting "immediate-inits" that may be required + // by prior discards in rare cases. + handle_src_texture_init( + encoder, + tracker, + texture_memory_actions, + device, + source, + copy_size, + &src_texture, + )?; + + let snatch_guard = device.snatchable_lock.read(); + + let src_pending = tracker + .textures + .set_single(&src_texture, src_range, hal::TextureUses::COPY_SRC) + .ok_or(TransferError::InvalidTexture(source.texture))?; + let src_raw = src_texture + .raw(&snatch_guard) + .ok_or(TransferError::InvalidTexture(source.texture))?; + if !src_texture.desc.usage.contains(TextureUsages::COPY_SRC) { + return Err(TransferError::MissingCopySrcUsageFlag.into()); + } + if src_texture.desc.sample_count != 1 { + return Err(TransferError::InvalidSampleCount { + sample_count: src_texture.desc.sample_count, + } + .into()); + } + if source.mip_level >= src_texture.desc.mip_level_count { + return Err(TransferError::InvalidMipLevel { + requested: source.mip_level, + count: src_texture.desc.mip_level_count, + } + .into()); + } + let src_barrier = src_pending.map(|pending| pending.into_hal(src_raw)); + + let (dst_buffer, dst_pending) = { + let buffer_guard = hub.buffers.read(); + let dst_buffer = buffer_guard + .get(destination.buffer) + .map_err(|_| TransferError::InvalidBuffer(destination.buffer))?; + tracker + .buffers + .set_single(dst_buffer, hal::BufferUses::COPY_DST) + .ok_or(TransferError::InvalidBuffer(destination.buffer))? + }; + let dst_raw = dst_buffer + .raw + .get(&snatch_guard) + .ok_or(TransferError::InvalidBuffer(destination.buffer))?; + if !dst_buffer.usage.contains(BufferUsages::COPY_DST) { + return Err( + TransferError::MissingCopyDstUsageFlag(Some(destination.buffer), None).into(), + ); + } + let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard)); + + if !src_base.aspect.is_one() { + return Err(TransferError::CopyAspectNotOne.into()); + } + + if !conv::is_valid_copy_src_texture_format(src_texture.desc.format, source.aspect) { + return Err(TransferError::CopyFromForbiddenTextureFormat { + format: src_texture.desc.format, + aspect: source.aspect, + } + .into()); + } + + let (required_buffer_bytes_in_copy, bytes_per_array_layer) = validate_linear_texture_data( + &destination.layout, + src_texture.desc.format, + source.aspect, + dst_buffer.size, + CopySide::Destination, + copy_size, + true, + )?; + + if src_texture.desc.format.is_depth_stencil_format() { + device + .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES) + .map_err(TransferError::from)?; + } + + buffer_memory_init_actions.extend(dst_buffer.initialization_status.read().create_action( + &dst_buffer, + destination.layout.offset..(destination.layout.offset + required_buffer_bytes_in_copy), + MemoryInitKind::ImplicitlyInitialized, + )); + + let regions = (0..array_layer_count).map(|rel_array_layer| { + let mut texture_base = src_base.clone(); + texture_base.array_layer += rel_array_layer; + let mut buffer_layout = destination.layout; + buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer; + hal::BufferTextureCopy { + buffer_layout, + texture_base, + size: hal_copy_size, + } + }); + let cmd_buf_raw = encoder.open()?; + unsafe { + cmd_buf_raw.transition_buffers(dst_barrier.into_iter()); + cmd_buf_raw.transition_textures(src_barrier.into_iter()); + cmd_buf_raw.copy_texture_to_buffer( + src_raw, + hal::TextureUses::COPY_SRC, + dst_raw, + regions, + ); + } + Ok(()) + } + + pub fn command_encoder_copy_texture_to_texture<A: HalApi>( + &self, + command_encoder_id: CommandEncoderId, + source: &ImageCopyTexture, + destination: &ImageCopyTexture, + copy_size: &Extent3d, + ) -> Result<(), CopyError> { + profiling::scope!("CommandEncoder::copy_texture_to_texture"); + api_log!( + "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}", + source.texture, + destination.texture + ); + + let hub = A::hub(self); + + let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?; + let device = &cmd_buf.device; + if !device.is_valid() { + return Err(TransferError::InvalidDevice(cmd_buf.device.as_info().id()).into()); + } + + let snatch_guard = device.snatchable_lock.read(); + + let mut cmd_buf_data = cmd_buf.data.lock(); + let cmd_buf_data = cmd_buf_data.as_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf_data.commands { + list.push(TraceCommand::CopyTextureToTexture { + src: *source, + dst: *destination, + size: *copy_size, + }); + } + let encoder = &mut cmd_buf_data.encoder; + let tracker = &mut cmd_buf_data.trackers; + let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions; + + if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 { + log::trace!("Ignoring copy_texture_to_texture of size 0"); + return Ok(()); + } + + let src_texture = hub + .textures + .get(source.texture) + .map_err(|_| TransferError::InvalidTexture(source.texture))?; + let dst_texture = hub + .textures + .get(destination.texture) + .map_err(|_| TransferError::InvalidTexture(source.texture))?; + + // src and dst texture format must be copy-compatible + // https://gpuweb.github.io/gpuweb/#copy-compatible + if src_texture.desc.format.remove_srgb_suffix() + != dst_texture.desc.format.remove_srgb_suffix() + { + return Err(TransferError::TextureFormatsNotCopyCompatible { + src_format: src_texture.desc.format, + dst_format: dst_texture.desc.format, + } + .into()); + } + + let (src_copy_size, array_layer_count) = + validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?; + let (dst_copy_size, _) = validate_texture_copy_range( + destination, + &dst_texture.desc, + CopySide::Destination, + copy_size, + )?; + + let (src_range, src_tex_base) = extract_texture_selector(source, copy_size, &src_texture)?; + let (dst_range, dst_tex_base) = + extract_texture_selector(destination, copy_size, &dst_texture)?; + let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format); + let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format); + if src_tex_base.aspect != src_texture_aspects { + return Err(TransferError::CopySrcMissingAspects.into()); + } + if dst_tex_base.aspect != dst_texture_aspects { + return Err(TransferError::CopyDstMissingAspects.into()); + } + + // Handle texture init *before* dealing with barrier transitions so we + // have an easier time inserting "immediate-inits" that may be required + // by prior discards in rare cases. + handle_src_texture_init( + encoder, + tracker, + texture_memory_actions, + device, + source, + copy_size, + &src_texture, + )?; + handle_dst_texture_init( + encoder, + tracker, + texture_memory_actions, + device, + destination, + copy_size, + &dst_texture, + )?; + + let src_pending = cmd_buf_data + .trackers + .textures + .set_single(&src_texture, src_range, hal::TextureUses::COPY_SRC) + .ok_or(TransferError::InvalidTexture(source.texture))?; + let src_raw = src_texture + .raw(&snatch_guard) + .ok_or(TransferError::InvalidTexture(source.texture))?; + if !src_texture.desc.usage.contains(TextureUsages::COPY_SRC) { + return Err(TransferError::MissingCopySrcUsageFlag.into()); + } + + //TODO: try to avoid this the collection. It's needed because both + // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably. + let mut barriers: ArrayVec<_, 2> = src_pending + .map(|pending| pending.into_hal(src_raw)) + .collect(); + + let dst_pending = cmd_buf_data + .trackers + .textures + .set_single(&dst_texture, dst_range, hal::TextureUses::COPY_DST) + .ok_or(TransferError::InvalidTexture(destination.texture))?; + let dst_raw = dst_texture + .raw(&snatch_guard) + .ok_or(TransferError::InvalidTexture(destination.texture))?; + if !dst_texture.desc.usage.contains(TextureUsages::COPY_DST) { + return Err( + TransferError::MissingCopyDstUsageFlag(None, Some(destination.texture)).into(), + ); + } + + barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw))); + + let hal_copy_size = hal::CopyExtent { + width: src_copy_size.width.min(dst_copy_size.width), + height: src_copy_size.height.min(dst_copy_size.height), + depth: src_copy_size.depth.min(dst_copy_size.depth), + }; + let regions = (0..array_layer_count).map(|rel_array_layer| { + let mut src_base = src_tex_base.clone(); + let mut dst_base = dst_tex_base.clone(); + src_base.array_layer += rel_array_layer; + dst_base.array_layer += rel_array_layer; + hal::TextureCopy { + src_base, + dst_base, + size: hal_copy_size, + } + }); + let cmd_buf_raw = cmd_buf_data.encoder.open()?; + unsafe { + cmd_buf_raw.transition_textures(barriers.into_iter()); + cmd_buf_raw.copy_texture_to_texture( + src_raw, + hal::TextureUses::COPY_SRC, + dst_raw, + regions, + ); + } + + Ok(()) + } +} |