summaryrefslogtreecommitdiffstats
path: root/third_party/rust/wgpu-core/src/command
diff options
context:
space:
mode:
Diffstat (limited to 'third_party/rust/wgpu-core/src/command')
-rw-r--r--third_party/rust/wgpu-core/src/command/bind.rs451
-rw-r--r--third_party/rust/wgpu-core/src/command/bundle.rs1722
-rw-r--r--third_party/rust/wgpu-core/src/command/clear.rs493
-rw-r--r--third_party/rust/wgpu-core/src/command/compute.rs1074
-rw-r--r--third_party/rust/wgpu-core/src/command/draw.rs247
-rw-r--r--third_party/rust/wgpu-core/src/command/memory_init.rs342
-rw-r--r--third_party/rust/wgpu-core/src/command/mod.rs732
-rw-r--r--third_party/rust/wgpu-core/src/command/query.rs501
-rw-r--r--third_party/rust/wgpu-core/src/command/render.rs2866
-rw-r--r--third_party/rust/wgpu-core/src/command/transfer.rs1229
10 files changed, 9657 insertions, 0 deletions
diff --git a/third_party/rust/wgpu-core/src/command/bind.rs b/third_party/rust/wgpu-core/src/command/bind.rs
new file mode 100644
index 0000000000..7b2ac54552
--- /dev/null
+++ b/third_party/rust/wgpu-core/src/command/bind.rs
@@ -0,0 +1,451 @@
+use std::sync::Arc;
+
+use crate::{
+ binding_model::{BindGroup, LateMinBufferBindingSizeMismatch, PipelineLayout},
+ device::SHADER_STAGE_COUNT,
+ hal_api::HalApi,
+ id::BindGroupId,
+ pipeline::LateSizedBufferGroup,
+ resource::Resource,
+};
+
+use arrayvec::ArrayVec;
+
+type BindGroupMask = u8;
+
+mod compat {
+ use arrayvec::ArrayVec;
+
+ use crate::{binding_model::BindGroupLayout, device::bgl, hal_api::HalApi, resource::Resource};
+ use std::{ops::Range, sync::Arc};
+
+ #[derive(Debug, Clone)]
+ struct Entry<A: HalApi> {
+ assigned: Option<Arc<BindGroupLayout<A>>>,
+ expected: Option<Arc<BindGroupLayout<A>>>,
+ }
+
+ impl<A: HalApi> Entry<A> {
+ fn empty() -> Self {
+ Self {
+ assigned: None,
+ expected: None,
+ }
+ }
+ fn is_active(&self) -> bool {
+ self.assigned.is_some() && self.expected.is_some()
+ }
+
+ fn is_valid(&self) -> bool {
+ if self.expected.is_none() {
+ return true;
+ }
+ if let Some(expected_bgl) = self.expected.as_ref() {
+ if let Some(assigned_bgl) = self.assigned.as_ref() {
+ if expected_bgl.is_equal(assigned_bgl) {
+ return true;
+ }
+ }
+ }
+ false
+ }
+
+ fn is_incompatible(&self) -> bool {
+ self.expected.is_none() || !self.is_valid()
+ }
+
+ // Describe how bind group layouts are incompatible, for validation
+ // error message.
+ fn bgl_diff(&self) -> Vec<String> {
+ let mut diff = Vec::new();
+
+ if let Some(expected_bgl) = self.expected.as_ref() {
+ let expected_bgl_type = match expected_bgl.origin {
+ bgl::Origin::Derived => "implicit",
+ bgl::Origin::Pool => "explicit",
+ };
+ let expected_label = expected_bgl.label();
+ diff.push(format!(
+ "Should be compatible an with an {expected_bgl_type} bind group layout {}",
+ if expected_label.is_empty() {
+ "without label".to_string()
+ } else {
+ format!("with label = `{}`", expected_label)
+ }
+ ));
+ if let Some(assigned_bgl) = self.assigned.as_ref() {
+ let assigned_bgl_type = match assigned_bgl.origin {
+ bgl::Origin::Derived => "implicit",
+ bgl::Origin::Pool => "explicit",
+ };
+ let assigned_label = assigned_bgl.label();
+ diff.push(format!(
+ "Assigned {assigned_bgl_type} bind group layout {}",
+ if assigned_label.is_empty() {
+ "without label".to_string()
+ } else {
+ format!("with label = `{}`", assigned_label)
+ }
+ ));
+ for (id, e_entry) in expected_bgl.entries.iter() {
+ if let Some(a_entry) = assigned_bgl.entries.get(*id) {
+ if a_entry.binding != e_entry.binding {
+ diff.push(format!(
+ "Entry {id} binding expected {}, got {}",
+ e_entry.binding, a_entry.binding
+ ));
+ }
+ if a_entry.count != e_entry.count {
+ diff.push(format!(
+ "Entry {id} count expected {:?}, got {:?}",
+ e_entry.count, a_entry.count
+ ));
+ }
+ if a_entry.ty != e_entry.ty {
+ diff.push(format!(
+ "Entry {id} type expected {:?}, got {:?}",
+ e_entry.ty, a_entry.ty
+ ));
+ }
+ if a_entry.visibility != e_entry.visibility {
+ diff.push(format!(
+ "Entry {id} visibility expected {:?}, got {:?}",
+ e_entry.visibility, a_entry.visibility
+ ));
+ }
+ } else {
+ diff.push(format!(
+ "Entry {id} not found in assigned bind group layout"
+ ))
+ }
+ }
+
+ assigned_bgl.entries.iter().for_each(|(id, _e_entry)| {
+ if !expected_bgl.entries.contains_key(*id) {
+ diff.push(format!(
+ "Entry {id} not found in expected bind group layout"
+ ))
+ }
+ });
+
+ if expected_bgl.origin != assigned_bgl.origin {
+ diff.push(format!("Expected {expected_bgl_type} bind group layout, got {assigned_bgl_type}"))
+ }
+ } else {
+ diff.push("Assigned bind group layout not found (internal error)".to_owned());
+ }
+ } else {
+ diff.push("Expected bind group layout not found (internal error)".to_owned());
+ }
+
+ diff
+ }
+ }
+
+ #[derive(Debug, Default)]
+ pub(crate) struct BoundBindGroupLayouts<A: HalApi> {
+ entries: ArrayVec<Entry<A>, { hal::MAX_BIND_GROUPS }>,
+ }
+
+ impl<A: HalApi> BoundBindGroupLayouts<A> {
+ pub fn new() -> Self {
+ Self {
+ entries: (0..hal::MAX_BIND_GROUPS).map(|_| Entry::empty()).collect(),
+ }
+ }
+ fn make_range(&self, start_index: usize) -> Range<usize> {
+ // find first incompatible entry
+ let end = self
+ .entries
+ .iter()
+ .position(|e| e.is_incompatible())
+ .unwrap_or(self.entries.len());
+ start_index..end.max(start_index)
+ }
+
+ pub fn update_expectations(
+ &mut self,
+ expectations: &[Arc<BindGroupLayout<A>>],
+ ) -> Range<usize> {
+ let start_index = self
+ .entries
+ .iter()
+ .zip(expectations)
+ .position(|(e, expect)| {
+ e.expected.is_none() || !e.expected.as_ref().unwrap().is_equal(expect)
+ })
+ .unwrap_or(expectations.len());
+ for (e, expect) in self.entries[start_index..]
+ .iter_mut()
+ .zip(expectations[start_index..].iter())
+ {
+ e.expected = Some(expect.clone());
+ }
+ for e in self.entries[expectations.len()..].iter_mut() {
+ e.expected = None;
+ }
+ self.make_range(start_index)
+ }
+
+ pub fn assign(&mut self, index: usize, value: Arc<BindGroupLayout<A>>) -> Range<usize> {
+ self.entries[index].assigned = Some(value);
+ self.make_range(index)
+ }
+
+ pub fn list_active(&self) -> impl Iterator<Item = usize> + '_ {
+ self.entries
+ .iter()
+ .enumerate()
+ .filter_map(|(i, e)| if e.is_active() { Some(i) } else { None })
+ }
+
+ pub fn invalid_mask(&self) -> super::BindGroupMask {
+ self.entries.iter().enumerate().fold(0, |mask, (i, entry)| {
+ if entry.is_valid() {
+ mask
+ } else {
+ mask | 1u8 << i
+ }
+ })
+ }
+
+ pub fn bgl_diff(&self) -> Vec<String> {
+ for e in &self.entries {
+ if !e.is_valid() {
+ return e.bgl_diff();
+ }
+ }
+ vec![String::from("No differences detected? (internal error)")]
+ }
+ }
+}
+
+#[derive(Debug)]
+struct LateBufferBinding {
+ shader_expect_size: wgt::BufferAddress,
+ bound_size: wgt::BufferAddress,
+}
+
+#[derive(Debug)]
+pub(super) struct EntryPayload<A: HalApi> {
+ pub(super) group: Option<Arc<BindGroup<A>>>,
+ pub(super) dynamic_offsets: Vec<wgt::DynamicOffset>,
+ late_buffer_bindings: Vec<LateBufferBinding>,
+ /// Since `LateBufferBinding` may contain information about the bindings
+ /// not used by the pipeline, we need to know when to stop validating.
+ pub(super) late_bindings_effective_count: usize,
+}
+
+impl<A: HalApi> Default for EntryPayload<A> {
+ fn default() -> Self {
+ Self {
+ group: None,
+ dynamic_offsets: Default::default(),
+ late_buffer_bindings: Default::default(),
+ late_bindings_effective_count: Default::default(),
+ }
+ }
+}
+
+impl<A: HalApi> EntryPayload<A> {
+ fn reset(&mut self) {
+ self.group = None;
+ self.dynamic_offsets.clear();
+ self.late_buffer_bindings.clear();
+ self.late_bindings_effective_count = 0;
+ }
+}
+
+#[derive(Debug, Default)]
+pub(super) struct Binder<A: HalApi> {
+ pub(super) pipeline_layout: Option<Arc<PipelineLayout<A>>>,
+ manager: compat::BoundBindGroupLayouts<A>,
+ payloads: [EntryPayload<A>; hal::MAX_BIND_GROUPS],
+}
+
+impl<A: HalApi> Binder<A> {
+ pub(super) fn new() -> Self {
+ Self {
+ pipeline_layout: None,
+ manager: compat::BoundBindGroupLayouts::new(),
+ payloads: Default::default(),
+ }
+ }
+ pub(super) fn reset(&mut self) {
+ self.pipeline_layout = None;
+ self.manager = compat::BoundBindGroupLayouts::new();
+ for payload in self.payloads.iter_mut() {
+ payload.reset();
+ }
+ }
+
+ pub(super) fn change_pipeline_layout<'a>(
+ &'a mut self,
+ new: &Arc<PipelineLayout<A>>,
+ late_sized_buffer_groups: &[LateSizedBufferGroup],
+ ) -> (usize, &'a [EntryPayload<A>]) {
+ let old_id_opt = self.pipeline_layout.replace(new.clone());
+
+ let mut bind_range = self.manager.update_expectations(&new.bind_group_layouts);
+
+ // Update the buffer binding sizes that are required by shaders.
+ for (payload, late_group) in self.payloads.iter_mut().zip(late_sized_buffer_groups) {
+ payload.late_bindings_effective_count = late_group.shader_sizes.len();
+ for (late_binding, &shader_expect_size) in payload
+ .late_buffer_bindings
+ .iter_mut()
+ .zip(late_group.shader_sizes.iter())
+ {
+ late_binding.shader_expect_size = shader_expect_size;
+ }
+ if late_group.shader_sizes.len() > payload.late_buffer_bindings.len() {
+ for &shader_expect_size in
+ late_group.shader_sizes[payload.late_buffer_bindings.len()..].iter()
+ {
+ payload.late_buffer_bindings.push(LateBufferBinding {
+ shader_expect_size,
+ bound_size: 0,
+ });
+ }
+ }
+ }
+
+ if let Some(old) = old_id_opt {
+ // root constants are the base compatibility property
+ if old.push_constant_ranges != new.push_constant_ranges {
+ bind_range.start = 0;
+ }
+ }
+
+ (bind_range.start, &self.payloads[bind_range])
+ }
+
+ pub(super) fn assign_group<'a>(
+ &'a mut self,
+ index: usize,
+ bind_group: &Arc<BindGroup<A>>,
+ offsets: &[wgt::DynamicOffset],
+ ) -> &'a [EntryPayload<A>] {
+ let bind_group_id = bind_group.as_info().id();
+ log::trace!("\tBinding [{}] = group {:?}", index, bind_group_id);
+ debug_assert_eq!(A::VARIANT, bind_group_id.backend());
+
+ let payload = &mut self.payloads[index];
+ payload.group = Some(bind_group.clone());
+ payload.dynamic_offsets.clear();
+ payload.dynamic_offsets.extend_from_slice(offsets);
+
+ // Fill out the actual binding sizes for buffers,
+ // whose layout doesn't specify `min_binding_size`.
+ for (late_binding, late_size) in payload
+ .late_buffer_bindings
+ .iter_mut()
+ .zip(bind_group.late_buffer_binding_sizes.iter())
+ {
+ late_binding.bound_size = late_size.get();
+ }
+ if bind_group.late_buffer_binding_sizes.len() > payload.late_buffer_bindings.len() {
+ for late_size in
+ bind_group.late_buffer_binding_sizes[payload.late_buffer_bindings.len()..].iter()
+ {
+ payload.late_buffer_bindings.push(LateBufferBinding {
+ shader_expect_size: 0,
+ bound_size: late_size.get(),
+ });
+ }
+ }
+
+ let bind_range = self.manager.assign(index, bind_group.layout.clone());
+ &self.payloads[bind_range]
+ }
+
+ pub(super) fn list_active(&self) -> impl Iterator<Item = BindGroupId> + '_ {
+ let payloads = &self.payloads;
+ self.manager
+ .list_active()
+ .map(move |index| payloads[index].group.as_ref().unwrap().as_info().id())
+ }
+
+ pub(super) fn invalid_mask(&self) -> BindGroupMask {
+ self.manager.invalid_mask()
+ }
+
+ pub(super) fn bgl_diff(&self) -> Vec<String> {
+ self.manager.bgl_diff()
+ }
+
+ /// Scan active buffer bindings corresponding to layouts without `min_binding_size` specified.
+ pub(super) fn check_late_buffer_bindings(
+ &self,
+ ) -> Result<(), LateMinBufferBindingSizeMismatch> {
+ for group_index in self.manager.list_active() {
+ let payload = &self.payloads[group_index];
+ for (compact_index, late_binding) in payload.late_buffer_bindings
+ [..payload.late_bindings_effective_count]
+ .iter()
+ .enumerate()
+ {
+ if late_binding.bound_size < late_binding.shader_expect_size {
+ return Err(LateMinBufferBindingSizeMismatch {
+ group_index: group_index as u32,
+ compact_index,
+ shader_size: late_binding.shader_expect_size,
+ bound_size: late_binding.bound_size,
+ });
+ }
+ }
+ }
+ Ok(())
+ }
+}
+
+struct PushConstantChange {
+ stages: wgt::ShaderStages,
+ offset: u32,
+ enable: bool,
+}
+
+/// Break up possibly overlapping push constant ranges into a set of
+/// non-overlapping ranges which contain all the stage flags of the
+/// original ranges. This allows us to zero out (or write any value)
+/// to every possible value.
+pub fn compute_nonoverlapping_ranges(
+ ranges: &[wgt::PushConstantRange],
+) -> ArrayVec<wgt::PushConstantRange, { SHADER_STAGE_COUNT * 2 }> {
+ if ranges.is_empty() {
+ return ArrayVec::new();
+ }
+ debug_assert!(ranges.len() <= SHADER_STAGE_COUNT);
+
+ let mut breaks: ArrayVec<PushConstantChange, { SHADER_STAGE_COUNT * 2 }> = ArrayVec::new();
+ for range in ranges {
+ breaks.push(PushConstantChange {
+ stages: range.stages,
+ offset: range.range.start,
+ enable: true,
+ });
+ breaks.push(PushConstantChange {
+ stages: range.stages,
+ offset: range.range.end,
+ enable: false,
+ });
+ }
+ breaks.sort_unstable_by_key(|change| change.offset);
+
+ let mut output_ranges = ArrayVec::new();
+ let mut position = 0_u32;
+ let mut stages = wgt::ShaderStages::NONE;
+
+ for bk in breaks {
+ if bk.offset - position > 0 && !stages.is_empty() {
+ output_ranges.push(wgt::PushConstantRange {
+ stages,
+ range: position..bk.offset,
+ })
+ }
+ position = bk.offset;
+ stages.set(bk.stages, bk.enable);
+ }
+
+ output_ranges
+}
diff --git a/third_party/rust/wgpu-core/src/command/bundle.rs b/third_party/rust/wgpu-core/src/command/bundle.rs
new file mode 100644
index 0000000000..9d80c62f85
--- /dev/null
+++ b/third_party/rust/wgpu-core/src/command/bundle.rs
@@ -0,0 +1,1722 @@
+/*! Render Bundles
+
+A render bundle is a prerecorded sequence of commands that can be replayed on a
+command encoder with a single call. A single bundle can replayed any number of
+times, on different encoders. Constructing a render bundle lets `wgpu` validate
+and analyze its commands up front, so that replaying a bundle can be more
+efficient than simply re-recording its commands each time.
+
+Not all commands are available in bundles; for example, a render bundle may not
+contain a [`RenderCommand::SetViewport`] command.
+
+Most of `wgpu`'s backend graphics APIs have something like bundles. For example,
+Vulkan calls them "secondary command buffers", and Metal calls them "indirect
+command buffers". Although we plan to take advantage of these platform features
+at some point in the future, for now `wgpu`'s implementation of render bundles
+does not use them: at the hal level, `wgpu` render bundles just replay the
+commands.
+
+## Render Bundle Isolation
+
+One important property of render bundles is that the draw calls in a render
+bundle depend solely on the pipeline and state established within the render
+bundle itself. A draw call in a bundle will never use a vertex buffer, say, that
+was set in the `RenderPass` before executing the bundle. We call this property
+'isolation', in that a render bundle is somewhat isolated from the passes that
+use it.
+
+Render passes are also isolated from the effects of bundles. After executing a
+render bundle, a render pass's pipeline, bind groups, and vertex and index
+buffers are are unset, so the bundle cannot affect later draw calls in the pass.
+
+A render pass is not fully isolated from a bundle's effects on push constant
+values. Draw calls following a bundle's execution will see whatever values the
+bundle writes to push constant storage. Setting a pipeline initializes any push
+constant storage it could access to zero, and this initialization may also be
+visible after bundle execution.
+
+## Render Bundle Lifecycle
+
+To create a render bundle:
+
+1) Create a [`RenderBundleEncoder`] by calling
+ [`Global::device_create_render_bundle_encoder`][Gdcrbe].
+
+2) Record commands in the `RenderBundleEncoder` using functions from the
+ [`bundle_ffi`] module.
+
+3) Call [`Global::render_bundle_encoder_finish`][Grbef], which analyzes and cleans up
+ the command stream and returns a `RenderBundleId`.
+
+4) Then, any number of times, call [`wgpu_render_pass_execute_bundles`][wrpeb] to
+ execute the bundle as part of some render pass.
+
+## Implementation
+
+The most complex part of render bundles is the "finish" step, mostly implemented
+in [`RenderBundleEncoder::finish`]. This consumes the commands stored in the
+encoder's [`BasePass`], while validating everything, tracking the state,
+dropping redundant or unnecessary commands, and presenting the results as a new
+[`RenderBundle`]. It doesn't actually execute any commands.
+
+This step also enforces the 'isolation' property mentioned above: every draw
+call is checked to ensure that the resources it uses on were established since
+the last time the pipeline was set. This means the bundle can be executed
+verbatim without any state tracking.
+
+### Execution
+
+When the bundle is used in an actual render pass, `RenderBundle::execute` is
+called. It goes through the commands and issues them into the native command
+buffer. Thanks to isolation, it doesn't track any bind group invalidations or
+index format changes.
+
+[Gdcrbe]: crate::global::Global::device_create_render_bundle_encoder
+[Grbef]: crate::global::Global::render_bundle_encoder_finish
+[wrpeb]: crate::command::render_ffi::wgpu_render_pass_execute_bundles
+!*/
+
+#![allow(clippy::reversed_empty_ranges)]
+
+#[cfg(feature = "trace")]
+use crate::device::trace;
+use crate::{
+ binding_model::{buffer_binding_type_alignment, BindGroup, BindGroupLayout, PipelineLayout},
+ command::{
+ BasePass, BindGroupStateChange, ColorAttachmentError, DrawError, MapPassErr,
+ PassErrorScope, RenderCommand, RenderCommandError, StateChange,
+ },
+ conv,
+ device::{
+ AttachmentData, Device, DeviceError, MissingDownlevelFlags,
+ RenderPassCompatibilityCheckType, RenderPassContext, SHADER_STAGE_COUNT,
+ },
+ error::{ErrorFormatter, PrettyError},
+ hal_api::HalApi,
+ hub::Hub,
+ id,
+ init_tracker::{BufferInitTrackerAction, MemoryInitKind, TextureInitTrackerAction},
+ pipeline::{PipelineFlags, RenderPipeline, VertexStep},
+ resource::{Resource, ResourceInfo, ResourceType},
+ resource_log,
+ track::RenderBundleScope,
+ validation::check_buffer_usage,
+ Label, LabelHelpers,
+};
+use arrayvec::ArrayVec;
+
+use std::{borrow::Cow, mem, num::NonZeroU32, ops::Range, sync::Arc};
+use thiserror::Error;
+
+use hal::CommandEncoder as _;
+
+/// https://gpuweb.github.io/gpuweb/#dom-gpurendercommandsmixin-draw
+fn validate_draw(
+ vertex: &[Option<VertexState>],
+ step: &[VertexStep],
+ first_vertex: u32,
+ vertex_count: u32,
+ first_instance: u32,
+ instance_count: u32,
+) -> Result<(), DrawError> {
+ let vertices_end = first_vertex as u64 + vertex_count as u64;
+ let instances_end = first_instance as u64 + instance_count as u64;
+
+ for (idx, (vbs, step)) in vertex.iter().zip(step).enumerate() {
+ let Some(vbs) = vbs else {
+ continue;
+ };
+
+ let stride_count = match step.mode {
+ wgt::VertexStepMode::Vertex => vertices_end,
+ wgt::VertexStepMode::Instance => instances_end,
+ };
+
+ if stride_count == 0 {
+ continue;
+ }
+
+ let offset = (stride_count - 1) * step.stride + step.last_stride;
+ let limit = vbs.range.end - vbs.range.start;
+ if offset > limit {
+ return Err(DrawError::VertexOutOfBounds {
+ step_mode: step.mode,
+ offset,
+ limit,
+ slot: idx as u32,
+ });
+ }
+ }
+
+ Ok(())
+}
+
+// See https://gpuweb.github.io/gpuweb/#dom-gpurendercommandsmixin-drawindexed
+fn validate_indexed_draw(
+ vertex: &[Option<VertexState>],
+ step: &[VertexStep],
+ index_state: &IndexState,
+ first_index: u32,
+ index_count: u32,
+ first_instance: u32,
+ instance_count: u32,
+) -> Result<(), DrawError> {
+ let last_index = first_index as u64 + index_count as u64;
+ let index_limit = index_state.limit();
+ if last_index <= index_limit {
+ return Err(DrawError::IndexBeyondLimit {
+ last_index,
+ index_limit,
+ });
+ }
+
+ let stride_count = first_instance as u64 + instance_count as u64;
+ for (idx, (vbs, step)) in vertex.iter().zip(step).enumerate() {
+ let Some(vbs) = vbs else {
+ continue;
+ };
+
+ if stride_count == 0 || step.mode != wgt::VertexStepMode::Instance {
+ continue;
+ }
+
+ let offset = (stride_count - 1) * step.stride + step.last_stride;
+ let limit = vbs.range.end - vbs.range.start;
+ if offset > limit {
+ return Err(DrawError::VertexOutOfBounds {
+ step_mode: step.mode,
+ offset,
+ limit,
+ slot: idx as u32,
+ });
+ }
+ }
+
+ Ok(())
+}
+
+/// Describes a [`RenderBundleEncoder`].
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub struct RenderBundleEncoderDescriptor<'a> {
+ /// Debug label of the render bundle encoder.
+ ///
+ /// This will show up in graphics debuggers for easy identification.
+ pub label: Label<'a>,
+ /// The formats of the color attachments that this render bundle is capable
+ /// to rendering to.
+ ///
+ /// This must match the formats of the color attachments in the
+ /// renderpass this render bundle is executed in.
+ pub color_formats: Cow<'a, [Option<wgt::TextureFormat>]>,
+ /// Information about the depth attachment that this render bundle is
+ /// capable to rendering to.
+ ///
+ /// The format must match the format of the depth attachments in the
+ /// renderpass this render bundle is executed in.
+ pub depth_stencil: Option<wgt::RenderBundleDepthStencil>,
+ /// Sample count this render bundle is capable of rendering to.
+ ///
+ /// This must match the pipelines and the renderpasses it is used in.
+ pub sample_count: u32,
+ /// If this render bundle will rendering to multiple array layers in the
+ /// attachments at the same time.
+ pub multiview: Option<NonZeroU32>,
+}
+
+#[derive(Debug)]
+#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
+pub struct RenderBundleEncoder {
+ base: BasePass<RenderCommand>,
+ parent_id: id::DeviceId,
+ pub(crate) context: RenderPassContext,
+ pub(crate) is_depth_read_only: bool,
+ pub(crate) is_stencil_read_only: bool,
+
+ // Resource binding dedupe state.
+ #[cfg_attr(feature = "serde", serde(skip))]
+ current_bind_groups: BindGroupStateChange,
+ #[cfg_attr(feature = "serde", serde(skip))]
+ current_pipeline: StateChange<id::RenderPipelineId>,
+}
+
+impl RenderBundleEncoder {
+ pub fn new(
+ desc: &RenderBundleEncoderDescriptor,
+ parent_id: id::DeviceId,
+ base: Option<BasePass<RenderCommand>>,
+ ) -> Result<Self, CreateRenderBundleError> {
+ let (is_depth_read_only, is_stencil_read_only) = match desc.depth_stencil {
+ Some(ds) => {
+ let aspects = hal::FormatAspects::from(ds.format);
+ (
+ !aspects.contains(hal::FormatAspects::DEPTH) || ds.depth_read_only,
+ !aspects.contains(hal::FormatAspects::STENCIL) || ds.stencil_read_only,
+ )
+ }
+ // There's no depth/stencil attachment, so these values just don't
+ // matter. Choose the most accommodating value, to simplify
+ // validation.
+ None => (true, true),
+ };
+
+ //TODO: validate that attachment formats are renderable,
+ // have expected aspects, support multisampling.
+ Ok(Self {
+ base: base.unwrap_or_else(|| BasePass::new(&desc.label)),
+ parent_id,
+ context: RenderPassContext {
+ attachments: AttachmentData {
+ colors: if desc.color_formats.len() > hal::MAX_COLOR_ATTACHMENTS {
+ return Err(CreateRenderBundleError::ColorAttachment(
+ ColorAttachmentError::TooMany {
+ given: desc.color_formats.len(),
+ limit: hal::MAX_COLOR_ATTACHMENTS,
+ },
+ ));
+ } else {
+ desc.color_formats.iter().cloned().collect()
+ },
+ resolves: ArrayVec::new(),
+ depth_stencil: desc.depth_stencil.map(|ds| ds.format),
+ },
+ sample_count: {
+ let sc = desc.sample_count;
+ if sc == 0 || sc > 32 || !conv::is_power_of_two_u32(sc) {
+ return Err(CreateRenderBundleError::InvalidSampleCount(sc));
+ }
+ sc
+ },
+ multiview: desc.multiview,
+ },
+
+ is_depth_read_only,
+ is_stencil_read_only,
+ current_bind_groups: BindGroupStateChange::new(),
+ current_pipeline: StateChange::new(),
+ })
+ }
+
+ pub fn dummy(parent_id: id::DeviceId) -> Self {
+ Self {
+ base: BasePass::new(&None),
+ parent_id,
+ context: RenderPassContext {
+ attachments: AttachmentData {
+ colors: ArrayVec::new(),
+ resolves: ArrayVec::new(),
+ depth_stencil: None,
+ },
+ sample_count: 0,
+ multiview: None,
+ },
+ is_depth_read_only: false,
+ is_stencil_read_only: false,
+
+ current_bind_groups: BindGroupStateChange::new(),
+ current_pipeline: StateChange::new(),
+ }
+ }
+
+ #[cfg(feature = "trace")]
+ pub(crate) fn to_base_pass(&self) -> BasePass<RenderCommand> {
+ BasePass::from_ref(self.base.as_ref())
+ }
+
+ pub fn parent(&self) -> id::DeviceId {
+ self.parent_id
+ }
+
+ /// Convert this encoder's commands into a [`RenderBundle`].
+ ///
+ /// We want executing a [`RenderBundle`] to be quick, so we take
+ /// this opportunity to clean up the [`RenderBundleEncoder`]'s
+ /// command stream and gather metadata about it that will help
+ /// keep [`ExecuteBundle`] simple and fast. We remove redundant
+ /// commands (along with their side data), note resource usage,
+ /// and accumulate buffer and texture initialization actions.
+ ///
+ /// [`ExecuteBundle`]: RenderCommand::ExecuteBundle
+ pub(crate) fn finish<A: HalApi>(
+ self,
+ desc: &RenderBundleDescriptor,
+ device: &Arc<Device<A>>,
+ hub: &Hub<A>,
+ ) -> Result<RenderBundle<A>, RenderBundleError> {
+ let bind_group_guard = hub.bind_groups.read();
+ let pipeline_guard = hub.render_pipelines.read();
+ let query_set_guard = hub.query_sets.read();
+ let buffer_guard = hub.buffers.read();
+ let texture_guard = hub.textures.read();
+
+ let mut state = State {
+ trackers: RenderBundleScope::new(
+ &*buffer_guard,
+ &*texture_guard,
+ &*bind_group_guard,
+ &*pipeline_guard,
+ &*query_set_guard,
+ ),
+ pipeline: None,
+ bind: (0..hal::MAX_BIND_GROUPS).map(|_| None).collect(),
+ vertex: (0..hal::MAX_VERTEX_BUFFERS).map(|_| None).collect(),
+ index: None,
+ flat_dynamic_offsets: Vec::new(),
+ };
+ let mut commands = Vec::new();
+ let mut buffer_memory_init_actions = Vec::new();
+ let mut texture_memory_init_actions = Vec::new();
+
+ let base = self.base.as_ref();
+ let mut next_dynamic_offset = 0;
+
+ for &command in base.commands {
+ match command {
+ RenderCommand::SetBindGroup {
+ index,
+ num_dynamic_offsets,
+ bind_group_id,
+ } => {
+ let scope = PassErrorScope::SetBindGroup(bind_group_id);
+
+ let bind_group = state
+ .trackers
+ .bind_groups
+ .write()
+ .add_single(&*bind_group_guard, bind_group_id)
+ .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id))
+ .map_pass_err(scope)?;
+ self.check_valid_to_use(bind_group.device.info.id())
+ .map_pass_err(scope)?;
+
+ let max_bind_groups = device.limits.max_bind_groups;
+ if index >= max_bind_groups {
+ return Err(RenderCommandError::BindGroupIndexOutOfRange {
+ index,
+ max: max_bind_groups,
+ })
+ .map_pass_err(scope);
+ }
+
+ // Identify the next `num_dynamic_offsets` entries from `base.dynamic_offsets`.
+ let num_dynamic_offsets = num_dynamic_offsets;
+ let offsets_range =
+ next_dynamic_offset..next_dynamic_offset + num_dynamic_offsets;
+ next_dynamic_offset = offsets_range.end;
+ let offsets = &base.dynamic_offsets[offsets_range.clone()];
+
+ if bind_group.dynamic_binding_info.len() != offsets.len() {
+ return Err(RenderCommandError::InvalidDynamicOffsetCount {
+ actual: offsets.len(),
+ expected: bind_group.dynamic_binding_info.len(),
+ })
+ .map_pass_err(scope);
+ }
+
+ // Check for misaligned offsets.
+ for (offset, info) in offsets
+ .iter()
+ .map(|offset| *offset as wgt::BufferAddress)
+ .zip(bind_group.dynamic_binding_info.iter())
+ {
+ let (alignment, limit_name) =
+ buffer_binding_type_alignment(&device.limits, info.binding_type);
+ if offset % alignment as u64 != 0 {
+ return Err(RenderCommandError::UnalignedBufferOffset(
+ offset, limit_name, alignment,
+ ))
+ .map_pass_err(scope);
+ }
+ }
+
+ buffer_memory_init_actions.extend_from_slice(&bind_group.used_buffer_ranges);
+ texture_memory_init_actions.extend_from_slice(&bind_group.used_texture_ranges);
+
+ state.set_bind_group(index, bind_group_guard.get(bind_group_id).as_ref().unwrap(), &bind_group.layout, offsets_range);
+ unsafe {
+ state
+ .trackers
+ .merge_bind_group(&bind_group.used)
+ .map_pass_err(scope)?
+ };
+ //Note: stateless trackers are not merged: the lifetime reference
+ // is held to the bind group itself.
+ }
+ RenderCommand::SetPipeline(pipeline_id) => {
+ let scope = PassErrorScope::SetPipelineRender(pipeline_id);
+
+ let pipeline = state
+ .trackers
+ .render_pipelines
+ .write()
+ .add_single(&*pipeline_guard, pipeline_id)
+ .ok_or(RenderCommandError::InvalidPipeline(pipeline_id))
+ .map_pass_err(scope)?;
+ self.check_valid_to_use(pipeline.device.info.id())
+ .map_pass_err(scope)?;
+
+ self.context
+ .check_compatible(&pipeline.pass_context, RenderPassCompatibilityCheckType::RenderPipeline)
+ .map_err(RenderCommandError::IncompatiblePipelineTargets)
+ .map_pass_err(scope)?;
+
+ if (pipeline.flags.contains(PipelineFlags::WRITES_DEPTH)
+ && self.is_depth_read_only)
+ || (pipeline.flags.contains(PipelineFlags::WRITES_STENCIL)
+ && self.is_stencil_read_only)
+ {
+ return Err(RenderCommandError::IncompatiblePipelineRods)
+ .map_pass_err(scope);
+ }
+
+ let pipeline_state = PipelineState::new(pipeline);
+
+ commands.push(command);
+
+ // If this pipeline uses push constants, zero out their values.
+ if let Some(iter) = pipeline_state.zero_push_constants() {
+ commands.extend(iter)
+ }
+
+ state.invalidate_bind_groups(&pipeline_state, &pipeline.layout);
+ state.pipeline = Some(pipeline_state);
+ }
+ RenderCommand::SetIndexBuffer {
+ buffer_id,
+ index_format,
+ offset,
+ size,
+ } => {
+ let scope = PassErrorScope::SetIndexBuffer(buffer_id);
+ let buffer = state
+ .trackers
+ .buffers
+ .write()
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDEX)
+ .map_pass_err(scope)?;
+ self.check_valid_to_use(buffer.device.info.id())
+ .map_pass_err(scope)?;
+ check_buffer_usage(buffer.usage, wgt::BufferUsages::INDEX)
+ .map_pass_err(scope)?;
+
+ let end = match size {
+ Some(s) => offset + s.get(),
+ None => buffer.size,
+ };
+ buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action(
+ buffer,
+ offset..end,
+ MemoryInitKind::NeedsInitializedMemory,
+ ));
+ state.set_index_buffer(buffer_id, index_format, offset..end);
+ }
+ RenderCommand::SetVertexBuffer {
+ slot,
+ buffer_id,
+ offset,
+ size,
+ } => {
+ let scope = PassErrorScope::SetVertexBuffer(buffer_id);
+
+ let max_vertex_buffers = device.limits.max_vertex_buffers;
+ if slot >= max_vertex_buffers {
+ return Err(RenderCommandError::VertexBufferIndexOutOfRange {
+ index: slot,
+ max: max_vertex_buffers,
+ })
+ .map_pass_err(scope);
+ }
+
+ let buffer = state
+ .trackers
+ .buffers
+ .write()
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX)
+ .map_pass_err(scope)?;
+ self.check_valid_to_use(buffer.device.info.id())
+ .map_pass_err(scope)?;
+ check_buffer_usage(buffer.usage, wgt::BufferUsages::VERTEX)
+ .map_pass_err(scope)?;
+
+ let end = match size {
+ Some(s) => offset + s.get(),
+ None => buffer.size,
+ };
+ buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action(
+ buffer,
+ offset..end,
+ MemoryInitKind::NeedsInitializedMemory,
+ ));
+ state.vertex[slot as usize] = Some(VertexState::new(buffer_id, offset..end));
+ }
+ RenderCommand::SetPushConstant {
+ stages,
+ offset,
+ size_bytes,
+ values_offset: _,
+ } => {
+ let scope = PassErrorScope::SetPushConstant;
+ let end_offset = offset + size_bytes;
+
+ let pipeline_state = state.pipeline(scope)?;
+
+ pipeline_state.pipeline.layout
+ .validate_push_constant_ranges(stages, offset, end_offset)
+ .map_pass_err(scope)?;
+
+ commands.push(command);
+ }
+ RenderCommand::Draw {
+ vertex_count,
+ instance_count,
+ first_vertex,
+ first_instance,
+ } => {
+ let scope = PassErrorScope::Draw {
+ indexed: false,
+ indirect: false,
+ pipeline: state.pipeline_id(),
+ };
+ let pipeline = state.pipeline(scope)?;
+ let used_bind_groups = pipeline.used_bind_groups;
+
+ validate_draw(
+ &state.vertex[..],
+ &pipeline.steps,
+ first_vertex,
+ vertex_count,
+ first_instance,
+ instance_count,
+ ).map_pass_err(scope)?;
+
+ if instance_count > 0 && vertex_count > 0 {
+ commands.extend(state.flush_vertices());
+ commands.extend(state.flush_binds(used_bind_groups, base.dynamic_offsets));
+ commands.push(command);
+ }
+ }
+ RenderCommand::DrawIndexed {
+ index_count,
+ instance_count,
+ first_index,
+ base_vertex: _,
+ first_instance,
+ } => {
+ let scope = PassErrorScope::Draw {
+ indexed: true,
+ indirect: false,
+ pipeline: state.pipeline_id(),
+ };
+ let pipeline = state.pipeline(scope)?;
+ let used_bind_groups = pipeline.used_bind_groups;
+ let index = match state.index {
+ Some(ref index) => index,
+ None => return Err(DrawError::MissingIndexBuffer).map_pass_err(scope),
+ };
+
+ validate_indexed_draw(
+ &state.vertex[..],
+ &pipeline.steps,
+ index,
+ first_index,
+ index_count,
+ first_instance,
+ instance_count,
+ ).map_pass_err(scope)?;
+
+ if instance_count > 0 && index_count > 0 {
+ commands.extend(state.flush_index());
+ commands.extend(state.flush_vertices());
+ commands.extend(state.flush_binds(used_bind_groups, base.dynamic_offsets));
+ commands.push(command);
+ }
+ }
+ RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count: None,
+ indexed: false,
+ } => {
+ let scope = PassErrorScope::Draw {
+ indexed: false,
+ indirect: true,
+ pipeline: state.pipeline_id(),
+ };
+ device
+ .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)
+ .map_pass_err(scope)?;
+
+ let pipeline = state.pipeline(scope)?;
+ let used_bind_groups = pipeline.used_bind_groups;
+
+ let buffer = state
+ .trackers
+ .buffers
+ .write()
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT)
+ .map_pass_err(scope)?;
+ self.check_valid_to_use(buffer.device.info.id())
+ .map_pass_err(scope)?;
+ check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT)
+ .map_pass_err(scope)?;
+
+ buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action(
+ buffer,
+ offset..(offset + mem::size_of::<wgt::DrawIndirectArgs>() as u64),
+ MemoryInitKind::NeedsInitializedMemory,
+ ));
+
+ commands.extend(state.flush_vertices());
+ commands.extend(state.flush_binds(used_bind_groups, base.dynamic_offsets));
+ commands.push(command);
+ }
+ RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count: None,
+ indexed: true,
+ } => {
+ let scope = PassErrorScope::Draw {
+ indexed: true,
+ indirect: true,
+ pipeline: state.pipeline_id(),
+ };
+ device
+ .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)
+ .map_pass_err(scope)?;
+
+ let pipeline = state.pipeline(scope)?;
+ let used_bind_groups = pipeline.used_bind_groups;
+
+ let buffer = state
+ .trackers
+ .buffers
+ .write()
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT)
+ .map_pass_err(scope)?;
+ self.check_valid_to_use(buffer.device.info.id())
+ .map_pass_err(scope)?;
+ check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT)
+ .map_pass_err(scope)?;
+
+ buffer_memory_init_actions.extend(buffer.initialization_status.read().create_action(
+ buffer,
+ offset..(offset + mem::size_of::<wgt::DrawIndirectArgs>() as u64),
+ MemoryInitKind::NeedsInitializedMemory,
+ ));
+
+ let index = match state.index {
+ Some(ref mut index) => index,
+ None => return Err(DrawError::MissingIndexBuffer).map_pass_err(scope),
+ };
+
+ commands.extend(index.flush());
+ commands.extend(state.flush_vertices());
+ commands.extend(state.flush_binds(used_bind_groups, base.dynamic_offsets));
+ commands.push(command);
+ }
+ RenderCommand::MultiDrawIndirect { .. }
+ | RenderCommand::MultiDrawIndirectCount { .. } => unimplemented!(),
+ RenderCommand::PushDebugGroup { color: _, len: _ } => unimplemented!(),
+ RenderCommand::InsertDebugMarker { color: _, len: _ } => unimplemented!(),
+ RenderCommand::PopDebugGroup => unimplemented!(),
+ RenderCommand::WriteTimestamp { .. } // Must check the TIMESTAMP_QUERY_INSIDE_PASSES feature
+ | RenderCommand::BeginOcclusionQuery { .. }
+ | RenderCommand::EndOcclusionQuery
+ | RenderCommand::BeginPipelineStatisticsQuery { .. }
+ | RenderCommand::EndPipelineStatisticsQuery => unimplemented!(),
+ RenderCommand::ExecuteBundle(_)
+ | RenderCommand::SetBlendConstant(_)
+ | RenderCommand::SetStencilReference(_)
+ | RenderCommand::SetViewport { .. }
+ | RenderCommand::SetScissor(_) => unreachable!("not supported by a render bundle"),
+ }
+ }
+
+ Ok(RenderBundle {
+ base: BasePass {
+ label: desc.label.as_ref().map(|cow| cow.to_string()),
+ commands,
+ dynamic_offsets: state.flat_dynamic_offsets,
+ string_data: Vec::new(),
+ push_constant_data: Vec::new(),
+ },
+ is_depth_read_only: self.is_depth_read_only,
+ is_stencil_read_only: self.is_stencil_read_only,
+ device: device.clone(),
+ used: state.trackers,
+ buffer_memory_init_actions,
+ texture_memory_init_actions,
+ context: self.context,
+ info: ResourceInfo::new(desc.label.borrow_or_default()),
+ discard_hal_labels: device
+ .instance_flags
+ .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS),
+ })
+ }
+
+ fn check_valid_to_use(&self, device_id: id::DeviceId) -> Result<(), RenderBundleErrorInner> {
+ if device_id != self.parent_id {
+ return Err(RenderBundleErrorInner::NotValidToUse);
+ }
+
+ Ok(())
+ }
+
+ pub fn set_index_buffer(
+ &mut self,
+ buffer_id: id::BufferId,
+ index_format: wgt::IndexFormat,
+ offset: wgt::BufferAddress,
+ size: Option<wgt::BufferSize>,
+ ) {
+ self.base.commands.push(RenderCommand::SetIndexBuffer {
+ buffer_id,
+ index_format,
+ offset,
+ size,
+ });
+ }
+}
+
+/// Error type returned from `RenderBundleEncoder::new` if the sample count is invalid.
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum CreateRenderBundleError {
+ #[error(transparent)]
+ ColorAttachment(#[from] ColorAttachmentError),
+ #[error("Invalid number of samples {0}")]
+ InvalidSampleCount(u32),
+}
+
+/// Error type returned from `RenderBundleEncoder::new` if the sample count is invalid.
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum ExecutionError {
+ #[error("Buffer {0:?} is destroyed")]
+ DestroyedBuffer(id::BufferId),
+ #[error("BindGroup {0:?} is invalid")]
+ InvalidBindGroup(id::BindGroupId),
+ #[error("Using {0} in a render bundle is not implemented")]
+ Unimplemented(&'static str),
+}
+impl PrettyError for ExecutionError {
+ fn fmt_pretty(&self, fmt: &mut ErrorFormatter) {
+ fmt.error(self);
+ match *self {
+ Self::DestroyedBuffer(id) => {
+ fmt.buffer_label(&id);
+ }
+ Self::InvalidBindGroup(id) => {
+ fmt.bind_group_label(&id);
+ }
+ Self::Unimplemented(_reason) => {}
+ };
+ }
+}
+
+pub type RenderBundleDescriptor<'a> = wgt::RenderBundleDescriptor<Label<'a>>;
+
+//Note: here, `RenderBundle` is just wrapping a raw stream of render commands.
+// The plan is to back it by an actual Vulkan secondary buffer, D3D12 Bundle,
+// or Metal indirect command buffer.
+#[derive(Debug)]
+pub struct RenderBundle<A: HalApi> {
+ // Normalized command stream. It can be executed verbatim,
+ // without re-binding anything on the pipeline change.
+ base: BasePass<RenderCommand>,
+ pub(super) is_depth_read_only: bool,
+ pub(super) is_stencil_read_only: bool,
+ pub(crate) device: Arc<Device<A>>,
+ pub(crate) used: RenderBundleScope<A>,
+ pub(super) buffer_memory_init_actions: Vec<BufferInitTrackerAction<A>>,
+ pub(super) texture_memory_init_actions: Vec<TextureInitTrackerAction<A>>,
+ pub(super) context: RenderPassContext,
+ pub(crate) info: ResourceInfo<RenderBundle<A>>,
+ discard_hal_labels: bool,
+}
+
+impl<A: HalApi> Drop for RenderBundle<A> {
+ fn drop(&mut self) {
+ resource_log!("Destroy raw RenderBundle {:?}", self.info.label());
+
+ #[cfg(feature = "trace")]
+ if let Some(t) = self.device.trace.lock().as_mut() {
+ t.add(trace::Action::DestroyRenderBundle(self.info.id()));
+ }
+ }
+}
+
+#[cfg(send_sync)]
+unsafe impl<A: HalApi> Send for RenderBundle<A> {}
+#[cfg(send_sync)]
+unsafe impl<A: HalApi> Sync for RenderBundle<A> {}
+
+impl<A: HalApi> RenderBundle<A> {
+ /// Actually encode the contents into a native command buffer.
+ ///
+ /// This is partially duplicating the logic of `command_encoder_run_render_pass`.
+ /// However the point of this function is to be lighter, since we already had
+ /// a chance to go through the commands in `render_bundle_encoder_finish`.
+ ///
+ /// Note that the function isn't expected to fail, generally.
+ /// All the validation has already been done by this point.
+ /// The only failure condition is if some of the used buffers are destroyed.
+ pub(super) unsafe fn execute(&self, raw: &mut A::CommandEncoder) -> Result<(), ExecutionError> {
+ let trackers = &self.used;
+ let mut offsets = self.base.dynamic_offsets.as_slice();
+ let mut pipeline_layout = None::<Arc<PipelineLayout<A>>>;
+ if !self.discard_hal_labels {
+ if let Some(ref label) = self.base.label {
+ unsafe { raw.begin_debug_marker(label) };
+ }
+ }
+
+ let snatch_guard = self.device.snatchable_lock.read();
+
+ for command in self.base.commands.iter() {
+ match *command {
+ RenderCommand::SetBindGroup {
+ index,
+ num_dynamic_offsets,
+ bind_group_id,
+ } => {
+ let bind_groups = trackers.bind_groups.read();
+ let bind_group = bind_groups.get(bind_group_id).unwrap();
+ let raw_bg = bind_group
+ .raw(&snatch_guard)
+ .ok_or(ExecutionError::InvalidBindGroup(bind_group_id))?;
+ unsafe {
+ raw.set_bind_group(
+ pipeline_layout.as_ref().unwrap().raw(),
+ index,
+ raw_bg,
+ &offsets[..num_dynamic_offsets],
+ )
+ };
+ offsets = &offsets[num_dynamic_offsets..];
+ }
+ RenderCommand::SetPipeline(pipeline_id) => {
+ let render_pipelines = trackers.render_pipelines.read();
+ let pipeline = render_pipelines.get(pipeline_id).unwrap();
+ unsafe { raw.set_render_pipeline(pipeline.raw()) };
+
+ pipeline_layout = Some(pipeline.layout.clone());
+ }
+ RenderCommand::SetIndexBuffer {
+ buffer_id,
+ index_format,
+ offset,
+ size,
+ } => {
+ let buffers = trackers.buffers.read();
+ let buffer: &A::Buffer = buffers
+ .get(buffer_id)
+ .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?
+ .raw(&snatch_guard)
+ .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?;
+ let bb = hal::BufferBinding {
+ buffer,
+ offset,
+ size,
+ };
+ unsafe { raw.set_index_buffer(bb, index_format) };
+ }
+ RenderCommand::SetVertexBuffer {
+ slot,
+ buffer_id,
+ offset,
+ size,
+ } => {
+ let buffers = trackers.buffers.read();
+ let buffer = buffers
+ .get(buffer_id)
+ .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?
+ .raw(&snatch_guard)
+ .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?;
+ let bb = hal::BufferBinding {
+ buffer,
+ offset,
+ size,
+ };
+ unsafe { raw.set_vertex_buffer(slot, bb) };
+ }
+ RenderCommand::SetPushConstant {
+ stages,
+ offset,
+ size_bytes,
+ values_offset,
+ } => {
+ let pipeline_layout = pipeline_layout.as_ref().unwrap();
+
+ if let Some(values_offset) = values_offset {
+ let values_end_offset =
+ (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize;
+ let data_slice = &self.base.push_constant_data
+ [(values_offset as usize)..values_end_offset];
+
+ unsafe {
+ raw.set_push_constants(
+ pipeline_layout.raw(),
+ stages,
+ offset,
+ data_slice,
+ )
+ }
+ } else {
+ super::push_constant_clear(
+ offset,
+ size_bytes,
+ |clear_offset, clear_data| {
+ unsafe {
+ raw.set_push_constants(
+ pipeline_layout.raw(),
+ stages,
+ clear_offset,
+ clear_data,
+ )
+ };
+ },
+ );
+ }
+ }
+ RenderCommand::Draw {
+ vertex_count,
+ instance_count,
+ first_vertex,
+ first_instance,
+ } => {
+ unsafe { raw.draw(first_vertex, vertex_count, first_instance, instance_count) };
+ }
+ RenderCommand::DrawIndexed {
+ index_count,
+ instance_count,
+ first_index,
+ base_vertex,
+ first_instance,
+ } => {
+ unsafe {
+ raw.draw_indexed(
+ first_index,
+ index_count,
+ base_vertex,
+ first_instance,
+ instance_count,
+ )
+ };
+ }
+ RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count: None,
+ indexed: false,
+ } => {
+ let buffers = trackers.buffers.read();
+ let buffer = buffers
+ .get(buffer_id)
+ .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?
+ .raw(&snatch_guard)
+ .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?;
+ unsafe { raw.draw_indirect(buffer, offset, 1) };
+ }
+ RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count: None,
+ indexed: true,
+ } => {
+ let buffers = trackers.buffers.read();
+ let buffer = buffers
+ .get(buffer_id)
+ .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?
+ .raw(&snatch_guard)
+ .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?;
+ unsafe { raw.draw_indexed_indirect(buffer, offset, 1) };
+ }
+ RenderCommand::MultiDrawIndirect { .. }
+ | RenderCommand::MultiDrawIndirectCount { .. } => {
+ return Err(ExecutionError::Unimplemented("multi-draw-indirect"))
+ }
+ RenderCommand::PushDebugGroup { .. }
+ | RenderCommand::InsertDebugMarker { .. }
+ | RenderCommand::PopDebugGroup => {
+ return Err(ExecutionError::Unimplemented("debug-markers"))
+ }
+ RenderCommand::WriteTimestamp { .. }
+ | RenderCommand::BeginOcclusionQuery { .. }
+ | RenderCommand::EndOcclusionQuery
+ | RenderCommand::BeginPipelineStatisticsQuery { .. }
+ | RenderCommand::EndPipelineStatisticsQuery => {
+ return Err(ExecutionError::Unimplemented("queries"))
+ }
+ RenderCommand::ExecuteBundle(_)
+ | RenderCommand::SetBlendConstant(_)
+ | RenderCommand::SetStencilReference(_)
+ | RenderCommand::SetViewport { .. }
+ | RenderCommand::SetScissor(_) => unreachable!(),
+ }
+ }
+
+ if !self.discard_hal_labels {
+ if let Some(_) = self.base.label {
+ unsafe { raw.end_debug_marker() };
+ }
+ }
+
+ Ok(())
+ }
+}
+
+impl<A: HalApi> Resource for RenderBundle<A> {
+ const TYPE: ResourceType = "RenderBundle";
+
+ type Marker = crate::id::markers::RenderBundle;
+
+ fn as_info(&self) -> &ResourceInfo<Self> {
+ &self.info
+ }
+
+ fn as_info_mut(&mut self) -> &mut ResourceInfo<Self> {
+ &mut self.info
+ }
+}
+
+/// A render bundle's current index buffer state.
+///
+/// [`RenderBundleEncoder::finish`] records the currently set index buffer here,
+/// and calls [`State::flush_index`] before any indexed draw command to produce
+/// a `SetIndexBuffer` command if one is necessary.
+#[derive(Debug)]
+struct IndexState {
+ buffer: id::BufferId,
+ format: wgt::IndexFormat,
+ range: Range<wgt::BufferAddress>,
+ is_dirty: bool,
+}
+
+impl IndexState {
+ /// Return the number of entries in the current index buffer.
+ ///
+ /// Panic if no index buffer has been set.
+ fn limit(&self) -> u64 {
+ let bytes_per_index = match self.format {
+ wgt::IndexFormat::Uint16 => 2,
+ wgt::IndexFormat::Uint32 => 4,
+ };
+
+ (self.range.end - self.range.start) / bytes_per_index
+ }
+
+ /// Generate a `SetIndexBuffer` command to prepare for an indexed draw
+ /// command, if needed.
+ fn flush(&mut self) -> Option<RenderCommand> {
+ if self.is_dirty {
+ self.is_dirty = false;
+ Some(RenderCommand::SetIndexBuffer {
+ buffer_id: self.buffer,
+ index_format: self.format,
+ offset: self.range.start,
+ size: wgt::BufferSize::new(self.range.end - self.range.start),
+ })
+ } else {
+ None
+ }
+ }
+}
+
+/// The state of a single vertex buffer slot during render bundle encoding.
+///
+/// [`RenderBundleEncoder::finish`] uses this to drop redundant
+/// `SetVertexBuffer` commands from the final [`RenderBundle`]. It
+/// records one vertex buffer slot's state changes here, and then
+/// calls this type's [`flush`] method just before any draw command to
+/// produce a `SetVertexBuffer` commands if one is necessary.
+///
+/// [`flush`]: IndexState::flush
+#[derive(Debug)]
+struct VertexState {
+ buffer: id::BufferId,
+ range: Range<wgt::BufferAddress>,
+ is_dirty: bool,
+}
+
+impl VertexState {
+ fn new(buffer: id::BufferId, range: Range<wgt::BufferAddress>) -> Self {
+ Self {
+ buffer,
+ range,
+ is_dirty: true,
+ }
+ }
+
+ /// Generate a `SetVertexBuffer` command for this slot, if necessary.
+ ///
+ /// `slot` is the index of the vertex buffer slot that `self` tracks.
+ fn flush(&mut self, slot: u32) -> Option<RenderCommand> {
+ if self.is_dirty {
+ self.is_dirty = false;
+ Some(RenderCommand::SetVertexBuffer {
+ slot,
+ buffer_id: self.buffer,
+ offset: self.range.start,
+ size: wgt::BufferSize::new(self.range.end - self.range.start),
+ })
+ } else {
+ None
+ }
+ }
+}
+
+/// A bind group that has been set at a particular index during render bundle encoding.
+#[derive(Debug)]
+struct BindState<A: HalApi> {
+ /// The id of the bind group set at this index.
+ bind_group: Arc<BindGroup<A>>,
+
+ /// The layout of `group`.
+ layout: Arc<BindGroupLayout<A>>,
+
+ /// The range of dynamic offsets for this bind group, in the original
+ /// command stream's `BassPass::dynamic_offsets` array.
+ dynamic_offsets: Range<usize>,
+
+ /// True if this index's contents have been changed since the last time we
+ /// generated a `SetBindGroup` command.
+ is_dirty: bool,
+}
+
+/// The bundle's current pipeline, and some cached information needed for validation.
+struct PipelineState<A: HalApi> {
+ /// The pipeline
+ pipeline: Arc<RenderPipeline<A>>,
+
+ /// How this pipeline's vertex shader traverses each vertex buffer, indexed
+ /// by vertex buffer slot number.
+ steps: Vec<VertexStep>,
+
+ /// Ranges of push constants this pipeline uses, copied from the pipeline
+ /// layout.
+ push_constant_ranges: ArrayVec<wgt::PushConstantRange, { SHADER_STAGE_COUNT }>,
+
+ /// The number of bind groups this pipeline uses.
+ used_bind_groups: usize,
+}
+
+impl<A: HalApi> PipelineState<A> {
+ fn new(pipeline: &Arc<RenderPipeline<A>>) -> Self {
+ Self {
+ pipeline: pipeline.clone(),
+ steps: pipeline.vertex_steps.to_vec(),
+ push_constant_ranges: pipeline
+ .layout
+ .push_constant_ranges
+ .iter()
+ .cloned()
+ .collect(),
+ used_bind_groups: pipeline.layout.bind_group_layouts.len(),
+ }
+ }
+
+ /// Return a sequence of commands to zero the push constant ranges this
+ /// pipeline uses. If no initialization is necessary, return `None`.
+ fn zero_push_constants(&self) -> Option<impl Iterator<Item = RenderCommand>> {
+ if !self.push_constant_ranges.is_empty() {
+ let nonoverlapping_ranges =
+ super::bind::compute_nonoverlapping_ranges(&self.push_constant_ranges);
+
+ Some(
+ nonoverlapping_ranges
+ .into_iter()
+ .map(|range| RenderCommand::SetPushConstant {
+ stages: range.stages,
+ offset: range.range.start,
+ size_bytes: range.range.end - range.range.start,
+ values_offset: None, // write zeros
+ }),
+ )
+ } else {
+ None
+ }
+ }
+}
+
+/// State for analyzing and cleaning up bundle command streams.
+///
+/// To minimize state updates, [`RenderBundleEncoder::finish`]
+/// actually just applies commands like [`SetBindGroup`] and
+/// [`SetIndexBuffer`] to the simulated state stored here, and then
+/// calls the `flush_foo` methods before draw calls to produce the
+/// update commands we actually need.
+///
+/// [`SetBindGroup`]: RenderCommand::SetBindGroup
+/// [`SetIndexBuffer`]: RenderCommand::SetIndexBuffer
+struct State<A: HalApi> {
+ /// Resources used by this bundle. This will become [`RenderBundle::used`].
+ trackers: RenderBundleScope<A>,
+
+ /// The currently set pipeline, if any.
+ pipeline: Option<PipelineState<A>>,
+
+ /// The bind group set at each index, if any.
+ bind: ArrayVec<Option<BindState<A>>, { hal::MAX_BIND_GROUPS }>,
+
+ /// The state of each vertex buffer slot.
+ vertex: ArrayVec<Option<VertexState>, { hal::MAX_VERTEX_BUFFERS }>,
+
+ /// The current index buffer, if one has been set. We flush this state
+ /// before indexed draw commands.
+ index: Option<IndexState>,
+
+ /// Dynamic offset values used by the cleaned-up command sequence.
+ ///
+ /// This becomes the final [`RenderBundle`]'s [`BasePass`]'s
+ /// [`dynamic_offsets`] list.
+ ///
+ /// [`dynamic_offsets`]: BasePass::dynamic_offsets
+ flat_dynamic_offsets: Vec<wgt::DynamicOffset>,
+}
+
+impl<A: HalApi> State<A> {
+ /// Return the id of the current pipeline, if any.
+ fn pipeline_id(&self) -> Option<id::RenderPipelineId> {
+ self.pipeline.as_ref().map(|p| p.pipeline.as_info().id())
+ }
+
+ /// Return the current pipeline state. Return an error if none is set.
+ fn pipeline(&self, scope: PassErrorScope) -> Result<&PipelineState<A>, RenderBundleError> {
+ self.pipeline
+ .as_ref()
+ .ok_or(DrawError::MissingPipeline)
+ .map_pass_err(scope)
+ }
+
+ /// Mark all non-empty bind group table entries from `index` onwards as dirty.
+ fn invalidate_bind_group_from(&mut self, index: usize) {
+ for contents in self.bind[index..].iter_mut().flatten() {
+ contents.is_dirty = true;
+ }
+ }
+
+ fn set_bind_group(
+ &mut self,
+ slot: u32,
+ bind_group: &Arc<BindGroup<A>>,
+ layout: &Arc<BindGroupLayout<A>>,
+ dynamic_offsets: Range<usize>,
+ ) {
+ // If this call wouldn't actually change this index's state, we can
+ // return early. (If there are dynamic offsets, the range will always
+ // be different.)
+ if dynamic_offsets.is_empty() {
+ if let Some(ref contents) = self.bind[slot as usize] {
+ if contents.bind_group.is_equal(bind_group) {
+ return;
+ }
+ }
+ }
+
+ // Record the index's new state.
+ self.bind[slot as usize] = Some(BindState {
+ bind_group: bind_group.clone(),
+ layout: layout.clone(),
+ dynamic_offsets,
+ is_dirty: true,
+ });
+
+ // Once we've changed the bind group at a particular index, all
+ // subsequent indices need to be rewritten.
+ self.invalidate_bind_group_from(slot as usize + 1);
+ }
+
+ /// Determine which bind group slots need to be re-set after a pipeline change.
+ ///
+ /// Given that we are switching from the current pipeline state to `new`,
+ /// whose layout is `layout`, mark all the bind group slots that we need to
+ /// emit new `SetBindGroup` commands for as dirty.
+ ///
+ /// According to `wgpu_hal`'s rules:
+ ///
+ /// - If the layout of any bind group slot changes, then that slot and
+ /// all following slots must have their bind groups re-established.
+ ///
+ /// - Changing the push constant ranges at all requires re-establishing
+ /// all bind groups.
+ fn invalidate_bind_groups(&mut self, new: &PipelineState<A>, layout: &PipelineLayout<A>) {
+ match self.pipeline {
+ None => {
+ // Establishing entirely new pipeline state.
+ self.invalidate_bind_group_from(0);
+ }
+ Some(ref old) => {
+ if old.pipeline.is_equal(&new.pipeline) {
+ // Everything is derived from the pipeline, so if the id has
+ // not changed, there's no need to consider anything else.
+ return;
+ }
+
+ // Any push constant change invalidates all groups.
+ if old.push_constant_ranges != new.push_constant_ranges {
+ self.invalidate_bind_group_from(0);
+ } else {
+ let first_changed = self.bind.iter().zip(&layout.bind_group_layouts).position(
+ |(entry, layout)| match *entry {
+ Some(ref contents) => !contents.layout.is_equal(layout),
+ None => false,
+ },
+ );
+ if let Some(slot) = first_changed {
+ self.invalidate_bind_group_from(slot);
+ }
+ }
+ }
+ }
+ }
+
+ /// Set the bundle's current index buffer and its associated parameters.
+ fn set_index_buffer(
+ &mut self,
+ buffer: id::BufferId,
+ format: wgt::IndexFormat,
+ range: Range<wgt::BufferAddress>,
+ ) {
+ match self.index {
+ Some(ref current)
+ if current.buffer == buffer
+ && current.format == format
+ && current.range == range =>
+ {
+ return
+ }
+ _ => (),
+ }
+
+ self.index = Some(IndexState {
+ buffer,
+ format,
+ range,
+ is_dirty: true,
+ });
+ }
+
+ /// Generate a `SetIndexBuffer` command to prepare for an indexed draw
+ /// command, if needed.
+ fn flush_index(&mut self) -> Option<RenderCommand> {
+ self.index.as_mut().and_then(|index| index.flush())
+ }
+
+ fn flush_vertices(&mut self) -> impl Iterator<Item = RenderCommand> + '_ {
+ self.vertex
+ .iter_mut()
+ .enumerate()
+ .flat_map(|(i, vs)| vs.as_mut().and_then(|vs| vs.flush(i as u32)))
+ }
+
+ /// Generate `SetBindGroup` commands for any bind groups that need to be updated.
+ fn flush_binds(
+ &mut self,
+ used_bind_groups: usize,
+ dynamic_offsets: &[wgt::DynamicOffset],
+ ) -> impl Iterator<Item = RenderCommand> + '_ {
+ // Append each dirty bind group's dynamic offsets to `flat_dynamic_offsets`.
+ for contents in self.bind[..used_bind_groups].iter().flatten() {
+ if contents.is_dirty {
+ self.flat_dynamic_offsets
+ .extend_from_slice(&dynamic_offsets[contents.dynamic_offsets.clone()]);
+ }
+ }
+
+ // Then, generate `SetBindGroup` commands to update the dirty bind
+ // groups. After this, all bind groups are clean.
+ self.bind[..used_bind_groups]
+ .iter_mut()
+ .enumerate()
+ .flat_map(|(i, entry)| {
+ if let Some(ref mut contents) = *entry {
+ if contents.is_dirty {
+ contents.is_dirty = false;
+ let offsets = &contents.dynamic_offsets;
+ return Some(RenderCommand::SetBindGroup {
+ index: i.try_into().unwrap(),
+ bind_group_id: contents.bind_group.as_info().id(),
+ num_dynamic_offsets: offsets.end - offsets.start,
+ });
+ }
+ }
+ None
+ })
+ }
+}
+
+/// Error encountered when finishing recording a render bundle.
+#[derive(Clone, Debug, Error)]
+pub(super) enum RenderBundleErrorInner {
+ #[error("Resource is not valid to use with this render bundle because the resource and the bundle come from different devices")]
+ NotValidToUse,
+ #[error(transparent)]
+ Device(#[from] DeviceError),
+ #[error(transparent)]
+ RenderCommand(RenderCommandError),
+ #[error(transparent)]
+ Draw(#[from] DrawError),
+ #[error(transparent)]
+ MissingDownlevelFlags(#[from] MissingDownlevelFlags),
+}
+
+impl<T> From<T> for RenderBundleErrorInner
+where
+ T: Into<RenderCommandError>,
+{
+ fn from(t: T) -> Self {
+ Self::RenderCommand(t.into())
+ }
+}
+
+/// Error encountered when finishing recording a render bundle.
+#[derive(Clone, Debug, Error)]
+#[error("{scope}")]
+pub struct RenderBundleError {
+ pub scope: PassErrorScope,
+ #[source]
+ inner: RenderBundleErrorInner,
+}
+
+impl RenderBundleError {
+ pub(crate) const INVALID_DEVICE: Self = RenderBundleError {
+ scope: PassErrorScope::Bundle,
+ inner: RenderBundleErrorInner::Device(DeviceError::Invalid),
+ };
+}
+impl PrettyError for RenderBundleError {
+ fn fmt_pretty(&self, fmt: &mut ErrorFormatter) {
+ // This error is wrapper for the inner error,
+ // but the scope has useful labels
+ fmt.error(self);
+ self.scope.fmt_pretty(fmt);
+ }
+}
+
+impl<T, E> MapPassErr<T, RenderBundleError> for Result<T, E>
+where
+ E: Into<RenderBundleErrorInner>,
+{
+ fn map_pass_err(self, scope: PassErrorScope) -> Result<T, RenderBundleError> {
+ self.map_err(|inner| RenderBundleError {
+ scope,
+ inner: inner.into(),
+ })
+ }
+}
+
+pub mod bundle_ffi {
+ use super::{RenderBundleEncoder, RenderCommand};
+ use crate::{id, RawString};
+ use std::{convert::TryInto, slice};
+ use wgt::{BufferAddress, BufferSize, DynamicOffset, IndexFormat};
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given pointer is
+ /// valid for `offset_length` elements.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_render_bundle_set_bind_group(
+ bundle: &mut RenderBundleEncoder,
+ index: u32,
+ bind_group_id: id::BindGroupId,
+ offsets: *const DynamicOffset,
+ offset_length: usize,
+ ) {
+ let redundant = unsafe {
+ bundle.current_bind_groups.set_and_check_redundant(
+ bind_group_id,
+ index,
+ &mut bundle.base.dynamic_offsets,
+ offsets,
+ offset_length,
+ )
+ };
+
+ if redundant {
+ return;
+ }
+
+ bundle.base.commands.push(RenderCommand::SetBindGroup {
+ index,
+ num_dynamic_offsets: offset_length,
+ bind_group_id,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_bundle_set_pipeline(
+ bundle: &mut RenderBundleEncoder,
+ pipeline_id: id::RenderPipelineId,
+ ) {
+ if bundle.current_pipeline.set_and_check_redundant(pipeline_id) {
+ return;
+ }
+
+ bundle
+ .base
+ .commands
+ .push(RenderCommand::SetPipeline(pipeline_id));
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_bundle_set_vertex_buffer(
+ bundle: &mut RenderBundleEncoder,
+ slot: u32,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ size: Option<BufferSize>,
+ ) {
+ bundle.base.commands.push(RenderCommand::SetVertexBuffer {
+ slot,
+ buffer_id,
+ offset,
+ size,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_bundle_set_index_buffer(
+ encoder: &mut RenderBundleEncoder,
+ buffer: id::BufferId,
+ index_format: IndexFormat,
+ offset: BufferAddress,
+ size: Option<BufferSize>,
+ ) {
+ encoder.set_index_buffer(buffer, index_format, offset, size);
+ }
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given pointer is
+ /// valid for `data` elements.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_render_bundle_set_push_constants(
+ pass: &mut RenderBundleEncoder,
+ stages: wgt::ShaderStages,
+ offset: u32,
+ size_bytes: u32,
+ data: *const u8,
+ ) {
+ assert_eq!(
+ offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1),
+ 0,
+ "Push constant offset must be aligned to 4 bytes."
+ );
+ assert_eq!(
+ size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1),
+ 0,
+ "Push constant size must be aligned to 4 bytes."
+ );
+ let data_slice = unsafe { slice::from_raw_parts(data, size_bytes as usize) };
+ let value_offset = pass.base.push_constant_data.len().try_into().expect(
+ "Ran out of push constant space. Don't set 4gb of push constants per RenderBundle.",
+ );
+
+ pass.base.push_constant_data.extend(
+ data_slice
+ .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize)
+ .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])),
+ );
+
+ pass.base.commands.push(RenderCommand::SetPushConstant {
+ stages,
+ offset,
+ size_bytes,
+ values_offset: Some(value_offset),
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_bundle_draw(
+ bundle: &mut RenderBundleEncoder,
+ vertex_count: u32,
+ instance_count: u32,
+ first_vertex: u32,
+ first_instance: u32,
+ ) {
+ bundle.base.commands.push(RenderCommand::Draw {
+ vertex_count,
+ instance_count,
+ first_vertex,
+ first_instance,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_bundle_draw_indexed(
+ bundle: &mut RenderBundleEncoder,
+ index_count: u32,
+ instance_count: u32,
+ first_index: u32,
+ base_vertex: i32,
+ first_instance: u32,
+ ) {
+ bundle.base.commands.push(RenderCommand::DrawIndexed {
+ index_count,
+ instance_count,
+ first_index,
+ base_vertex,
+ first_instance,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_bundle_draw_indirect(
+ bundle: &mut RenderBundleEncoder,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ ) {
+ bundle.base.commands.push(RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count: None,
+ indexed: false,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_bundle_draw_indexed_indirect(
+ bundle: &mut RenderBundleEncoder,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ ) {
+ bundle.base.commands.push(RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count: None,
+ indexed: true,
+ });
+ }
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given `label`
+ /// is a valid null-terminated string.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_render_bundle_push_debug_group(
+ _bundle: &mut RenderBundleEncoder,
+ _label: RawString,
+ ) {
+ //TODO
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_bundle_pop_debug_group(_bundle: &mut RenderBundleEncoder) {
+ //TODO
+ }
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given `label`
+ /// is a valid null-terminated string.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_render_bundle_insert_debug_marker(
+ _bundle: &mut RenderBundleEncoder,
+ _label: RawString,
+ ) {
+ //TODO
+ }
+}
diff --git a/third_party/rust/wgpu-core/src/command/clear.rs b/third_party/rust/wgpu-core/src/command/clear.rs
new file mode 100644
index 0000000000..2569fea1a4
--- /dev/null
+++ b/third_party/rust/wgpu-core/src/command/clear.rs
@@ -0,0 +1,493 @@
+use std::{ops::Range, sync::Arc};
+
+#[cfg(feature = "trace")]
+use crate::device::trace::Command as TraceCommand;
+use crate::{
+ api_log,
+ command::CommandBuffer,
+ device::DeviceError,
+ get_lowest_common_denom,
+ global::Global,
+ hal_api::HalApi,
+ id::{BufferId, CommandEncoderId, DeviceId, TextureId},
+ init_tracker::{MemoryInitKind, TextureInitRange},
+ resource::{Resource, Texture, TextureClearMode},
+ track::{TextureSelector, TextureTracker},
+};
+
+use hal::CommandEncoder as _;
+use thiserror::Error;
+use wgt::{math::align_to, BufferAddress, BufferUsages, ImageSubresourceRange, TextureAspect};
+
+/// Error encountered while attempting a clear.
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum ClearError {
+ #[error("To use clear_texture the CLEAR_TEXTURE feature needs to be enabled")]
+ MissingClearTextureFeature,
+ #[error("Command encoder {0:?} is invalid")]
+ InvalidCommandEncoder(CommandEncoderId),
+ #[error("Device {0:?} is invalid")]
+ InvalidDevice(DeviceId),
+ #[error("Buffer {0:?} is invalid or destroyed")]
+ InvalidBuffer(BufferId),
+ #[error("Texture {0:?} is invalid or destroyed")]
+ InvalidTexture(TextureId),
+ #[error("Texture {0:?} can not be cleared")]
+ NoValidTextureClearMode(TextureId),
+ #[error("Buffer clear size {0:?} is not a multiple of `COPY_BUFFER_ALIGNMENT`")]
+ UnalignedFillSize(BufferAddress),
+ #[error("Buffer offset {0:?} is not a multiple of `COPY_BUFFER_ALIGNMENT`")]
+ UnalignedBufferOffset(BufferAddress),
+ #[error("Clear of {start_offset}..{end_offset} would end up overrunning the bounds of the buffer of size {buffer_size}")]
+ BufferOverrun {
+ start_offset: BufferAddress,
+ end_offset: BufferAddress,
+ buffer_size: BufferAddress,
+ },
+ #[error("Destination buffer is missing the `COPY_DST` usage flag")]
+ MissingCopyDstUsageFlag(Option<BufferId>, Option<TextureId>),
+ #[error("Texture lacks the aspects that were specified in the image subresource range. Texture with format {texture_format:?}, specified was {subresource_range_aspects:?}")]
+ MissingTextureAspect {
+ texture_format: wgt::TextureFormat,
+ subresource_range_aspects: TextureAspect,
+ },
+ #[error("Image subresource level range is outside of the texture's level range. texture range is {texture_level_range:?}, \
+whereas subesource range specified start {subresource_base_mip_level} and count {subresource_mip_level_count:?}")]
+ InvalidTextureLevelRange {
+ texture_level_range: Range<u32>,
+ subresource_base_mip_level: u32,
+ subresource_mip_level_count: Option<u32>,
+ },
+ #[error("Image subresource layer range is outside of the texture's layer range. texture range is {texture_layer_range:?}, \
+whereas subesource range specified start {subresource_base_array_layer} and count {subresource_array_layer_count:?}")]
+ InvalidTextureLayerRange {
+ texture_layer_range: Range<u32>,
+ subresource_base_array_layer: u32,
+ subresource_array_layer_count: Option<u32>,
+ },
+ #[error(transparent)]
+ Device(#[from] DeviceError),
+}
+
+impl Global {
+ pub fn command_encoder_clear_buffer<A: HalApi>(
+ &self,
+ command_encoder_id: CommandEncoderId,
+ dst: BufferId,
+ offset: BufferAddress,
+ size: Option<BufferAddress>,
+ ) -> Result<(), ClearError> {
+ profiling::scope!("CommandEncoder::clear_buffer");
+ api_log!("CommandEncoder::clear_buffer {dst:?}");
+
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)
+ .map_err(|_| ClearError::InvalidCommandEncoder(command_encoder_id))?;
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::ClearBuffer { dst, offset, size });
+ }
+
+ let (dst_buffer, dst_pending) = {
+ let buffer_guard = hub.buffers.read();
+ let dst_buffer = buffer_guard
+ .get(dst)
+ .map_err(|_| ClearError::InvalidBuffer(dst))?;
+ cmd_buf_data
+ .trackers
+ .buffers
+ .set_single(dst_buffer, hal::BufferUses::COPY_DST)
+ .ok_or(ClearError::InvalidBuffer(dst))?
+ };
+ let snatch_guard = dst_buffer.device.snatchable_lock.read();
+ let dst_raw = dst_buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(ClearError::InvalidBuffer(dst))?;
+ if !dst_buffer.usage.contains(BufferUsages::COPY_DST) {
+ return Err(ClearError::MissingCopyDstUsageFlag(Some(dst), None));
+ }
+
+ // Check if offset & size are valid.
+ if offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
+ return Err(ClearError::UnalignedBufferOffset(offset));
+ }
+ if let Some(size) = size {
+ if size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
+ return Err(ClearError::UnalignedFillSize(size));
+ }
+ let destination_end_offset = offset + size;
+ if destination_end_offset > dst_buffer.size {
+ return Err(ClearError::BufferOverrun {
+ start_offset: offset,
+ end_offset: destination_end_offset,
+ buffer_size: dst_buffer.size,
+ });
+ }
+ }
+
+ let end = match size {
+ Some(size) => offset + size,
+ None => dst_buffer.size,
+ };
+ if offset == end {
+ log::trace!("Ignoring fill_buffer of size 0");
+ return Ok(());
+ }
+
+ // Mark dest as initialized.
+ cmd_buf_data.buffer_memory_init_actions.extend(
+ dst_buffer.initialization_status.read().create_action(
+ &dst_buffer,
+ offset..end,
+ MemoryInitKind::ImplicitlyInitialized,
+ ),
+ );
+
+ // actual hal barrier & operation
+ let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
+ let cmd_buf_raw = cmd_buf_data.encoder.open()?;
+ unsafe {
+ cmd_buf_raw.transition_buffers(dst_barrier.into_iter());
+ cmd_buf_raw.clear_buffer(dst_raw, offset..end);
+ }
+ Ok(())
+ }
+
+ pub fn command_encoder_clear_texture<A: HalApi>(
+ &self,
+ command_encoder_id: CommandEncoderId,
+ dst: TextureId,
+ subresource_range: &ImageSubresourceRange,
+ ) -> Result<(), ClearError> {
+ profiling::scope!("CommandEncoder::clear_texture");
+ api_log!("CommandEncoder::clear_texture {dst:?}");
+
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)
+ .map_err(|_| ClearError::InvalidCommandEncoder(command_encoder_id))?;
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::ClearTexture {
+ dst,
+ subresource_range: *subresource_range,
+ });
+ }
+
+ if !cmd_buf.support_clear_texture {
+ return Err(ClearError::MissingClearTextureFeature);
+ }
+
+ let dst_texture = hub
+ .textures
+ .get(dst)
+ .map_err(|_| ClearError::InvalidTexture(dst))?;
+
+ // Check if subresource aspects are valid.
+ let clear_aspects =
+ hal::FormatAspects::new(dst_texture.desc.format, subresource_range.aspect);
+ if clear_aspects.is_empty() {
+ return Err(ClearError::MissingTextureAspect {
+ texture_format: dst_texture.desc.format,
+ subresource_range_aspects: subresource_range.aspect,
+ });
+ };
+
+ // Check if subresource level range is valid
+ let subresource_mip_range = subresource_range.mip_range(dst_texture.full_range.mips.end);
+ if dst_texture.full_range.mips.start > subresource_mip_range.start
+ || dst_texture.full_range.mips.end < subresource_mip_range.end
+ {
+ return Err(ClearError::InvalidTextureLevelRange {
+ texture_level_range: dst_texture.full_range.mips.clone(),
+ subresource_base_mip_level: subresource_range.base_mip_level,
+ subresource_mip_level_count: subresource_range.mip_level_count,
+ });
+ }
+ // Check if subresource layer range is valid
+ let subresource_layer_range =
+ subresource_range.layer_range(dst_texture.full_range.layers.end);
+ if dst_texture.full_range.layers.start > subresource_layer_range.start
+ || dst_texture.full_range.layers.end < subresource_layer_range.end
+ {
+ return Err(ClearError::InvalidTextureLayerRange {
+ texture_layer_range: dst_texture.full_range.layers.clone(),
+ subresource_base_array_layer: subresource_range.base_array_layer,
+ subresource_array_layer_count: subresource_range.array_layer_count,
+ });
+ }
+
+ let device = &cmd_buf.device;
+ if !device.is_valid() {
+ return Err(ClearError::InvalidDevice(cmd_buf.device.as_info().id()));
+ }
+ let (encoder, tracker) = cmd_buf_data.open_encoder_and_tracker()?;
+
+ clear_texture(
+ &dst_texture,
+ TextureInitRange {
+ mip_range: subresource_mip_range,
+ layer_range: subresource_layer_range,
+ },
+ encoder,
+ &mut tracker.textures,
+ &device.alignments,
+ device.zero_buffer.as_ref().unwrap(),
+ )
+ }
+}
+
+pub(crate) fn clear_texture<A: HalApi>(
+ dst_texture: &Arc<Texture<A>>,
+ range: TextureInitRange,
+ encoder: &mut A::CommandEncoder,
+ texture_tracker: &mut TextureTracker<A>,
+ alignments: &hal::Alignments,
+ zero_buffer: &A::Buffer,
+) -> Result<(), ClearError> {
+ let snatch_guard = dst_texture.device.snatchable_lock.read();
+ let dst_raw = dst_texture
+ .raw(&snatch_guard)
+ .ok_or_else(|| ClearError::InvalidTexture(dst_texture.as_info().id()))?;
+
+ // Issue the right barrier.
+ let clear_usage = match *dst_texture.clear_mode.read() {
+ TextureClearMode::BufferCopy => hal::TextureUses::COPY_DST,
+ TextureClearMode::RenderPass {
+ is_color: false, ..
+ } => hal::TextureUses::DEPTH_STENCIL_WRITE,
+ TextureClearMode::Surface { .. } | TextureClearMode::RenderPass { is_color: true, .. } => {
+ hal::TextureUses::COLOR_TARGET
+ }
+ TextureClearMode::None => {
+ return Err(ClearError::NoValidTextureClearMode(
+ dst_texture.as_info().id(),
+ ));
+ }
+ };
+
+ let selector = TextureSelector {
+ mips: range.mip_range.clone(),
+ layers: range.layer_range.clone(),
+ };
+
+ // If we're in a texture-init usecase, we know that the texture is already
+ // tracked since whatever caused the init requirement, will have caused the
+ // usage tracker to be aware of the texture. Meaning, that it is safe to
+ // call call change_replace_tracked if the life_guard is already gone (i.e.
+ // the user no longer holds on to this texture).
+ //
+ // On the other hand, when coming via command_encoder_clear_texture, the
+ // life_guard is still there since in order to call it a texture object is
+ // needed.
+ //
+ // We could in theory distinguish these two scenarios in the internal
+ // clear_texture api in order to remove this check and call the cheaper
+ // change_replace_tracked whenever possible.
+ let dst_barrier = texture_tracker
+ .set_single(dst_texture, selector, clear_usage)
+ .unwrap()
+ .map(|pending| pending.into_hal(dst_raw));
+ unsafe {
+ encoder.transition_textures(dst_barrier.into_iter());
+ }
+
+ // Record actual clearing
+ match *dst_texture.clear_mode.read() {
+ TextureClearMode::BufferCopy => clear_texture_via_buffer_copies::<A>(
+ &dst_texture.desc,
+ alignments,
+ zero_buffer,
+ range,
+ encoder,
+ dst_raw,
+ ),
+ TextureClearMode::Surface { .. } => {
+ clear_texture_via_render_passes(dst_texture, range, true, encoder)?
+ }
+ TextureClearMode::RenderPass { is_color, .. } => {
+ clear_texture_via_render_passes(dst_texture, range, is_color, encoder)?
+ }
+ TextureClearMode::None => {
+ return Err(ClearError::NoValidTextureClearMode(
+ dst_texture.as_info().id(),
+ ));
+ }
+ }
+ Ok(())
+}
+
+fn clear_texture_via_buffer_copies<A: HalApi>(
+ texture_desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
+ alignments: &hal::Alignments,
+ zero_buffer: &A::Buffer, // Buffer of size device::ZERO_BUFFER_SIZE
+ range: TextureInitRange,
+ encoder: &mut A::CommandEncoder,
+ dst_raw: &A::Texture,
+) {
+ assert!(!texture_desc.format.is_depth_stencil_format());
+
+ if texture_desc.format == wgt::TextureFormat::NV12 {
+ // TODO: Currently COPY_DST for NV12 textures is unsupported.
+ return;
+ }
+
+ // Gather list of zero_buffer copies and issue a single command then to perform them
+ let mut zero_buffer_copy_regions = Vec::new();
+ let buffer_copy_pitch = alignments.buffer_copy_pitch.get() as u32;
+ let (block_width, block_height) = texture_desc.format.block_dimensions();
+ let block_size = texture_desc.format.block_copy_size(None).unwrap();
+
+ let bytes_per_row_alignment = get_lowest_common_denom(buffer_copy_pitch, block_size);
+
+ for mip_level in range.mip_range {
+ let mut mip_size = texture_desc.mip_level_size(mip_level).unwrap();
+ // Round to multiple of block size
+ mip_size.width = align_to(mip_size.width, block_width);
+ mip_size.height = align_to(mip_size.height, block_height);
+
+ let bytes_per_row = align_to(
+ mip_size.width / block_width * block_size,
+ bytes_per_row_alignment,
+ );
+
+ let max_rows_per_copy = crate::device::ZERO_BUFFER_SIZE as u32 / bytes_per_row;
+ // round down to a multiple of rows needed by the texture format
+ let max_rows_per_copy = max_rows_per_copy / block_height * block_height;
+ assert!(
+ max_rows_per_copy > 0,
+ "Zero buffer size is too small to fill a single row \
+ of a texture with format {:?} and desc {:?}",
+ texture_desc.format,
+ texture_desc.size
+ );
+
+ let z_range = 0..(if texture_desc.dimension == wgt::TextureDimension::D3 {
+ mip_size.depth_or_array_layers
+ } else {
+ 1
+ });
+
+ for array_layer in range.layer_range.clone() {
+ // TODO: Only doing one layer at a time for volume textures right now.
+ for z in z_range.clone() {
+ // May need multiple copies for each subresource! However, we
+ // assume that we never need to split a row.
+ let mut num_rows_left = mip_size.height;
+ while num_rows_left > 0 {
+ let num_rows = num_rows_left.min(max_rows_per_copy);
+
+ zero_buffer_copy_regions.push(hal::BufferTextureCopy {
+ buffer_layout: wgt::ImageDataLayout {
+ offset: 0,
+ bytes_per_row: Some(bytes_per_row),
+ rows_per_image: None,
+ },
+ texture_base: hal::TextureCopyBase {
+ mip_level,
+ array_layer,
+ origin: wgt::Origin3d {
+ x: 0, // Always full rows
+ y: mip_size.height - num_rows_left,
+ z,
+ },
+ aspect: hal::FormatAspects::COLOR,
+ },
+ size: hal::CopyExtent {
+ width: mip_size.width, // full row
+ height: num_rows,
+ depth: 1, // Only single slice of volume texture at a time right now
+ },
+ });
+
+ num_rows_left -= num_rows;
+ }
+ }
+ }
+ }
+
+ unsafe {
+ encoder.copy_buffer_to_texture(zero_buffer, dst_raw, zero_buffer_copy_regions.into_iter());
+ }
+}
+
+fn clear_texture_via_render_passes<A: HalApi>(
+ dst_texture: &Texture<A>,
+ range: TextureInitRange,
+ is_color: bool,
+ encoder: &mut A::CommandEncoder,
+) -> Result<(), ClearError> {
+ assert_eq!(dst_texture.desc.dimension, wgt::TextureDimension::D2);
+
+ let extent_base = wgt::Extent3d {
+ width: dst_texture.desc.size.width,
+ height: dst_texture.desc.size.height,
+ depth_or_array_layers: 1, // Only one layer is cleared at a time.
+ };
+ let clear_mode = &dst_texture.clear_mode.read();
+
+ for mip_level in range.mip_range {
+ let extent = extent_base.mip_level_size(mip_level, dst_texture.desc.dimension);
+ for depth_or_layer in range.layer_range.clone() {
+ let color_attachments_tmp;
+ let (color_attachments, depth_stencil_attachment) = if is_color {
+ color_attachments_tmp = [Some(hal::ColorAttachment {
+ target: hal::Attachment {
+ view: Texture::get_clear_view(
+ clear_mode,
+ &dst_texture.desc,
+ mip_level,
+ depth_or_layer,
+ ),
+ usage: hal::TextureUses::COLOR_TARGET,
+ },
+ resolve_target: None,
+ ops: hal::AttachmentOps::STORE,
+ clear_value: wgt::Color::TRANSPARENT,
+ })];
+ (&color_attachments_tmp[..], None)
+ } else {
+ (
+ &[][..],
+ Some(hal::DepthStencilAttachment {
+ target: hal::Attachment {
+ view: Texture::get_clear_view(
+ clear_mode,
+ &dst_texture.desc,
+ mip_level,
+ depth_or_layer,
+ ),
+ usage: hal::TextureUses::DEPTH_STENCIL_WRITE,
+ },
+ depth_ops: hal::AttachmentOps::STORE,
+ stencil_ops: hal::AttachmentOps::STORE,
+ clear_value: (0.0, 0),
+ }),
+ )
+ };
+ unsafe {
+ encoder.begin_render_pass(&hal::RenderPassDescriptor {
+ label: Some("(wgpu internal) clear_texture clear pass"),
+ extent,
+ sample_count: dst_texture.desc.sample_count,
+ color_attachments,
+ depth_stencil_attachment,
+ multiview: None,
+ timestamp_writes: None,
+ occlusion_query_set: None,
+ });
+ encoder.end_render_pass();
+ }
+ }
+ }
+ Ok(())
+}
diff --git a/third_party/rust/wgpu-core/src/command/compute.rs b/third_party/rust/wgpu-core/src/command/compute.rs
new file mode 100644
index 0000000000..804186a01e
--- /dev/null
+++ b/third_party/rust/wgpu-core/src/command/compute.rs
@@ -0,0 +1,1074 @@
+use crate::device::DeviceError;
+use crate::resource::Resource;
+use crate::snatch::SnatchGuard;
+use crate::{
+ binding_model::{
+ BindError, BindGroup, LateMinBufferBindingSizeMismatch, PushConstantUploadError,
+ },
+ command::{
+ bind::Binder,
+ end_pipeline_statistics_query,
+ memory_init::{fixup_discarded_surfaces, SurfacesInDiscardState},
+ BasePass, BasePassRef, BindGroupStateChange, CommandBuffer, CommandEncoderError,
+ CommandEncoderStatus, MapPassErr, PassErrorScope, QueryUseError, StateChange,
+ },
+ device::{MissingDownlevelFlags, MissingFeatures},
+ error::{ErrorFormatter, PrettyError},
+ global::Global,
+ hal_api::HalApi,
+ hal_label, id,
+ id::DeviceId,
+ init_tracker::MemoryInitKind,
+ pipeline,
+ resource::{self},
+ storage::Storage,
+ track::{Tracker, UsageConflict, UsageScope},
+ validation::{check_buffer_usage, MissingBufferUsageError},
+ Label,
+};
+
+use hal::CommandEncoder as _;
+#[cfg(feature = "serde")]
+use serde::Deserialize;
+#[cfg(feature = "serde")]
+use serde::Serialize;
+
+use thiserror::Error;
+
+use std::{fmt, mem, str};
+
+#[doc(hidden)]
+#[derive(Clone, Copy, Debug)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub enum ComputeCommand {
+ SetBindGroup {
+ index: u32,
+ num_dynamic_offsets: usize,
+ bind_group_id: id::BindGroupId,
+ },
+ SetPipeline(id::ComputePipelineId),
+
+ /// Set a range of push constants to values stored in [`BasePass::push_constant_data`].
+ SetPushConstant {
+ /// The byte offset within the push constant storage to write to. This
+ /// must be a multiple of four.
+ offset: u32,
+
+ /// The number of bytes to write. This must be a multiple of four.
+ size_bytes: u32,
+
+ /// Index in [`BasePass::push_constant_data`] of the start of the data
+ /// to be written.
+ ///
+ /// Note: this is not a byte offset like `offset`. Rather, it is the
+ /// index of the first `u32` element in `push_constant_data` to read.
+ values_offset: u32,
+ },
+
+ Dispatch([u32; 3]),
+ DispatchIndirect {
+ buffer_id: id::BufferId,
+ offset: wgt::BufferAddress,
+ },
+ PushDebugGroup {
+ color: u32,
+ len: usize,
+ },
+ PopDebugGroup,
+ InsertDebugMarker {
+ color: u32,
+ len: usize,
+ },
+ WriteTimestamp {
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ },
+ BeginPipelineStatisticsQuery {
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ },
+ EndPipelineStatisticsQuery,
+}
+
+#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
+pub struct ComputePass {
+ base: BasePass<ComputeCommand>,
+ parent_id: id::CommandEncoderId,
+ timestamp_writes: Option<ComputePassTimestampWrites>,
+
+ // Resource binding dedupe state.
+ #[cfg_attr(feature = "serde", serde(skip))]
+ current_bind_groups: BindGroupStateChange,
+ #[cfg_attr(feature = "serde", serde(skip))]
+ current_pipeline: StateChange<id::ComputePipelineId>,
+}
+
+impl ComputePass {
+ pub fn new(parent_id: id::CommandEncoderId, desc: &ComputePassDescriptor) -> Self {
+ Self {
+ base: BasePass::new(&desc.label),
+ parent_id,
+ timestamp_writes: desc.timestamp_writes.cloned(),
+
+ current_bind_groups: BindGroupStateChange::new(),
+ current_pipeline: StateChange::new(),
+ }
+ }
+
+ pub fn parent_id(&self) -> id::CommandEncoderId {
+ self.parent_id
+ }
+
+ #[cfg(feature = "trace")]
+ pub fn into_command(self) -> crate::device::trace::Command {
+ crate::device::trace::Command::RunComputePass {
+ base: self.base,
+ timestamp_writes: self.timestamp_writes,
+ }
+ }
+}
+
+impl fmt::Debug for ComputePass {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(
+ f,
+ "ComputePass {{ encoder_id: {:?}, data: {:?} commands and {:?} dynamic offsets }}",
+ self.parent_id,
+ self.base.commands.len(),
+ self.base.dynamic_offsets.len()
+ )
+ }
+}
+
+/// Describes the writing of timestamp values in a compute pass.
+#[repr(C)]
+#[derive(Clone, Debug, PartialEq, Eq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct ComputePassTimestampWrites {
+ /// The query set to write the timestamps to.
+ pub query_set: id::QuerySetId,
+ /// The index of the query set at which a start timestamp of this pass is written, if any.
+ pub beginning_of_pass_write_index: Option<u32>,
+ /// The index of the query set at which an end timestamp of this pass is written, if any.
+ pub end_of_pass_write_index: Option<u32>,
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct ComputePassDescriptor<'a> {
+ pub label: Label<'a>,
+ /// Defines where and when timestamp values will be written for this pass.
+ pub timestamp_writes: Option<&'a ComputePassTimestampWrites>,
+}
+
+#[derive(Clone, Debug, Error, Eq, PartialEq)]
+#[non_exhaustive]
+pub enum DispatchError {
+ #[error("Compute pipeline must be set")]
+ MissingPipeline,
+ #[error("Incompatible bind group at index {index} in the current compute pipeline")]
+ IncompatibleBindGroup { index: u32, diff: Vec<String> },
+ #[error(
+ "Each current dispatch group size dimension ({current:?}) must be less or equal to {limit}"
+ )]
+ InvalidGroupSize { current: [u32; 3], limit: u32 },
+ #[error(transparent)]
+ BindingSizeTooSmall(#[from] LateMinBufferBindingSizeMismatch),
+}
+
+/// Error encountered when performing a compute pass.
+#[derive(Clone, Debug, Error)]
+pub enum ComputePassErrorInner {
+ #[error(transparent)]
+ Device(#[from] DeviceError),
+ #[error(transparent)]
+ Encoder(#[from] CommandEncoderError),
+ #[error("Bind group at index {0:?} is invalid")]
+ InvalidBindGroup(usize),
+ #[error("Device {0:?} is invalid")]
+ InvalidDevice(DeviceId),
+ #[error("Bind group index {index} is greater than the device's requested `max_bind_group` limit {max}")]
+ BindGroupIndexOutOfRange { index: u32, max: u32 },
+ #[error("Compute pipeline {0:?} is invalid")]
+ InvalidPipeline(id::ComputePipelineId),
+ #[error("QuerySet {0:?} is invalid")]
+ InvalidQuerySet(id::QuerySetId),
+ #[error("Indirect buffer {0:?} is invalid or destroyed")]
+ InvalidIndirectBuffer(id::BufferId),
+ #[error("Indirect buffer uses bytes {offset}..{end_offset} which overruns indirect buffer of size {buffer_size}")]
+ IndirectBufferOverrun {
+ offset: u64,
+ end_offset: u64,
+ buffer_size: u64,
+ },
+ #[error("Buffer {0:?} is invalid or destroyed")]
+ InvalidBuffer(id::BufferId),
+ #[error(transparent)]
+ ResourceUsageConflict(#[from] UsageConflict),
+ #[error(transparent)]
+ MissingBufferUsage(#[from] MissingBufferUsageError),
+ #[error("Cannot pop debug group, because number of pushed debug groups is zero")]
+ InvalidPopDebugGroup,
+ #[error(transparent)]
+ Dispatch(#[from] DispatchError),
+ #[error(transparent)]
+ Bind(#[from] BindError),
+ #[error(transparent)]
+ PushConstants(#[from] PushConstantUploadError),
+ #[error(transparent)]
+ QueryUse(#[from] QueryUseError),
+ #[error(transparent)]
+ MissingFeatures(#[from] MissingFeatures),
+ #[error(transparent)]
+ MissingDownlevelFlags(#[from] MissingDownlevelFlags),
+}
+
+impl PrettyError for ComputePassErrorInner {
+ fn fmt_pretty(&self, fmt: &mut ErrorFormatter) {
+ fmt.error(self);
+ match *self {
+ Self::InvalidPipeline(id) => {
+ fmt.compute_pipeline_label(&id);
+ }
+ Self::InvalidIndirectBuffer(id) => {
+ fmt.buffer_label(&id);
+ }
+ Self::Dispatch(DispatchError::IncompatibleBindGroup { ref diff, .. }) => {
+ for d in diff {
+ fmt.note(&d);
+ }
+ }
+ _ => {}
+ };
+ }
+}
+
+/// Error encountered when performing a compute pass.
+#[derive(Clone, Debug, Error)]
+#[error("{scope}")]
+pub struct ComputePassError {
+ pub scope: PassErrorScope,
+ #[source]
+ inner: ComputePassErrorInner,
+}
+impl PrettyError for ComputePassError {
+ fn fmt_pretty(&self, fmt: &mut ErrorFormatter) {
+ // This error is wrapper for the inner error,
+ // but the scope has useful labels
+ fmt.error(self);
+ self.scope.fmt_pretty(fmt);
+ }
+}
+
+impl<T, E> MapPassErr<T, ComputePassError> for Result<T, E>
+where
+ E: Into<ComputePassErrorInner>,
+{
+ fn map_pass_err(self, scope: PassErrorScope) -> Result<T, ComputePassError> {
+ self.map_err(|inner| ComputePassError {
+ scope,
+ inner: inner.into(),
+ })
+ }
+}
+
+struct State<A: HalApi> {
+ binder: Binder<A>,
+ pipeline: Option<id::ComputePipelineId>,
+ scope: UsageScope<A>,
+ debug_scope_depth: u32,
+}
+
+impl<A: HalApi> State<A> {
+ fn is_ready(&self) -> Result<(), DispatchError> {
+ let bind_mask = self.binder.invalid_mask();
+ if bind_mask != 0 {
+ //let (expected, provided) = self.binder.entries[index as usize].info();
+ let index = bind_mask.trailing_zeros();
+
+ return Err(DispatchError::IncompatibleBindGroup {
+ index,
+ diff: self.binder.bgl_diff(),
+ });
+ }
+ if self.pipeline.is_none() {
+ return Err(DispatchError::MissingPipeline);
+ }
+ self.binder.check_late_buffer_bindings()?;
+
+ Ok(())
+ }
+
+ // `extra_buffer` is there to represent the indirect buffer that is also
+ // part of the usage scope.
+ fn flush_states(
+ &mut self,
+ raw_encoder: &mut A::CommandEncoder,
+ base_trackers: &mut Tracker<A>,
+ bind_group_guard: &Storage<BindGroup<A>>,
+ indirect_buffer: Option<id::BufferId>,
+ snatch_guard: &SnatchGuard,
+ ) -> Result<(), UsageConflict> {
+ for id in self.binder.list_active() {
+ unsafe { self.scope.merge_bind_group(&bind_group_guard[id].used)? };
+ // Note: stateless trackers are not merged: the lifetime reference
+ // is held to the bind group itself.
+ }
+
+ for id in self.binder.list_active() {
+ unsafe {
+ base_trackers.set_and_remove_from_usage_scope_sparse(
+ &mut self.scope,
+ &bind_group_guard[id].used,
+ )
+ }
+ }
+
+ // Add the state of the indirect buffer if it hasn't been hit before.
+ unsafe {
+ base_trackers
+ .buffers
+ .set_and_remove_from_usage_scope_sparse(&mut self.scope.buffers, indirect_buffer);
+ }
+
+ log::trace!("Encoding dispatch barriers");
+
+ CommandBuffer::drain_barriers(raw_encoder, base_trackers, snatch_guard);
+ Ok(())
+ }
+}
+
+// Common routines between render/compute
+
+impl Global {
+ pub fn command_encoder_run_compute_pass<A: HalApi>(
+ &self,
+ encoder_id: id::CommandEncoderId,
+ pass: &ComputePass,
+ ) -> Result<(), ComputePassError> {
+ self.command_encoder_run_compute_pass_impl::<A>(
+ encoder_id,
+ pass.base.as_ref(),
+ pass.timestamp_writes.as_ref(),
+ )
+ }
+
+ #[doc(hidden)]
+ pub fn command_encoder_run_compute_pass_impl<A: HalApi>(
+ &self,
+ encoder_id: id::CommandEncoderId,
+ base: BasePassRef<ComputeCommand>,
+ timestamp_writes: Option<&ComputePassTimestampWrites>,
+ ) -> Result<(), ComputePassError> {
+ profiling::scope!("CommandEncoder::run_compute_pass");
+ let pass_scope = PassErrorScope::Pass(encoder_id);
+
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id).map_pass_err(pass_scope)?;
+ let device = &cmd_buf.device;
+ if !device.is_valid() {
+ return Err(ComputePassErrorInner::InvalidDevice(
+ cmd_buf.device.as_info().id(),
+ ))
+ .map_pass_err(pass_scope);
+ }
+
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(crate::device::trace::Command::RunComputePass {
+ base: BasePass::from_ref(base),
+ timestamp_writes: timestamp_writes.cloned(),
+ });
+ }
+
+ let encoder = &mut cmd_buf_data.encoder;
+ let status = &mut cmd_buf_data.status;
+ let tracker = &mut cmd_buf_data.trackers;
+ let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions;
+ let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions;
+
+ // We automatically keep extending command buffers over time, and because
+ // we want to insert a command buffer _before_ what we're about to record,
+ // we need to make sure to close the previous one.
+ encoder.close().map_pass_err(pass_scope)?;
+ // will be reset to true if recording is done without errors
+ *status = CommandEncoderStatus::Error;
+ let raw = encoder.open().map_pass_err(pass_scope)?;
+
+ let bind_group_guard = hub.bind_groups.read();
+ let pipeline_guard = hub.compute_pipelines.read();
+ let query_set_guard = hub.query_sets.read();
+ let buffer_guard = hub.buffers.read();
+ let texture_guard = hub.textures.read();
+
+ let mut state = State {
+ binder: Binder::new(),
+ pipeline: None,
+ scope: UsageScope::new(&*buffer_guard, &*texture_guard),
+ debug_scope_depth: 0,
+ };
+ let mut temp_offsets = Vec::new();
+ let mut dynamic_offset_count = 0;
+ let mut string_offset = 0;
+ let mut active_query = None;
+
+ let timestamp_writes = if let Some(tw) = timestamp_writes {
+ let query_set: &resource::QuerySet<A> = tracker
+ .query_sets
+ .add_single(&*query_set_guard, tw.query_set)
+ .ok_or(ComputePassErrorInner::InvalidQuerySet(tw.query_set))
+ .map_pass_err(pass_scope)?;
+
+ // Unlike in render passes we can't delay resetting the query sets since
+ // there is no auxiliary pass.
+ let range = if let (Some(index_a), Some(index_b)) =
+ (tw.beginning_of_pass_write_index, tw.end_of_pass_write_index)
+ {
+ Some(index_a.min(index_b)..index_a.max(index_b) + 1)
+ } else {
+ tw.beginning_of_pass_write_index
+ .or(tw.end_of_pass_write_index)
+ .map(|i| i..i + 1)
+ };
+ // Range should always be Some, both values being None should lead to a validation error.
+ // But no point in erroring over that nuance here!
+ if let Some(range) = range {
+ unsafe {
+ raw.reset_queries(query_set.raw.as_ref().unwrap(), range);
+ }
+ }
+
+ Some(hal::ComputePassTimestampWrites {
+ query_set: query_set.raw.as_ref().unwrap(),
+ beginning_of_pass_write_index: tw.beginning_of_pass_write_index,
+ end_of_pass_write_index: tw.end_of_pass_write_index,
+ })
+ } else {
+ None
+ };
+
+ let snatch_guard = device.snatchable_lock.read();
+
+ tracker.set_size(
+ Some(&*buffer_guard),
+ Some(&*texture_guard),
+ None,
+ None,
+ Some(&*bind_group_guard),
+ Some(&*pipeline_guard),
+ None,
+ None,
+ Some(&*query_set_guard),
+ );
+
+ let discard_hal_labels = self
+ .instance
+ .flags
+ .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS);
+ let hal_desc = hal::ComputePassDescriptor {
+ label: hal_label(base.label, self.instance.flags),
+ timestamp_writes,
+ };
+
+ unsafe {
+ raw.begin_compute_pass(&hal_desc);
+ }
+
+ let mut intermediate_trackers = Tracker::<A>::new();
+
+ // Immediate texture inits required because of prior discards. Need to
+ // be inserted before texture reads.
+ let mut pending_discard_init_fixups = SurfacesInDiscardState::new();
+
+ for command in base.commands {
+ match *command {
+ ComputeCommand::SetBindGroup {
+ index,
+ num_dynamic_offsets,
+ bind_group_id,
+ } => {
+ let scope = PassErrorScope::SetBindGroup(bind_group_id);
+
+ let max_bind_groups = cmd_buf.limits.max_bind_groups;
+ if index >= max_bind_groups {
+ return Err(ComputePassErrorInner::BindGroupIndexOutOfRange {
+ index,
+ max: max_bind_groups,
+ })
+ .map_pass_err(scope);
+ }
+
+ temp_offsets.clear();
+ temp_offsets.extend_from_slice(
+ &base.dynamic_offsets
+ [dynamic_offset_count..dynamic_offset_count + num_dynamic_offsets],
+ );
+ dynamic_offset_count += num_dynamic_offsets;
+
+ let bind_group = tracker
+ .bind_groups
+ .add_single(&*bind_group_guard, bind_group_id)
+ .ok_or(ComputePassErrorInner::InvalidBindGroup(index as usize))
+ .map_pass_err(scope)?;
+ bind_group
+ .validate_dynamic_bindings(index, &temp_offsets, &cmd_buf.limits)
+ .map_pass_err(scope)?;
+
+ buffer_memory_init_actions.extend(
+ bind_group.used_buffer_ranges.iter().filter_map(|action| {
+ action
+ .buffer
+ .initialization_status
+ .read()
+ .check_action(action)
+ }),
+ );
+
+ for action in bind_group.used_texture_ranges.iter() {
+ pending_discard_init_fixups
+ .extend(texture_memory_actions.register_init_action(action));
+ }
+
+ let pipeline_layout = state.binder.pipeline_layout.clone();
+ let entries =
+ state
+ .binder
+ .assign_group(index as usize, bind_group, &temp_offsets);
+ if !entries.is_empty() && pipeline_layout.is_some() {
+ let pipeline_layout = pipeline_layout.as_ref().unwrap().raw();
+ for (i, e) in entries.iter().enumerate() {
+ if let Some(group) = e.group.as_ref() {
+ let raw_bg = group
+ .raw(&snatch_guard)
+ .ok_or(ComputePassErrorInner::InvalidBindGroup(i))
+ .map_pass_err(scope)?;
+ unsafe {
+ raw.set_bind_group(
+ pipeline_layout,
+ index + i as u32,
+ raw_bg,
+ &e.dynamic_offsets,
+ );
+ }
+ }
+ }
+ }
+ }
+ ComputeCommand::SetPipeline(pipeline_id) => {
+ let scope = PassErrorScope::SetPipelineCompute(pipeline_id);
+
+ state.pipeline = Some(pipeline_id);
+
+ let pipeline: &pipeline::ComputePipeline<A> = tracker
+ .compute_pipelines
+ .add_single(&*pipeline_guard, pipeline_id)
+ .ok_or(ComputePassErrorInner::InvalidPipeline(pipeline_id))
+ .map_pass_err(scope)?;
+
+ unsafe {
+ raw.set_compute_pipeline(pipeline.raw());
+ }
+
+ // Rebind resources
+ if state.binder.pipeline_layout.is_none()
+ || !state
+ .binder
+ .pipeline_layout
+ .as_ref()
+ .unwrap()
+ .is_equal(&pipeline.layout)
+ {
+ let (start_index, entries) = state.binder.change_pipeline_layout(
+ &pipeline.layout,
+ &pipeline.late_sized_buffer_groups,
+ );
+ if !entries.is_empty() {
+ for (i, e) in entries.iter().enumerate() {
+ if let Some(group) = e.group.as_ref() {
+ let raw_bg = group
+ .raw(&snatch_guard)
+ .ok_or(ComputePassErrorInner::InvalidBindGroup(i))
+ .map_pass_err(scope)?;
+ unsafe {
+ raw.set_bind_group(
+ pipeline.layout.raw(),
+ start_index as u32 + i as u32,
+ raw_bg,
+ &e.dynamic_offsets,
+ );
+ }
+ }
+ }
+ }
+
+ // Clear push constant ranges
+ let non_overlapping = super::bind::compute_nonoverlapping_ranges(
+ &pipeline.layout.push_constant_ranges,
+ );
+ for range in non_overlapping {
+ let offset = range.range.start;
+ let size_bytes = range.range.end - offset;
+ super::push_constant_clear(
+ offset,
+ size_bytes,
+ |clear_offset, clear_data| unsafe {
+ raw.set_push_constants(
+ pipeline.layout.raw(),
+ wgt::ShaderStages::COMPUTE,
+ clear_offset,
+ clear_data,
+ );
+ },
+ );
+ }
+ }
+ }
+ ComputeCommand::SetPushConstant {
+ offset,
+ size_bytes,
+ values_offset,
+ } => {
+ let scope = PassErrorScope::SetPushConstant;
+
+ let end_offset_bytes = offset + size_bytes;
+ let values_end_offset =
+ (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize;
+ let data_slice =
+ &base.push_constant_data[(values_offset as usize)..values_end_offset];
+
+ let pipeline_layout = state
+ .binder
+ .pipeline_layout
+ .as_ref()
+ //TODO: don't error here, lazily update the push constants
+ .ok_or(ComputePassErrorInner::Dispatch(
+ DispatchError::MissingPipeline,
+ ))
+ .map_pass_err(scope)?;
+
+ pipeline_layout
+ .validate_push_constant_ranges(
+ wgt::ShaderStages::COMPUTE,
+ offset,
+ end_offset_bytes,
+ )
+ .map_pass_err(scope)?;
+
+ unsafe {
+ raw.set_push_constants(
+ pipeline_layout.raw(),
+ wgt::ShaderStages::COMPUTE,
+ offset,
+ data_slice,
+ );
+ }
+ }
+ ComputeCommand::Dispatch(groups) => {
+ let scope = PassErrorScope::Dispatch {
+ indirect: false,
+ pipeline: state.pipeline,
+ };
+ state.is_ready().map_pass_err(scope)?;
+
+ state
+ .flush_states(
+ raw,
+ &mut intermediate_trackers,
+ &*bind_group_guard,
+ None,
+ &snatch_guard,
+ )
+ .map_pass_err(scope)?;
+
+ let groups_size_limit = cmd_buf.limits.max_compute_workgroups_per_dimension;
+
+ if groups[0] > groups_size_limit
+ || groups[1] > groups_size_limit
+ || groups[2] > groups_size_limit
+ {
+ return Err(ComputePassErrorInner::Dispatch(
+ DispatchError::InvalidGroupSize {
+ current: groups,
+ limit: groups_size_limit,
+ },
+ ))
+ .map_pass_err(scope);
+ }
+
+ unsafe {
+ raw.dispatch(groups);
+ }
+ }
+ ComputeCommand::DispatchIndirect { buffer_id, offset } => {
+ let scope = PassErrorScope::Dispatch {
+ indirect: true,
+ pipeline: state.pipeline,
+ };
+
+ state.is_ready().map_pass_err(scope)?;
+
+ device
+ .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)
+ .map_pass_err(scope)?;
+
+ let indirect_buffer = state
+ .scope
+ .buffers
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT)
+ .map_pass_err(scope)?;
+ check_buffer_usage(indirect_buffer.usage, wgt::BufferUsages::INDIRECT)
+ .map_pass_err(scope)?;
+
+ let end_offset = offset + mem::size_of::<wgt::DispatchIndirectArgs>() as u64;
+ if end_offset > indirect_buffer.size {
+ return Err(ComputePassErrorInner::IndirectBufferOverrun {
+ offset,
+ end_offset,
+ buffer_size: indirect_buffer.size,
+ })
+ .map_pass_err(scope);
+ }
+
+ let buf_raw = indirect_buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(ComputePassErrorInner::InvalidIndirectBuffer(buffer_id))
+ .map_pass_err(scope)?;
+
+ let stride = 3 * 4; // 3 integers, x/y/z group size
+
+ buffer_memory_init_actions.extend(
+ indirect_buffer.initialization_status.read().create_action(
+ indirect_buffer,
+ offset..(offset + stride),
+ MemoryInitKind::NeedsInitializedMemory,
+ ),
+ );
+
+ state
+ .flush_states(
+ raw,
+ &mut intermediate_trackers,
+ &*bind_group_guard,
+ Some(buffer_id),
+ &snatch_guard,
+ )
+ .map_pass_err(scope)?;
+ unsafe {
+ raw.dispatch_indirect(buf_raw, offset);
+ }
+ }
+ ComputeCommand::PushDebugGroup { color: _, len } => {
+ state.debug_scope_depth += 1;
+ if !discard_hal_labels {
+ let label =
+ str::from_utf8(&base.string_data[string_offset..string_offset + len])
+ .unwrap();
+ unsafe {
+ raw.begin_debug_marker(label);
+ }
+ }
+ string_offset += len;
+ }
+ ComputeCommand::PopDebugGroup => {
+ let scope = PassErrorScope::PopDebugGroup;
+
+ if state.debug_scope_depth == 0 {
+ return Err(ComputePassErrorInner::InvalidPopDebugGroup)
+ .map_pass_err(scope);
+ }
+ state.debug_scope_depth -= 1;
+ if !discard_hal_labels {
+ unsafe {
+ raw.end_debug_marker();
+ }
+ }
+ }
+ ComputeCommand::InsertDebugMarker { color: _, len } => {
+ if !discard_hal_labels {
+ let label =
+ str::from_utf8(&base.string_data[string_offset..string_offset + len])
+ .unwrap();
+ unsafe { raw.insert_debug_marker(label) }
+ }
+ string_offset += len;
+ }
+ ComputeCommand::WriteTimestamp {
+ query_set_id,
+ query_index,
+ } => {
+ let scope = PassErrorScope::WriteTimestamp;
+
+ device
+ .require_features(wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES)
+ .map_pass_err(scope)?;
+
+ let query_set: &resource::QuerySet<A> = tracker
+ .query_sets
+ .add_single(&*query_set_guard, query_set_id)
+ .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id))
+ .map_pass_err(scope)?;
+
+ query_set
+ .validate_and_write_timestamp(raw, query_set_id, query_index, None)
+ .map_pass_err(scope)?;
+ }
+ ComputeCommand::BeginPipelineStatisticsQuery {
+ query_set_id,
+ query_index,
+ } => {
+ let scope = PassErrorScope::BeginPipelineStatisticsQuery;
+
+ let query_set: &resource::QuerySet<A> = tracker
+ .query_sets
+ .add_single(&*query_set_guard, query_set_id)
+ .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id))
+ .map_pass_err(scope)?;
+
+ query_set
+ .validate_and_begin_pipeline_statistics_query(
+ raw,
+ query_set_id,
+ query_index,
+ None,
+ &mut active_query,
+ )
+ .map_pass_err(scope)?;
+ }
+ ComputeCommand::EndPipelineStatisticsQuery => {
+ let scope = PassErrorScope::EndPipelineStatisticsQuery;
+
+ end_pipeline_statistics_query(raw, &*query_set_guard, &mut active_query)
+ .map_pass_err(scope)?;
+ }
+ }
+ }
+
+ unsafe {
+ raw.end_compute_pass();
+ }
+
+ // We've successfully recorded the compute pass, bring the
+ // command buffer out of the error state.
+ *status = CommandEncoderStatus::Recording;
+
+ // Stop the current command buffer.
+ encoder.close().map_pass_err(pass_scope)?;
+
+ // Create a new command buffer, which we will insert _before_ the body of the compute pass.
+ //
+ // Use that buffer to insert barriers and clear discarded images.
+ let transit = encoder.open().map_pass_err(pass_scope)?;
+ fixup_discarded_surfaces(
+ pending_discard_init_fixups.into_iter(),
+ transit,
+ &mut tracker.textures,
+ device,
+ );
+ CommandBuffer::insert_barriers_from_tracker(
+ transit,
+ tracker,
+ &intermediate_trackers,
+ &snatch_guard,
+ );
+ // Close the command buffer, and swap it with the previous.
+ encoder.close_and_swap().map_pass_err(pass_scope)?;
+
+ Ok(())
+ }
+}
+
+pub mod compute_ffi {
+ use super::{ComputeCommand, ComputePass};
+ use crate::{id, RawString};
+ use std::{convert::TryInto, ffi, slice};
+ use wgt::{BufferAddress, DynamicOffset};
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given pointer is
+ /// valid for `offset_length` elements.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_compute_pass_set_bind_group(
+ pass: &mut ComputePass,
+ index: u32,
+ bind_group_id: id::BindGroupId,
+ offsets: *const DynamicOffset,
+ offset_length: usize,
+ ) {
+ let redundant = unsafe {
+ pass.current_bind_groups.set_and_check_redundant(
+ bind_group_id,
+ index,
+ &mut pass.base.dynamic_offsets,
+ offsets,
+ offset_length,
+ )
+ };
+
+ if redundant {
+ return;
+ }
+
+ pass.base.commands.push(ComputeCommand::SetBindGroup {
+ index,
+ num_dynamic_offsets: offset_length,
+ bind_group_id,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_compute_pass_set_pipeline(
+ pass: &mut ComputePass,
+ pipeline_id: id::ComputePipelineId,
+ ) {
+ if pass.current_pipeline.set_and_check_redundant(pipeline_id) {
+ return;
+ }
+
+ pass.base
+ .commands
+ .push(ComputeCommand::SetPipeline(pipeline_id));
+ }
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given pointer is
+ /// valid for `size_bytes` bytes.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_compute_pass_set_push_constant(
+ pass: &mut ComputePass,
+ offset: u32,
+ size_bytes: u32,
+ data: *const u8,
+ ) {
+ assert_eq!(
+ offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1),
+ 0,
+ "Push constant offset must be aligned to 4 bytes."
+ );
+ assert_eq!(
+ size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1),
+ 0,
+ "Push constant size must be aligned to 4 bytes."
+ );
+ let data_slice = unsafe { slice::from_raw_parts(data, size_bytes as usize) };
+ let value_offset = pass.base.push_constant_data.len().try_into().expect(
+ "Ran out of push constant space. Don't set 4gb of push constants per ComputePass.",
+ );
+
+ pass.base.push_constant_data.extend(
+ data_slice
+ .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize)
+ .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])),
+ );
+
+ pass.base.commands.push(ComputeCommand::SetPushConstant {
+ offset,
+ size_bytes,
+ values_offset: value_offset,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_compute_pass_dispatch_workgroups(
+ pass: &mut ComputePass,
+ groups_x: u32,
+ groups_y: u32,
+ groups_z: u32,
+ ) {
+ pass.base
+ .commands
+ .push(ComputeCommand::Dispatch([groups_x, groups_y, groups_z]));
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_compute_pass_dispatch_workgroups_indirect(
+ pass: &mut ComputePass,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ ) {
+ pass.base
+ .commands
+ .push(ComputeCommand::DispatchIndirect { buffer_id, offset });
+ }
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given `label`
+ /// is a valid null-terminated string.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_compute_pass_push_debug_group(
+ pass: &mut ComputePass,
+ label: RawString,
+ color: u32,
+ ) {
+ let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes();
+ pass.base.string_data.extend_from_slice(bytes);
+
+ pass.base.commands.push(ComputeCommand::PushDebugGroup {
+ color,
+ len: bytes.len(),
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_compute_pass_pop_debug_group(pass: &mut ComputePass) {
+ pass.base.commands.push(ComputeCommand::PopDebugGroup);
+ }
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given `label`
+ /// is a valid null-terminated string.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_compute_pass_insert_debug_marker(
+ pass: &mut ComputePass,
+ label: RawString,
+ color: u32,
+ ) {
+ let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes();
+ pass.base.string_data.extend_from_slice(bytes);
+
+ pass.base.commands.push(ComputeCommand::InsertDebugMarker {
+ color,
+ len: bytes.len(),
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_compute_pass_write_timestamp(
+ pass: &mut ComputePass,
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ ) {
+ pass.base.commands.push(ComputeCommand::WriteTimestamp {
+ query_set_id,
+ query_index,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_compute_pass_begin_pipeline_statistics_query(
+ pass: &mut ComputePass,
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ ) {
+ pass.base
+ .commands
+ .push(ComputeCommand::BeginPipelineStatisticsQuery {
+ query_set_id,
+ query_index,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_compute_pass_end_pipeline_statistics_query(pass: &mut ComputePass) {
+ pass.base
+ .commands
+ .push(ComputeCommand::EndPipelineStatisticsQuery);
+ }
+}
diff --git a/third_party/rust/wgpu-core/src/command/draw.rs b/third_party/rust/wgpu-core/src/command/draw.rs
new file mode 100644
index 0000000000..e03a78ee93
--- /dev/null
+++ b/third_party/rust/wgpu-core/src/command/draw.rs
@@ -0,0 +1,247 @@
+/*! Draw structures - shared between render passes and bundles.
+!*/
+
+use crate::{
+ binding_model::{LateMinBufferBindingSizeMismatch, PushConstantUploadError},
+ error::ErrorFormatter,
+ id,
+ track::UsageConflict,
+ validation::{MissingBufferUsageError, MissingTextureUsageError},
+};
+use wgt::{BufferAddress, BufferSize, Color, VertexStepMode};
+
+use std::num::NonZeroU32;
+use thiserror::Error;
+
+/// Error validating a draw call.
+#[derive(Clone, Debug, Error, Eq, PartialEq)]
+#[non_exhaustive]
+pub enum DrawError {
+ #[error("Blend constant needs to be set")]
+ MissingBlendConstant,
+ #[error("Render pipeline must be set")]
+ MissingPipeline,
+ #[error("Vertex buffer {index} must be set")]
+ MissingVertexBuffer { index: u32 },
+ #[error("Index buffer must be set")]
+ MissingIndexBuffer,
+ #[error("Incompatible bind group at index {index} in the current render pipeline")]
+ IncompatibleBindGroup { index: u32, diff: Vec<String> },
+ #[error("Vertex {last_vertex} extends beyond limit {vertex_limit} imposed by the buffer in slot {slot}. Did you bind the correct `Vertex` step-rate vertex buffer?")]
+ VertexBeyondLimit {
+ last_vertex: u64,
+ vertex_limit: u64,
+ slot: u32,
+ },
+ #[error("{step_mode:?} buffer out of bounds at slot {slot}. Offset {offset} beyond limit {limit}. Did you bind the correct `Vertex` step-rate vertex buffer?")]
+ VertexOutOfBounds {
+ step_mode: VertexStepMode,
+ offset: u64,
+ limit: u64,
+ slot: u32,
+ },
+ #[error("Instance {last_instance} extends beyond limit {instance_limit} imposed by the buffer in slot {slot}. Did you bind the correct `Instance` step-rate vertex buffer?")]
+ InstanceBeyondLimit {
+ last_instance: u64,
+ instance_limit: u64,
+ slot: u32,
+ },
+ #[error("Index {last_index} extends beyond limit {index_limit}. Did you bind the correct index buffer?")]
+ IndexBeyondLimit { last_index: u64, index_limit: u64 },
+ #[error(
+ "Pipeline index format ({pipeline:?}) and buffer index format ({buffer:?}) do not match"
+ )]
+ UnmatchedIndexFormats {
+ pipeline: wgt::IndexFormat,
+ buffer: wgt::IndexFormat,
+ },
+ #[error(transparent)]
+ BindingSizeTooSmall(#[from] LateMinBufferBindingSizeMismatch),
+}
+
+/// Error encountered when encoding a render command.
+/// This is the shared error set between render bundles and passes.
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum RenderCommandError {
+ #[error("Bind group {0:?} is invalid")]
+ InvalidBindGroup(id::BindGroupId),
+ #[error("Render bundle {0:?} is invalid")]
+ InvalidRenderBundle(id::RenderBundleId),
+ #[error("Bind group index {index} is greater than the device's requested `max_bind_group` limit {max}")]
+ BindGroupIndexOutOfRange { index: u32, max: u32 },
+ #[error("Vertex buffer index {index} is greater than the device's requested `max_vertex_buffers` limit {max}")]
+ VertexBufferIndexOutOfRange { index: u32, max: u32 },
+ #[error("Dynamic buffer offset {0} does not respect device's requested `{1}` limit {2}")]
+ UnalignedBufferOffset(u64, &'static str, u32),
+ #[error("Number of buffer offsets ({actual}) does not match the number of dynamic bindings ({expected})")]
+ InvalidDynamicOffsetCount { actual: usize, expected: usize },
+ #[error("Render pipeline {0:?} is invalid")]
+ InvalidPipeline(id::RenderPipelineId),
+ #[error("QuerySet {0:?} is invalid")]
+ InvalidQuerySet(id::QuerySetId),
+ #[error("Render pipeline targets are incompatible with render pass")]
+ IncompatiblePipelineTargets(#[from] crate::device::RenderPassCompatibilityError),
+ #[error("Pipeline writes to depth/stencil, while the pass has read-only depth/stencil")]
+ IncompatiblePipelineRods,
+ #[error(transparent)]
+ UsageConflict(#[from] UsageConflict),
+ #[error("Buffer {0:?} is destroyed")]
+ DestroyedBuffer(id::BufferId),
+ #[error(transparent)]
+ MissingBufferUsage(#[from] MissingBufferUsageError),
+ #[error(transparent)]
+ MissingTextureUsage(#[from] MissingTextureUsageError),
+ #[error(transparent)]
+ PushConstants(#[from] PushConstantUploadError),
+ #[error("Viewport has invalid rect {0:?}; origin and/or size is less than or equal to 0, and/or is not contained in the render target {1:?}")]
+ InvalidViewportRect(Rect<f32>, wgt::Extent3d),
+ #[error("Viewport minDepth {0} and/or maxDepth {1} are not in [0, 1]")]
+ InvalidViewportDepth(f32, f32),
+ #[error("Scissor {0:?} is not contained in the render target {1:?}")]
+ InvalidScissorRect(Rect<u32>, wgt::Extent3d),
+ #[error("Support for {0} is not implemented yet")]
+ Unimplemented(&'static str),
+}
+impl crate::error::PrettyError for RenderCommandError {
+ fn fmt_pretty(&self, fmt: &mut ErrorFormatter) {
+ fmt.error(self);
+ match *self {
+ Self::InvalidBindGroup(id) => {
+ fmt.bind_group_label(&id);
+ }
+ Self::InvalidPipeline(id) => {
+ fmt.render_pipeline_label(&id);
+ }
+ Self::UsageConflict(UsageConflict::TextureInvalid { id }) => {
+ fmt.texture_label(&id);
+ }
+ Self::UsageConflict(UsageConflict::BufferInvalid { id })
+ | Self::DestroyedBuffer(id) => {
+ fmt.buffer_label(&id);
+ }
+ _ => {}
+ };
+ }
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub struct Rect<T> {
+ pub x: T,
+ pub y: T,
+ pub w: T,
+ pub h: T,
+}
+
+#[doc(hidden)]
+#[derive(Clone, Copy, Debug)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub enum RenderCommand {
+ SetBindGroup {
+ index: u32,
+ num_dynamic_offsets: usize,
+ bind_group_id: id::BindGroupId,
+ },
+ SetPipeline(id::RenderPipelineId),
+ SetIndexBuffer {
+ buffer_id: id::BufferId,
+ index_format: wgt::IndexFormat,
+ offset: BufferAddress,
+ size: Option<BufferSize>,
+ },
+ SetVertexBuffer {
+ slot: u32,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ size: Option<BufferSize>,
+ },
+ SetBlendConstant(Color),
+ SetStencilReference(u32),
+ SetViewport {
+ rect: Rect<f32>,
+ //TODO: use half-float to reduce the size?
+ depth_min: f32,
+ depth_max: f32,
+ },
+ SetScissor(Rect<u32>),
+
+ /// Set a range of push constants to values stored in [`BasePass::push_constant_data`].
+ ///
+ /// See [`wgpu::RenderPass::set_push_constants`] for a detailed explanation
+ /// of the restrictions these commands must satisfy.
+ SetPushConstant {
+ /// Which stages we are setting push constant values for.
+ stages: wgt::ShaderStages,
+
+ /// The byte offset within the push constant storage to write to. This
+ /// must be a multiple of four.
+ offset: u32,
+
+ /// The number of bytes to write. This must be a multiple of four.
+ size_bytes: u32,
+
+ /// Index in [`BasePass::push_constant_data`] of the start of the data
+ /// to be written.
+ ///
+ /// Note: this is not a byte offset like `offset`. Rather, it is the
+ /// index of the first `u32` element in `push_constant_data` to read.
+ ///
+ /// `None` means zeros should be written to the destination range, and
+ /// there is no corresponding data in `push_constant_data`. This is used
+ /// by render bundles, which explicitly clear out any state that
+ /// post-bundle code might see.
+ values_offset: Option<u32>,
+ },
+ Draw {
+ vertex_count: u32,
+ instance_count: u32,
+ first_vertex: u32,
+ first_instance: u32,
+ },
+ DrawIndexed {
+ index_count: u32,
+ instance_count: u32,
+ first_index: u32,
+ base_vertex: i32,
+ first_instance: u32,
+ },
+ MultiDrawIndirect {
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ /// Count of `None` represents a non-multi call.
+ count: Option<NonZeroU32>,
+ indexed: bool,
+ },
+ MultiDrawIndirectCount {
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ count_buffer_id: id::BufferId,
+ count_buffer_offset: BufferAddress,
+ max_count: u32,
+ indexed: bool,
+ },
+ PushDebugGroup {
+ color: u32,
+ len: usize,
+ },
+ PopDebugGroup,
+ InsertDebugMarker {
+ color: u32,
+ len: usize,
+ },
+ WriteTimestamp {
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ },
+ BeginOcclusionQuery {
+ query_index: u32,
+ },
+ EndOcclusionQuery,
+ BeginPipelineStatisticsQuery {
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ },
+ EndPipelineStatisticsQuery,
+ ExecuteBundle(id::RenderBundleId),
+}
diff --git a/third_party/rust/wgpu-core/src/command/memory_init.rs b/third_party/rust/wgpu-core/src/command/memory_init.rs
new file mode 100644
index 0000000000..3bfc71f4f7
--- /dev/null
+++ b/third_party/rust/wgpu-core/src/command/memory_init.rs
@@ -0,0 +1,342 @@
+use std::{collections::hash_map::Entry, ops::Range, sync::Arc, vec::Drain};
+
+use hal::CommandEncoder;
+
+use crate::{
+ device::Device,
+ hal_api::HalApi,
+ init_tracker::*,
+ resource::{Resource, Texture},
+ track::{TextureTracker, Tracker},
+ FastHashMap,
+};
+
+use super::{
+ clear::clear_texture, BakedCommands, ClearError, DestroyedBufferError, DestroyedTextureError,
+};
+
+/// Surface that was discarded by `StoreOp::Discard` of a preceding renderpass.
+/// Any read access to this surface needs to be preceded by a texture initialization.
+#[derive(Clone)]
+pub(crate) struct TextureSurfaceDiscard<A: HalApi> {
+ pub texture: Arc<Texture<A>>,
+ pub mip_level: u32,
+ pub layer: u32,
+}
+
+pub(crate) type SurfacesInDiscardState<A> = Vec<TextureSurfaceDiscard<A>>;
+
+pub(crate) struct CommandBufferTextureMemoryActions<A: HalApi> {
+ /// The tracker actions that we need to be executed before the command
+ /// buffer is executed.
+ init_actions: Vec<TextureInitTrackerAction<A>>,
+ /// All the discards that haven't been followed by init again within the
+ /// command buffer i.e. everything in this list resets the texture init
+ /// state *after* the command buffer execution
+ discards: Vec<TextureSurfaceDiscard<A>>,
+}
+
+impl<A: HalApi> Default for CommandBufferTextureMemoryActions<A> {
+ fn default() -> Self {
+ Self {
+ init_actions: Default::default(),
+ discards: Default::default(),
+ }
+ }
+}
+
+impl<A: HalApi> CommandBufferTextureMemoryActions<A> {
+ pub(crate) fn drain_init_actions(&mut self) -> Drain<TextureInitTrackerAction<A>> {
+ self.init_actions.drain(..)
+ }
+
+ pub(crate) fn discard(&mut self, discard: TextureSurfaceDiscard<A>) {
+ self.discards.push(discard);
+ }
+
+ // Registers a TextureInitTrackerAction.
+ // Returns previously discarded surface that need to be initialized *immediately* now.
+ // Only returns a non-empty list if action is MemoryInitKind::NeedsInitializedMemory.
+ #[must_use]
+ pub(crate) fn register_init_action(
+ &mut self,
+ action: &TextureInitTrackerAction<A>,
+ ) -> SurfacesInDiscardState<A> {
+ let mut immediately_necessary_clears = SurfacesInDiscardState::new();
+
+ // Note that within a command buffer we may stack arbitrary memory init
+ // actions on the same texture Since we react to them in sequence, they
+ // are going to be dropped again at queue submit
+ //
+ // We don't need to add MemoryInitKind::NeedsInitializedMemory to
+ // init_actions if a surface is part of the discard list. But that would
+ // mean splitting up the action which is more than we'd win here.
+ self.init_actions.extend(
+ action
+ .texture
+ .initialization_status
+ .read()
+ .check_action(action),
+ );
+
+ // We expect very few discarded surfaces at any point in time which is
+ // why a simple linear search is likely best. (i.e. most of the time
+ // self.discards is empty!)
+ let init_actions = &mut self.init_actions;
+ self.discards.retain(|discarded_surface| {
+ if discarded_surface.texture.as_info().id() == action.texture.as_info().id()
+ && action.range.layer_range.contains(&discarded_surface.layer)
+ && action
+ .range
+ .mip_range
+ .contains(&discarded_surface.mip_level)
+ {
+ if let MemoryInitKind::NeedsInitializedMemory = action.kind {
+ immediately_necessary_clears.push(discarded_surface.clone());
+
+ // Mark surface as implicitly initialized (this is relevant
+ // because it might have been uninitialized prior to
+ // discarding
+ init_actions.push(TextureInitTrackerAction {
+ texture: discarded_surface.texture.clone(),
+ range: TextureInitRange {
+ mip_range: discarded_surface.mip_level
+ ..(discarded_surface.mip_level + 1),
+ layer_range: discarded_surface.layer..(discarded_surface.layer + 1),
+ },
+ kind: MemoryInitKind::ImplicitlyInitialized,
+ });
+ }
+ false
+ } else {
+ true
+ }
+ });
+
+ immediately_necessary_clears
+ }
+
+ // Shortcut for register_init_action when it is known that the action is an
+ // implicit init, not requiring any immediate resource init.
+ pub(crate) fn register_implicit_init(
+ &mut self,
+ texture: &Arc<Texture<A>>,
+ range: TextureInitRange,
+ ) {
+ let must_be_empty = self.register_init_action(&TextureInitTrackerAction {
+ texture: texture.clone(),
+ range,
+ kind: MemoryInitKind::ImplicitlyInitialized,
+ });
+ assert!(must_be_empty.is_empty());
+ }
+}
+
+// Utility function that takes discarded surfaces from (several calls to)
+// register_init_action and initializes them on the spot.
+//
+// Takes care of barriers as well!
+pub(crate) fn fixup_discarded_surfaces<
+ A: HalApi,
+ InitIter: Iterator<Item = TextureSurfaceDiscard<A>>,
+>(
+ inits: InitIter,
+ encoder: &mut A::CommandEncoder,
+ texture_tracker: &mut TextureTracker<A>,
+ device: &Device<A>,
+) {
+ for init in inits {
+ clear_texture(
+ &init.texture,
+ TextureInitRange {
+ mip_range: init.mip_level..(init.mip_level + 1),
+ layer_range: init.layer..(init.layer + 1),
+ },
+ encoder,
+ texture_tracker,
+ &device.alignments,
+ device.zero_buffer.as_ref().unwrap(),
+ )
+ .unwrap();
+ }
+}
+
+impl<A: HalApi> BakedCommands<A> {
+ // inserts all buffer initializations that are going to be needed for
+ // executing the commands and updates resource init states accordingly
+ pub(crate) fn initialize_buffer_memory(
+ &mut self,
+ device_tracker: &mut Tracker<A>,
+ ) -> Result<(), DestroyedBufferError> {
+ // Gather init ranges for each buffer so we can collapse them.
+ // It is not possible to do this at an earlier point since previously
+ // executed command buffer change the resource init state.
+ let mut uninitialized_ranges_per_buffer = FastHashMap::default();
+ for buffer_use in self.buffer_memory_init_actions.drain(..) {
+ let mut initialization_status = buffer_use.buffer.initialization_status.write();
+
+ // align the end to 4
+ let end_remainder = buffer_use.range.end % wgt::COPY_BUFFER_ALIGNMENT;
+ let end = if end_remainder == 0 {
+ buffer_use.range.end
+ } else {
+ buffer_use.range.end + wgt::COPY_BUFFER_ALIGNMENT - end_remainder
+ };
+ let uninitialized_ranges = initialization_status.drain(buffer_use.range.start..end);
+
+ match buffer_use.kind {
+ MemoryInitKind::ImplicitlyInitialized => {}
+ MemoryInitKind::NeedsInitializedMemory => {
+ match uninitialized_ranges_per_buffer.entry(buffer_use.buffer.as_info().id()) {
+ Entry::Vacant(e) => {
+ e.insert((
+ buffer_use.buffer.clone(),
+ uninitialized_ranges.collect::<Vec<Range<wgt::BufferAddress>>>(),
+ ));
+ }
+ Entry::Occupied(mut e) => {
+ e.get_mut().1.extend(uninitialized_ranges);
+ }
+ }
+ }
+ }
+ }
+
+ for (buffer_id, (buffer, mut ranges)) in uninitialized_ranges_per_buffer {
+ // Collapse touching ranges.
+ ranges.sort_by_key(|r| r.start);
+ for i in (1..ranges.len()).rev() {
+ // The memory init tracker made sure of this!
+ assert!(ranges[i - 1].end <= ranges[i].start);
+ if ranges[i].start == ranges[i - 1].end {
+ ranges[i - 1].end = ranges[i].end;
+ ranges.swap_remove(i); // Ordering not important at this point
+ }
+ }
+
+ // Don't do use_replace since the buffer may already no longer have
+ // a ref_count.
+ //
+ // However, we *know* that it is currently in use, so the tracker
+ // must already know about it.
+ let transition = device_tracker
+ .buffers
+ .set_single(&buffer, hal::BufferUses::COPY_DST)
+ .unwrap()
+ .1;
+
+ let snatch_guard = buffer.device.snatchable_lock.read();
+ let raw_buf = buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(DestroyedBufferError(buffer_id))?;
+
+ unsafe {
+ self.encoder.transition_buffers(
+ transition
+ .map(|pending| pending.into_hal(&buffer, &snatch_guard))
+ .into_iter(),
+ );
+ }
+
+ for range in ranges.iter() {
+ assert!(
+ range.start % wgt::COPY_BUFFER_ALIGNMENT == 0,
+ "Buffer {:?} has an uninitialized range with a start \
+ not aligned to 4 (start was {})",
+ raw_buf,
+ range.start
+ );
+ assert!(
+ range.end % wgt::COPY_BUFFER_ALIGNMENT == 0,
+ "Buffer {:?} has an uninitialized range with an end \
+ not aligned to 4 (end was {})",
+ raw_buf,
+ range.end
+ );
+
+ unsafe {
+ self.encoder.clear_buffer(raw_buf, range.clone());
+ }
+ }
+ }
+ Ok(())
+ }
+
+ // inserts all texture initializations that are going to be needed for
+ // executing the commands and updates resource init states accordingly any
+ // textures that are left discarded by this command buffer will be marked as
+ // uninitialized
+ pub(crate) fn initialize_texture_memory(
+ &mut self,
+ device_tracker: &mut Tracker<A>,
+ device: &Device<A>,
+ ) -> Result<(), DestroyedTextureError> {
+ let mut ranges: Vec<TextureInitRange> = Vec::new();
+ for texture_use in self.texture_memory_actions.drain_init_actions() {
+ let mut initialization_status = texture_use.texture.initialization_status.write();
+ let use_range = texture_use.range;
+ let affected_mip_trackers = initialization_status
+ .mips
+ .iter_mut()
+ .enumerate()
+ .skip(use_range.mip_range.start as usize)
+ .take((use_range.mip_range.end - use_range.mip_range.start) as usize);
+
+ match texture_use.kind {
+ MemoryInitKind::ImplicitlyInitialized => {
+ for (_, mip_tracker) in affected_mip_trackers {
+ mip_tracker.drain(use_range.layer_range.clone());
+ }
+ }
+ MemoryInitKind::NeedsInitializedMemory => {
+ for (mip_level, mip_tracker) in affected_mip_trackers {
+ for layer_range in mip_tracker.drain(use_range.layer_range.clone()) {
+ ranges.push(TextureInitRange {
+ mip_range: (mip_level as u32)..(mip_level as u32 + 1),
+ layer_range,
+ });
+ }
+ }
+ }
+ }
+
+ // TODO: Could we attempt some range collapsing here?
+ for range in ranges.drain(..) {
+ let clear_result = clear_texture(
+ &texture_use.texture,
+ range,
+ &mut self.encoder,
+ &mut device_tracker.textures,
+ &device.alignments,
+ device.zero_buffer.as_ref().unwrap(),
+ );
+
+ // A Texture can be destroyed between the command recording
+ // and now, this is out of our control so we have to handle
+ // it gracefully.
+ if let Err(ClearError::InvalidTexture(id)) = clear_result {
+ return Err(DestroyedTextureError(id));
+ }
+
+ // Other errors are unexpected.
+ if let Err(error) = clear_result {
+ panic!("{error}");
+ }
+ }
+ }
+
+ // Now that all buffers/textures have the proper init state for before
+ // cmdbuf start, we discard init states for textures it left discarded
+ // after its execution.
+ for surface_discard in self.texture_memory_actions.discards.iter() {
+ surface_discard
+ .texture
+ .initialization_status
+ .write()
+ .discard(surface_discard.mip_level, surface_discard.layer);
+ }
+
+ Ok(())
+ }
+}
diff --git a/third_party/rust/wgpu-core/src/command/mod.rs b/third_party/rust/wgpu-core/src/command/mod.rs
new file mode 100644
index 0000000000..2d5fca200a
--- /dev/null
+++ b/third_party/rust/wgpu-core/src/command/mod.rs
@@ -0,0 +1,732 @@
+mod bind;
+mod bundle;
+mod clear;
+mod compute;
+mod draw;
+mod memory_init;
+mod query;
+mod render;
+mod transfer;
+
+use std::slice;
+use std::sync::Arc;
+
+pub(crate) use self::clear::clear_texture;
+pub use self::{
+ bundle::*, clear::ClearError, compute::*, draw::*, query::*, render::*, transfer::*,
+};
+
+use self::memory_init::CommandBufferTextureMemoryActions;
+
+use crate::device::{Device, DeviceError};
+use crate::error::{ErrorFormatter, PrettyError};
+use crate::hub::Hub;
+use crate::id::CommandBufferId;
+use crate::snatch::SnatchGuard;
+
+use crate::init_tracker::BufferInitTrackerAction;
+use crate::resource::{Resource, ResourceInfo, ResourceType};
+use crate::track::{Tracker, UsageScope};
+use crate::{api_log, global::Global, hal_api::HalApi, id, resource_log, Label};
+
+use hal::CommandEncoder as _;
+use parking_lot::Mutex;
+use thiserror::Error;
+
+#[cfg(feature = "trace")]
+use crate::device::trace::Command as TraceCommand;
+
+const PUSH_CONSTANT_CLEAR_ARRAY: &[u32] = &[0_u32; 64];
+
+#[derive(Debug)]
+pub(crate) enum CommandEncoderStatus {
+ Recording,
+ Finished,
+ Error,
+}
+
+pub(crate) struct CommandEncoder<A: HalApi> {
+ raw: A::CommandEncoder,
+ list: Vec<A::CommandBuffer>,
+ is_open: bool,
+ label: Option<String>,
+}
+
+//TODO: handle errors better
+impl<A: HalApi> CommandEncoder<A> {
+ /// Closes the live encoder
+ fn close_and_swap(&mut self) -> Result<(), DeviceError> {
+ if self.is_open {
+ self.is_open = false;
+ let new = unsafe { self.raw.end_encoding()? };
+ self.list.insert(self.list.len() - 1, new);
+ }
+
+ Ok(())
+ }
+
+ fn close(&mut self) -> Result<(), DeviceError> {
+ if self.is_open {
+ self.is_open = false;
+ let cmd_buf = unsafe { self.raw.end_encoding()? };
+ self.list.push(cmd_buf);
+ }
+
+ Ok(())
+ }
+
+ fn discard(&mut self) {
+ if self.is_open {
+ self.is_open = false;
+ unsafe { self.raw.discard_encoding() };
+ }
+ }
+
+ fn open(&mut self) -> Result<&mut A::CommandEncoder, DeviceError> {
+ if !self.is_open {
+ self.is_open = true;
+ let label = self.label.as_deref();
+ unsafe { self.raw.begin_encoding(label)? };
+ }
+
+ Ok(&mut self.raw)
+ }
+
+ fn open_pass(&mut self, label: Option<&str>) -> Result<(), DeviceError> {
+ self.is_open = true;
+ unsafe { self.raw.begin_encoding(label)? };
+
+ Ok(())
+ }
+}
+
+pub struct BakedCommands<A: HalApi> {
+ pub(crate) encoder: A::CommandEncoder,
+ pub(crate) list: Vec<A::CommandBuffer>,
+ pub(crate) trackers: Tracker<A>,
+ buffer_memory_init_actions: Vec<BufferInitTrackerAction<A>>,
+ texture_memory_actions: CommandBufferTextureMemoryActions<A>,
+}
+
+pub(crate) struct DestroyedBufferError(pub id::BufferId);
+pub(crate) struct DestroyedTextureError(pub id::TextureId);
+
+pub struct CommandBufferMutable<A: HalApi> {
+ encoder: CommandEncoder<A>,
+ status: CommandEncoderStatus,
+ pub(crate) trackers: Tracker<A>,
+ buffer_memory_init_actions: Vec<BufferInitTrackerAction<A>>,
+ texture_memory_actions: CommandBufferTextureMemoryActions<A>,
+ pub(crate) pending_query_resets: QueryResetMap<A>,
+ #[cfg(feature = "trace")]
+ pub(crate) commands: Option<Vec<TraceCommand>>,
+}
+
+impl<A: HalApi> CommandBufferMutable<A> {
+ pub(crate) fn open_encoder_and_tracker(
+ &mut self,
+ ) -> Result<(&mut A::CommandEncoder, &mut Tracker<A>), DeviceError> {
+ let encoder = self.encoder.open()?;
+ let tracker = &mut self.trackers;
+
+ Ok((encoder, tracker))
+ }
+}
+
+pub struct CommandBuffer<A: HalApi> {
+ pub(crate) device: Arc<Device<A>>,
+ limits: wgt::Limits,
+ support_clear_texture: bool,
+ pub(crate) info: ResourceInfo<CommandBuffer<A>>,
+ pub(crate) data: Mutex<Option<CommandBufferMutable<A>>>,
+}
+
+impl<A: HalApi> Drop for CommandBuffer<A> {
+ fn drop(&mut self) {
+ if self.data.lock().is_none() {
+ return;
+ }
+ resource_log!("resource::CommandBuffer::drop {:?}", self.info.label());
+ let mut baked = self.extract_baked_commands();
+ unsafe {
+ baked.encoder.reset_all(baked.list.into_iter());
+ }
+ unsafe {
+ use hal::Device;
+ self.device.raw().destroy_command_encoder(baked.encoder);
+ }
+ }
+}
+
+impl<A: HalApi> CommandBuffer<A> {
+ pub(crate) fn new(
+ encoder: A::CommandEncoder,
+ device: &Arc<Device<A>>,
+ #[cfg(feature = "trace")] enable_tracing: bool,
+ label: Option<String>,
+ ) -> Self {
+ CommandBuffer {
+ device: device.clone(),
+ limits: device.limits.clone(),
+ support_clear_texture: device.features.contains(wgt::Features::CLEAR_TEXTURE),
+ info: ResourceInfo::new(
+ label
+ .as_ref()
+ .unwrap_or(&String::from("<CommandBuffer>"))
+ .as_str(),
+ ),
+ data: Mutex::new(Some(CommandBufferMutable {
+ encoder: CommandEncoder {
+ raw: encoder,
+ is_open: false,
+ list: Vec::new(),
+ label,
+ },
+ status: CommandEncoderStatus::Recording,
+ trackers: Tracker::new(),
+ buffer_memory_init_actions: Default::default(),
+ texture_memory_actions: Default::default(),
+ pending_query_resets: QueryResetMap::new(),
+ #[cfg(feature = "trace")]
+ commands: if enable_tracing {
+ Some(Vec::new())
+ } else {
+ None
+ },
+ })),
+ }
+ }
+
+ pub(crate) fn insert_barriers_from_tracker(
+ raw: &mut A::CommandEncoder,
+ base: &mut Tracker<A>,
+ head: &Tracker<A>,
+ snatch_guard: &SnatchGuard,
+ ) {
+ profiling::scope!("insert_barriers");
+
+ base.buffers.set_from_tracker(&head.buffers);
+ base.textures.set_from_tracker(&head.textures);
+
+ Self::drain_barriers(raw, base, snatch_guard);
+ }
+
+ pub(crate) fn insert_barriers_from_scope(
+ raw: &mut A::CommandEncoder,
+ base: &mut Tracker<A>,
+ head: &UsageScope<A>,
+ snatch_guard: &SnatchGuard,
+ ) {
+ profiling::scope!("insert_barriers");
+
+ base.buffers.set_from_usage_scope(&head.buffers);
+ base.textures.set_from_usage_scope(&head.textures);
+
+ Self::drain_barriers(raw, base, snatch_guard);
+ }
+
+ pub(crate) fn drain_barriers(
+ raw: &mut A::CommandEncoder,
+ base: &mut Tracker<A>,
+ snatch_guard: &SnatchGuard,
+ ) {
+ profiling::scope!("drain_barriers");
+
+ let buffer_barriers = base.buffers.drain_transitions(snatch_guard);
+ let (transitions, textures) = base.textures.drain_transitions(snatch_guard);
+ let texture_barriers = transitions
+ .into_iter()
+ .enumerate()
+ .map(|(i, p)| p.into_hal(textures[i].unwrap().raw().unwrap()));
+
+ unsafe {
+ raw.transition_buffers(buffer_barriers);
+ raw.transition_textures(texture_barriers);
+ }
+ }
+}
+
+impl<A: HalApi> CommandBuffer<A> {
+ fn get_encoder(
+ hub: &Hub<A>,
+ id: id::CommandEncoderId,
+ ) -> Result<Arc<Self>, CommandEncoderError> {
+ let storage = hub.command_buffers.read();
+ match storage.get(id.transmute()) {
+ Ok(cmd_buf) => match cmd_buf.data.lock().as_ref().unwrap().status {
+ CommandEncoderStatus::Recording => Ok(cmd_buf.clone()),
+ CommandEncoderStatus::Finished => Err(CommandEncoderError::NotRecording),
+ CommandEncoderStatus::Error => Err(CommandEncoderError::Invalid),
+ },
+ Err(_) => Err(CommandEncoderError::Invalid),
+ }
+ }
+
+ pub fn is_finished(&self) -> bool {
+ match self.data.lock().as_ref().unwrap().status {
+ CommandEncoderStatus::Finished => true,
+ _ => false,
+ }
+ }
+
+ pub(crate) fn extract_baked_commands(&mut self) -> BakedCommands<A> {
+ log::trace!(
+ "Extracting BakedCommands from CommandBuffer {:?}",
+ self.info.label()
+ );
+ let data = self.data.lock().take().unwrap();
+ BakedCommands {
+ encoder: data.encoder.raw,
+ list: data.encoder.list,
+ trackers: data.trackers,
+ buffer_memory_init_actions: data.buffer_memory_init_actions,
+ texture_memory_actions: data.texture_memory_actions,
+ }
+ }
+
+ pub(crate) fn from_arc_into_baked(self: Arc<Self>) -> BakedCommands<A> {
+ if let Some(mut command_buffer) = Arc::into_inner(self) {
+ command_buffer.extract_baked_commands()
+ } else {
+ panic!("CommandBuffer cannot be destroyed because is still in use");
+ }
+ }
+}
+
+impl<A: HalApi> Resource for CommandBuffer<A> {
+ const TYPE: ResourceType = "CommandBuffer";
+
+ type Marker = crate::id::markers::CommandBuffer;
+
+ fn as_info(&self) -> &ResourceInfo<Self> {
+ &self.info
+ }
+
+ fn as_info_mut(&mut self) -> &mut ResourceInfo<Self> {
+ &mut self.info
+ }
+
+ fn label(&self) -> String {
+ let str = match self.data.lock().as_ref().unwrap().encoder.label.as_ref() {
+ Some(label) => label.clone(),
+ _ => String::new(),
+ };
+ str
+ }
+}
+
+#[derive(Copy, Clone, Debug)]
+pub struct BasePassRef<'a, C> {
+ pub label: Option<&'a str>,
+ pub commands: &'a [C],
+ pub dynamic_offsets: &'a [wgt::DynamicOffset],
+ pub string_data: &'a [u8],
+ pub push_constant_data: &'a [u32],
+}
+
+/// A stream of commands for a render pass or compute pass.
+///
+/// This also contains side tables referred to by certain commands,
+/// like dynamic offsets for [`SetBindGroup`] or string data for
+/// [`InsertDebugMarker`].
+///
+/// Render passes use `BasePass<RenderCommand>`, whereas compute
+/// passes use `BasePass<ComputeCommand>`.
+///
+/// [`SetBindGroup`]: RenderCommand::SetBindGroup
+/// [`InsertDebugMarker`]: RenderCommand::InsertDebugMarker
+#[doc(hidden)]
+#[derive(Debug)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub struct BasePass<C> {
+ pub label: Option<String>,
+
+ /// The stream of commands.
+ pub commands: Vec<C>,
+
+ /// Dynamic offsets consumed by [`SetBindGroup`] commands in `commands`.
+ ///
+ /// Each successive `SetBindGroup` consumes the next
+ /// [`num_dynamic_offsets`] values from this list.
+ pub dynamic_offsets: Vec<wgt::DynamicOffset>,
+
+ /// Strings used by debug instructions.
+ ///
+ /// Each successive [`PushDebugGroup`] or [`InsertDebugMarker`]
+ /// instruction consumes the next `len` bytes from this vector.
+ pub string_data: Vec<u8>,
+
+ /// Data used by `SetPushConstant` instructions.
+ ///
+ /// See the documentation for [`RenderCommand::SetPushConstant`]
+ /// and [`ComputeCommand::SetPushConstant`] for details.
+ pub push_constant_data: Vec<u32>,
+}
+
+impl<C: Clone> BasePass<C> {
+ fn new(label: &Label) -> Self {
+ Self {
+ label: label.as_ref().map(|cow| cow.to_string()),
+ commands: Vec::new(),
+ dynamic_offsets: Vec::new(),
+ string_data: Vec::new(),
+ push_constant_data: Vec::new(),
+ }
+ }
+
+ #[cfg(feature = "trace")]
+ fn from_ref(base: BasePassRef<C>) -> Self {
+ Self {
+ label: base.label.map(str::to_string),
+ commands: base.commands.to_vec(),
+ dynamic_offsets: base.dynamic_offsets.to_vec(),
+ string_data: base.string_data.to_vec(),
+ push_constant_data: base.push_constant_data.to_vec(),
+ }
+ }
+
+ pub fn as_ref(&self) -> BasePassRef<C> {
+ BasePassRef {
+ label: self.label.as_deref(),
+ commands: &self.commands,
+ dynamic_offsets: &self.dynamic_offsets,
+ string_data: &self.string_data,
+ push_constant_data: &self.push_constant_data,
+ }
+ }
+}
+
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum CommandEncoderError {
+ #[error("Command encoder is invalid")]
+ Invalid,
+ #[error("Command encoder must be active")]
+ NotRecording,
+ #[error(transparent)]
+ Device(#[from] DeviceError),
+}
+
+impl Global {
+ pub fn command_encoder_finish<A: HalApi>(
+ &self,
+ encoder_id: id::CommandEncoderId,
+ _desc: &wgt::CommandBufferDescriptor<Label>,
+ ) -> (CommandBufferId, Option<CommandEncoderError>) {
+ profiling::scope!("CommandEncoder::finish");
+
+ let hub = A::hub(self);
+
+ let error = match hub.command_buffers.get(encoder_id.transmute()) {
+ Ok(cmd_buf) => {
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+ match cmd_buf_data.status {
+ CommandEncoderStatus::Recording => {
+ if let Err(e) = cmd_buf_data.encoder.close() {
+ Some(e.into())
+ } else {
+ cmd_buf_data.status = CommandEncoderStatus::Finished;
+ //Note: if we want to stop tracking the swapchain texture view,
+ // this is the place to do it.
+ log::trace!("Command buffer {:?}", encoder_id);
+ None
+ }
+ }
+ CommandEncoderStatus::Finished => Some(CommandEncoderError::NotRecording),
+ CommandEncoderStatus::Error => {
+ cmd_buf_data.encoder.discard();
+ Some(CommandEncoderError::Invalid)
+ }
+ }
+ }
+ Err(_) => Some(CommandEncoderError::Invalid),
+ };
+
+ (encoder_id.transmute(), error)
+ }
+
+ pub fn command_encoder_push_debug_group<A: HalApi>(
+ &self,
+ encoder_id: id::CommandEncoderId,
+ label: &str,
+ ) -> Result<(), CommandEncoderError> {
+ profiling::scope!("CommandEncoder::push_debug_group");
+ api_log!("CommandEncoder::push_debug_group {label}");
+
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id)?;
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::PushDebugGroup(label.to_string()));
+ }
+
+ let cmd_buf_raw = cmd_buf_data.encoder.open()?;
+ if !self
+ .instance
+ .flags
+ .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS)
+ {
+ unsafe {
+ cmd_buf_raw.begin_debug_marker(label);
+ }
+ }
+ Ok(())
+ }
+
+ pub fn command_encoder_insert_debug_marker<A: HalApi>(
+ &self,
+ encoder_id: id::CommandEncoderId,
+ label: &str,
+ ) -> Result<(), CommandEncoderError> {
+ profiling::scope!("CommandEncoder::insert_debug_marker");
+ api_log!("CommandEncoder::insert_debug_marker {label}");
+
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id)?;
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::InsertDebugMarker(label.to_string()));
+ }
+
+ if !self
+ .instance
+ .flags
+ .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS)
+ {
+ let cmd_buf_raw = cmd_buf_data.encoder.open()?;
+ unsafe {
+ cmd_buf_raw.insert_debug_marker(label);
+ }
+ }
+ Ok(())
+ }
+
+ pub fn command_encoder_pop_debug_group<A: HalApi>(
+ &self,
+ encoder_id: id::CommandEncoderId,
+ ) -> Result<(), CommandEncoderError> {
+ profiling::scope!("CommandEncoder::pop_debug_marker");
+ api_log!("CommandEncoder::pop_debug_group");
+
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id)?;
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::PopDebugGroup);
+ }
+
+ let cmd_buf_raw = cmd_buf_data.encoder.open()?;
+ if !self
+ .instance
+ .flags
+ .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS)
+ {
+ unsafe {
+ cmd_buf_raw.end_debug_marker();
+ }
+ }
+ Ok(())
+ }
+}
+
+fn push_constant_clear<PushFn>(offset: u32, size_bytes: u32, mut push_fn: PushFn)
+where
+ PushFn: FnMut(u32, &[u32]),
+{
+ let mut count_words = 0_u32;
+ let size_words = size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT;
+ while count_words < size_words {
+ let count_bytes = count_words * wgt::PUSH_CONSTANT_ALIGNMENT;
+ let size_to_write_words =
+ (size_words - count_words).min(PUSH_CONSTANT_CLEAR_ARRAY.len() as u32);
+
+ push_fn(
+ offset + count_bytes,
+ &PUSH_CONSTANT_CLEAR_ARRAY[0..size_to_write_words as usize],
+ );
+
+ count_words += size_to_write_words;
+ }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct StateChange<T> {
+ last_state: Option<T>,
+}
+
+impl<T: Copy + PartialEq> StateChange<T> {
+ fn new() -> Self {
+ Self { last_state: None }
+ }
+ fn set_and_check_redundant(&mut self, new_state: T) -> bool {
+ let already_set = self.last_state == Some(new_state);
+ self.last_state = Some(new_state);
+ already_set
+ }
+ fn reset(&mut self) {
+ self.last_state = None;
+ }
+}
+
+impl<T: Copy + PartialEq> Default for StateChange<T> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+#[derive(Debug)]
+struct BindGroupStateChange {
+ last_states: [StateChange<id::BindGroupId>; hal::MAX_BIND_GROUPS],
+}
+
+impl BindGroupStateChange {
+ fn new() -> Self {
+ Self {
+ last_states: [StateChange::new(); hal::MAX_BIND_GROUPS],
+ }
+ }
+
+ unsafe fn set_and_check_redundant(
+ &mut self,
+ bind_group_id: id::BindGroupId,
+ index: u32,
+ dynamic_offsets: &mut Vec<u32>,
+ offsets: *const wgt::DynamicOffset,
+ offset_length: usize,
+ ) -> bool {
+ // For now never deduplicate bind groups with dynamic offsets.
+ if offset_length == 0 {
+ // If this get returns None, that means we're well over the limit,
+ // so let the call through to get a proper error
+ if let Some(current_bind_group) = self.last_states.get_mut(index as usize) {
+ // Bail out if we're binding the same bind group.
+ if current_bind_group.set_and_check_redundant(bind_group_id) {
+ return true;
+ }
+ }
+ } else {
+ // We intentionally remove the memory of this bind group if we have dynamic offsets,
+ // such that if you try to bind this bind group later with _no_ dynamic offsets it
+ // tries to bind it again and gives a proper validation error.
+ if let Some(current_bind_group) = self.last_states.get_mut(index as usize) {
+ current_bind_group.reset();
+ }
+ dynamic_offsets
+ .extend_from_slice(unsafe { slice::from_raw_parts(offsets, offset_length) });
+ }
+ false
+ }
+ fn reset(&mut self) {
+ self.last_states = [StateChange::new(); hal::MAX_BIND_GROUPS];
+ }
+}
+
+impl Default for BindGroupStateChange {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+trait MapPassErr<T, O> {
+ fn map_pass_err(self, scope: PassErrorScope) -> Result<T, O>;
+}
+
+#[derive(Clone, Copy, Debug, Error)]
+pub enum PassErrorScope {
+ #[error("In a bundle parameter")]
+ Bundle,
+ #[error("In a pass parameter")]
+ Pass(id::CommandEncoderId),
+ #[error("In a set_bind_group command")]
+ SetBindGroup(id::BindGroupId),
+ #[error("In a set_pipeline command")]
+ SetPipelineRender(id::RenderPipelineId),
+ #[error("In a set_pipeline command")]
+ SetPipelineCompute(id::ComputePipelineId),
+ #[error("In a set_push_constant command")]
+ SetPushConstant,
+ #[error("In a set_vertex_buffer command")]
+ SetVertexBuffer(id::BufferId),
+ #[error("In a set_index_buffer command")]
+ SetIndexBuffer(id::BufferId),
+ #[error("In a set_viewport command")]
+ SetViewport,
+ #[error("In a set_scissor_rect command")]
+ SetScissorRect,
+ #[error("In a draw command, indexed:{indexed} indirect:{indirect}")]
+ Draw {
+ indexed: bool,
+ indirect: bool,
+ pipeline: Option<id::RenderPipelineId>,
+ },
+ #[error("While resetting queries after the renderpass was ran")]
+ QueryReset,
+ #[error("In a write_timestamp command")]
+ WriteTimestamp,
+ #[error("In a begin_occlusion_query command")]
+ BeginOcclusionQuery,
+ #[error("In a end_occlusion_query command")]
+ EndOcclusionQuery,
+ #[error("In a begin_pipeline_statistics_query command")]
+ BeginPipelineStatisticsQuery,
+ #[error("In a end_pipeline_statistics_query command")]
+ EndPipelineStatisticsQuery,
+ #[error("In a execute_bundle command")]
+ ExecuteBundle,
+ #[error("In a dispatch command, indirect:{indirect}")]
+ Dispatch {
+ indirect: bool,
+ pipeline: Option<id::ComputePipelineId>,
+ },
+ #[error("In a pop_debug_group command")]
+ PopDebugGroup,
+}
+
+impl PrettyError for PassErrorScope {
+ fn fmt_pretty(&self, fmt: &mut ErrorFormatter) {
+ // This error is not in the error chain, only notes are needed
+ match *self {
+ Self::Pass(id) => {
+ fmt.command_buffer_label(&id.transmute());
+ }
+ Self::SetBindGroup(id) => {
+ fmt.bind_group_label(&id);
+ }
+ Self::SetPipelineRender(id) => {
+ fmt.render_pipeline_label(&id);
+ }
+ Self::SetPipelineCompute(id) => {
+ fmt.compute_pipeline_label(&id);
+ }
+ Self::SetVertexBuffer(id) => {
+ fmt.buffer_label(&id);
+ }
+ Self::SetIndexBuffer(id) => {
+ fmt.buffer_label(&id);
+ }
+ Self::Draw {
+ pipeline: Some(id), ..
+ } => {
+ fmt.render_pipeline_label(&id);
+ }
+ Self::Dispatch {
+ pipeline: Some(id), ..
+ } => {
+ fmt.compute_pipeline_label(&id);
+ }
+ _ => {}
+ }
+ }
+}
diff --git a/third_party/rust/wgpu-core/src/command/query.rs b/third_party/rust/wgpu-core/src/command/query.rs
new file mode 100644
index 0000000000..39d7a9cc93
--- /dev/null
+++ b/third_party/rust/wgpu-core/src/command/query.rs
@@ -0,0 +1,501 @@
+use hal::CommandEncoder as _;
+
+#[cfg(feature = "trace")]
+use crate::device::trace::Command as TraceCommand;
+use crate::{
+ command::{CommandBuffer, CommandEncoderError},
+ device::DeviceError,
+ global::Global,
+ hal_api::HalApi,
+ id::{self, Id},
+ init_tracker::MemoryInitKind,
+ resource::QuerySet,
+ storage::Storage,
+ Epoch, FastHashMap, Index,
+};
+use std::{iter, marker::PhantomData};
+use thiserror::Error;
+use wgt::BufferAddress;
+
+#[derive(Debug)]
+pub(crate) struct QueryResetMap<A: HalApi> {
+ map: FastHashMap<Index, (Vec<bool>, Epoch)>,
+ _phantom: PhantomData<A>,
+}
+impl<A: HalApi> QueryResetMap<A> {
+ pub fn new() -> Self {
+ Self {
+ map: FastHashMap::default(),
+ _phantom: PhantomData,
+ }
+ }
+
+ pub fn use_query_set(
+ &mut self,
+ id: id::QuerySetId,
+ query_set: &QuerySet<A>,
+ query: u32,
+ ) -> bool {
+ let (index, epoch, _) = id.unzip();
+ let vec_pair = self
+ .map
+ .entry(index)
+ .or_insert_with(|| (vec![false; query_set.desc.count as usize], epoch));
+
+ std::mem::replace(&mut vec_pair.0[query as usize], true)
+ }
+
+ pub fn reset_queries(
+ &mut self,
+ raw_encoder: &mut A::CommandEncoder,
+ query_set_storage: &Storage<QuerySet<A>>,
+ backend: wgt::Backend,
+ ) -> Result<(), id::QuerySetId> {
+ for (query_set_id, (state, epoch)) in self.map.drain() {
+ let id = Id::zip(query_set_id, epoch, backend);
+ let query_set = query_set_storage.get(id).map_err(|_| id)?;
+
+ debug_assert_eq!(state.len(), query_set.desc.count as usize);
+
+ // Need to find all "runs" of values which need resets. If the state vector is:
+ // [false, true, true, false, true], we want to reset [1..3, 4..5]. This minimizes
+ // the amount of resets needed.
+ let mut run_start: Option<u32> = None;
+ for (idx, value) in state.into_iter().chain(iter::once(false)).enumerate() {
+ match (run_start, value) {
+ // We're inside of a run, do nothing
+ (Some(..), true) => {}
+ // We've hit the end of a run, dispatch a reset
+ (Some(start), false) => {
+ run_start = None;
+ unsafe { raw_encoder.reset_queries(query_set.raw(), start..idx as u32) };
+ }
+ // We're starting a run
+ (None, true) => {
+ run_start = Some(idx as u32);
+ }
+ // We're in a run of falses, do nothing.
+ (None, false) => {}
+ }
+ }
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum SimplifiedQueryType {
+ Occlusion,
+ Timestamp,
+ PipelineStatistics,
+}
+impl From<wgt::QueryType> for SimplifiedQueryType {
+ fn from(q: wgt::QueryType) -> Self {
+ match q {
+ wgt::QueryType::Occlusion => SimplifiedQueryType::Occlusion,
+ wgt::QueryType::Timestamp => SimplifiedQueryType::Timestamp,
+ wgt::QueryType::PipelineStatistics(..) => SimplifiedQueryType::PipelineStatistics,
+ }
+ }
+}
+
+/// Error encountered when dealing with queries
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum QueryError {
+ #[error(transparent)]
+ Device(#[from] DeviceError),
+ #[error(transparent)]
+ Encoder(#[from] CommandEncoderError),
+ #[error("Error encountered while trying to use queries")]
+ Use(#[from] QueryUseError),
+ #[error("Error encountered while trying to resolve a query")]
+ Resolve(#[from] ResolveError),
+ #[error("Buffer {0:?} is invalid or destroyed")]
+ InvalidBuffer(id::BufferId),
+ #[error("QuerySet {0:?} is invalid or destroyed")]
+ InvalidQuerySet(id::QuerySetId),
+}
+
+impl crate::error::PrettyError for QueryError {
+ fn fmt_pretty(&self, fmt: &mut crate::error::ErrorFormatter) {
+ fmt.error(self);
+ match *self {
+ Self::InvalidBuffer(id) => fmt.buffer_label(&id),
+ Self::InvalidQuerySet(id) => fmt.query_set_label(&id),
+
+ _ => {}
+ }
+ }
+}
+
+/// Error encountered while trying to use queries
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum QueryUseError {
+ #[error("Query {query_index} is out of bounds for a query set of size {query_set_size}")]
+ OutOfBounds {
+ query_index: u32,
+ query_set_size: u32,
+ },
+ #[error("Query {query_index} has already been used within the same renderpass. Queries must only be used once per renderpass")]
+ UsedTwiceInsideRenderpass { query_index: u32 },
+ #[error("Query {new_query_index} was started while query {active_query_index} was already active. No more than one statistic or occlusion query may be active at once")]
+ AlreadyStarted {
+ active_query_index: u32,
+ new_query_index: u32,
+ },
+ #[error("Query was stopped while there was no active query")]
+ AlreadyStopped,
+ #[error("A query of type {query_type:?} was started using a query set of type {set_type:?}")]
+ IncompatibleType {
+ set_type: SimplifiedQueryType,
+ query_type: SimplifiedQueryType,
+ },
+}
+
+/// Error encountered while trying to resolve a query.
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum ResolveError {
+ #[error("Queries can only be resolved to buffers that contain the QUERY_RESOLVE usage")]
+ MissingBufferUsage,
+ #[error("Resolve buffer offset has to be aligned to `QUERY_RESOLVE_BUFFER_ALIGNMENT")]
+ BufferOffsetAlignment,
+ #[error("Resolving queries {start_query}..{end_query} would overrun the query set of size {query_set_size}")]
+ QueryOverrun {
+ start_query: u32,
+ end_query: u32,
+ query_set_size: u32,
+ },
+ #[error("Resolving queries {start_query}..{end_query} ({stride} byte queries) will end up overrunning the bounds of the destination buffer of size {buffer_size} using offsets {buffer_start_offset}..{buffer_end_offset}")]
+ BufferOverrun {
+ start_query: u32,
+ end_query: u32,
+ stride: u32,
+ buffer_size: BufferAddress,
+ buffer_start_offset: BufferAddress,
+ buffer_end_offset: BufferAddress,
+ },
+}
+
+impl<A: HalApi> QuerySet<A> {
+ fn validate_query(
+ &self,
+ query_set_id: id::QuerySetId,
+ query_type: SimplifiedQueryType,
+ query_index: u32,
+ reset_state: Option<&mut QueryResetMap<A>>,
+ ) -> Result<&A::QuerySet, QueryUseError> {
+ // We need to defer our resets because we are in a renderpass,
+ // add the usage to the reset map.
+ if let Some(reset) = reset_state {
+ let used = reset.use_query_set(query_set_id, self, query_index);
+ if used {
+ return Err(QueryUseError::UsedTwiceInsideRenderpass { query_index });
+ }
+ }
+
+ let simple_set_type = SimplifiedQueryType::from(self.desc.ty);
+ if simple_set_type != query_type {
+ return Err(QueryUseError::IncompatibleType {
+ query_type,
+ set_type: simple_set_type,
+ });
+ }
+
+ if query_index >= self.desc.count {
+ return Err(QueryUseError::OutOfBounds {
+ query_index,
+ query_set_size: self.desc.count,
+ });
+ }
+
+ Ok(self.raw())
+ }
+
+ pub(super) fn validate_and_write_timestamp(
+ &self,
+ raw_encoder: &mut A::CommandEncoder,
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ reset_state: Option<&mut QueryResetMap<A>>,
+ ) -> Result<(), QueryUseError> {
+ let needs_reset = reset_state.is_none();
+ let query_set = self.validate_query(
+ query_set_id,
+ SimplifiedQueryType::Timestamp,
+ query_index,
+ reset_state,
+ )?;
+
+ unsafe {
+ // If we don't have a reset state tracker which can defer resets, we must reset now.
+ if needs_reset {
+ raw_encoder.reset_queries(self.raw(), query_index..(query_index + 1));
+ }
+ raw_encoder.write_timestamp(query_set, query_index);
+ }
+
+ Ok(())
+ }
+
+ pub(super) fn validate_and_begin_occlusion_query(
+ &self,
+ raw_encoder: &mut A::CommandEncoder,
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ reset_state: Option<&mut QueryResetMap<A>>,
+ active_query: &mut Option<(id::QuerySetId, u32)>,
+ ) -> Result<(), QueryUseError> {
+ let needs_reset = reset_state.is_none();
+ let query_set = self.validate_query(
+ query_set_id,
+ SimplifiedQueryType::Occlusion,
+ query_index,
+ reset_state,
+ )?;
+
+ if let Some((_old_id, old_idx)) = active_query.replace((query_set_id, query_index)) {
+ return Err(QueryUseError::AlreadyStarted {
+ active_query_index: old_idx,
+ new_query_index: query_index,
+ });
+ }
+
+ unsafe {
+ // If we don't have a reset state tracker which can defer resets, we must reset now.
+ if needs_reset {
+ raw_encoder
+ .reset_queries(self.raw.as_ref().unwrap(), query_index..(query_index + 1));
+ }
+ raw_encoder.begin_query(query_set, query_index);
+ }
+
+ Ok(())
+ }
+
+ pub(super) fn validate_and_begin_pipeline_statistics_query(
+ &self,
+ raw_encoder: &mut A::CommandEncoder,
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ reset_state: Option<&mut QueryResetMap<A>>,
+ active_query: &mut Option<(id::QuerySetId, u32)>,
+ ) -> Result<(), QueryUseError> {
+ let needs_reset = reset_state.is_none();
+ let query_set = self.validate_query(
+ query_set_id,
+ SimplifiedQueryType::PipelineStatistics,
+ query_index,
+ reset_state,
+ )?;
+
+ if let Some((_old_id, old_idx)) = active_query.replace((query_set_id, query_index)) {
+ return Err(QueryUseError::AlreadyStarted {
+ active_query_index: old_idx,
+ new_query_index: query_index,
+ });
+ }
+
+ unsafe {
+ // If we don't have a reset state tracker which can defer resets, we must reset now.
+ if needs_reset {
+ raw_encoder.reset_queries(self.raw(), query_index..(query_index + 1));
+ }
+ raw_encoder.begin_query(query_set, query_index);
+ }
+
+ Ok(())
+ }
+}
+
+pub(super) fn end_occlusion_query<A: HalApi>(
+ raw_encoder: &mut A::CommandEncoder,
+ storage: &Storage<QuerySet<A>>,
+ active_query: &mut Option<(id::QuerySetId, u32)>,
+) -> Result<(), QueryUseError> {
+ if let Some((query_set_id, query_index)) = active_query.take() {
+ // We can unwrap here as the validity was validated when the active query was set
+ let query_set = storage.get(query_set_id).unwrap();
+
+ unsafe { raw_encoder.end_query(query_set.raw.as_ref().unwrap(), query_index) };
+
+ Ok(())
+ } else {
+ Err(QueryUseError::AlreadyStopped)
+ }
+}
+
+pub(super) fn end_pipeline_statistics_query<A: HalApi>(
+ raw_encoder: &mut A::CommandEncoder,
+ storage: &Storage<QuerySet<A>>,
+ active_query: &mut Option<(id::QuerySetId, u32)>,
+) -> Result<(), QueryUseError> {
+ if let Some((query_set_id, query_index)) = active_query.take() {
+ // We can unwrap here as the validity was validated when the active query was set
+ let query_set = storage.get(query_set_id).unwrap();
+
+ unsafe { raw_encoder.end_query(query_set.raw(), query_index) };
+
+ Ok(())
+ } else {
+ Err(QueryUseError::AlreadyStopped)
+ }
+}
+
+impl Global {
+ pub fn command_encoder_write_timestamp<A: HalApi>(
+ &self,
+ command_encoder_id: id::CommandEncoderId,
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ ) -> Result<(), QueryError> {
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?;
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::WriteTimestamp {
+ query_set_id,
+ query_index,
+ });
+ }
+
+ let encoder = &mut cmd_buf_data.encoder;
+ let tracker = &mut cmd_buf_data.trackers;
+
+ let raw_encoder = encoder.open()?;
+
+ let query_set_guard = hub.query_sets.read();
+ let query_set = tracker
+ .query_sets
+ .add_single(&*query_set_guard, query_set_id)
+ .ok_or(QueryError::InvalidQuerySet(query_set_id))?;
+
+ query_set.validate_and_write_timestamp(raw_encoder, query_set_id, query_index, None)?;
+
+ Ok(())
+ }
+
+ pub fn command_encoder_resolve_query_set<A: HalApi>(
+ &self,
+ command_encoder_id: id::CommandEncoderId,
+ query_set_id: id::QuerySetId,
+ start_query: u32,
+ query_count: u32,
+ destination: id::BufferId,
+ destination_offset: BufferAddress,
+ ) -> Result<(), QueryError> {
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?;
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::ResolveQuerySet {
+ query_set_id,
+ start_query,
+ query_count,
+ destination,
+ destination_offset,
+ });
+ }
+
+ let encoder = &mut cmd_buf_data.encoder;
+ let tracker = &mut cmd_buf_data.trackers;
+ let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions;
+ let raw_encoder = encoder.open()?;
+
+ if destination_offset % wgt::QUERY_RESOLVE_BUFFER_ALIGNMENT != 0 {
+ return Err(QueryError::Resolve(ResolveError::BufferOffsetAlignment));
+ }
+ let query_set_guard = hub.query_sets.read();
+ let query_set = tracker
+ .query_sets
+ .add_single(&*query_set_guard, query_set_id)
+ .ok_or(QueryError::InvalidQuerySet(query_set_id))?;
+
+ let (dst_buffer, dst_pending) = {
+ let buffer_guard = hub.buffers.read();
+ let dst_buffer = buffer_guard
+ .get(destination)
+ .map_err(|_| QueryError::InvalidBuffer(destination))?;
+ tracker
+ .buffers
+ .set_single(dst_buffer, hal::BufferUses::COPY_DST)
+ .ok_or(QueryError::InvalidBuffer(destination))?
+ };
+
+ let snatch_guard = dst_buffer.device.snatchable_lock.read();
+
+ let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
+
+ if !dst_buffer.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) {
+ return Err(ResolveError::MissingBufferUsage.into());
+ }
+
+ let end_query = start_query + query_count;
+ if end_query > query_set.desc.count {
+ return Err(ResolveError::QueryOverrun {
+ start_query,
+ end_query,
+ query_set_size: query_set.desc.count,
+ }
+ .into());
+ }
+
+ let elements_per_query = match query_set.desc.ty {
+ wgt::QueryType::Occlusion => 1,
+ wgt::QueryType::PipelineStatistics(ps) => ps.bits().count_ones(),
+ wgt::QueryType::Timestamp => 1,
+ };
+ let stride = elements_per_query * wgt::QUERY_SIZE;
+ let bytes_used = (stride * query_count) as BufferAddress;
+
+ let buffer_start_offset = destination_offset;
+ let buffer_end_offset = buffer_start_offset + bytes_used;
+
+ if buffer_end_offset > dst_buffer.size {
+ return Err(ResolveError::BufferOverrun {
+ start_query,
+ end_query,
+ stride,
+ buffer_size: dst_buffer.size,
+ buffer_start_offset,
+ buffer_end_offset,
+ }
+ .into());
+ }
+
+ // TODO(https://github.com/gfx-rs/wgpu/issues/3993): Need to track initialization state.
+ buffer_memory_init_actions.extend(dst_buffer.initialization_status.read().create_action(
+ &dst_buffer,
+ buffer_start_offset..buffer_end_offset,
+ MemoryInitKind::ImplicitlyInitialized,
+ ));
+
+ let raw_dst_buffer = dst_buffer
+ .raw(&snatch_guard)
+ .ok_or(QueryError::InvalidBuffer(destination))?;
+
+ unsafe {
+ raw_encoder.transition_buffers(dst_barrier.into_iter());
+ raw_encoder.copy_query_results(
+ query_set.raw(),
+ start_query..end_query,
+ raw_dst_buffer,
+ destination_offset,
+ wgt::BufferSize::new_unchecked(stride as u64),
+ );
+ }
+
+ Ok(())
+ }
+}
diff --git a/third_party/rust/wgpu-core/src/command/render.rs b/third_party/rust/wgpu-core/src/command/render.rs
new file mode 100644
index 0000000000..d3de3e26e1
--- /dev/null
+++ b/third_party/rust/wgpu-core/src/command/render.rs
@@ -0,0 +1,2866 @@
+use crate::resource::Resource;
+use crate::snatch::SnatchGuard;
+use crate::{
+ api_log,
+ binding_model::BindError,
+ command::{
+ self,
+ bind::Binder,
+ end_occlusion_query, end_pipeline_statistics_query,
+ memory_init::{fixup_discarded_surfaces, SurfacesInDiscardState},
+ BasePass, BasePassRef, BindGroupStateChange, CommandBuffer, CommandEncoderError,
+ CommandEncoderStatus, DrawError, ExecutionError, MapPassErr, PassErrorScope, QueryUseError,
+ RenderCommand, RenderCommandError, StateChange,
+ },
+ device::{
+ AttachmentData, Device, DeviceError, MissingDownlevelFlags, MissingFeatures,
+ RenderPassCompatibilityCheckType, RenderPassCompatibilityError, RenderPassContext,
+ },
+ error::{ErrorFormatter, PrettyError},
+ global::Global,
+ hal_api::HalApi,
+ hal_label, id,
+ init_tracker::{MemoryInitKind, TextureInitRange, TextureInitTrackerAction},
+ pipeline::{self, PipelineFlags},
+ resource::{Buffer, QuerySet, Texture, TextureView, TextureViewNotRenderableReason},
+ storage::Storage,
+ track::{TextureSelector, Tracker, UsageConflict, UsageScope},
+ validation::{
+ check_buffer_usage, check_texture_usage, MissingBufferUsageError, MissingTextureUsageError,
+ },
+ Label,
+};
+
+use arrayvec::ArrayVec;
+use hal::CommandEncoder as _;
+use thiserror::Error;
+use wgt::{
+ BufferAddress, BufferSize, BufferUsages, Color, IndexFormat, TextureUsages,
+ TextureViewDimension, VertexStepMode,
+};
+
+#[cfg(feature = "serde")]
+use serde::Deserialize;
+#[cfg(feature = "serde")]
+use serde::Serialize;
+
+use std::sync::Arc;
+use std::{borrow::Cow, fmt, iter, marker::PhantomData, mem, num::NonZeroU32, ops::Range, str};
+
+use super::{
+ memory_init::TextureSurfaceDiscard, CommandBufferTextureMemoryActions, CommandEncoder,
+ QueryResetMap,
+};
+
+/// Operation to perform to the output attachment at the start of a renderpass.
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[cfg_attr(feature = "serde", serde(rename_all = "kebab-case"))]
+pub enum LoadOp {
+ /// Clear the output attachment with the clear color. Clearing is faster than loading.
+ Clear = 0,
+ /// Do not clear output attachment.
+ Load = 1,
+}
+
+/// Operation to perform to the output attachment at the end of a renderpass.
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[cfg_attr(feature = "serde", serde(rename_all = "kebab-case"))]
+pub enum StoreOp {
+ /// Discards the content of the render target.
+ ///
+ /// If you don't care about the contents of the target, this can be faster.
+ Discard = 0,
+ /// Store the result of the renderpass.
+ Store = 1,
+}
+
+/// Describes an individual channel within a render pass, such as color, depth, or stencil.
+#[repr(C)]
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct PassChannel<V> {
+ /// Operation to perform to the output attachment at the start of a
+ /// renderpass.
+ ///
+ /// This must be clear if it is the first renderpass rendering to a swap
+ /// chain image.
+ pub load_op: LoadOp,
+ /// Operation to perform to the output attachment at the end of a renderpass.
+ pub store_op: StoreOp,
+ /// If load_op is [`LoadOp::Clear`], the attachment will be cleared to this
+ /// color.
+ pub clear_value: V,
+ /// If true, the relevant channel is not changed by a renderpass, and the
+ /// corresponding attachment can be used inside the pass by other read-only
+ /// usages.
+ pub read_only: bool,
+}
+
+impl<V> PassChannel<V> {
+ fn hal_ops(&self) -> hal::AttachmentOps {
+ let mut ops = hal::AttachmentOps::empty();
+ match self.load_op {
+ LoadOp::Load => ops |= hal::AttachmentOps::LOAD,
+ LoadOp::Clear => (),
+ };
+ match self.store_op {
+ StoreOp::Store => ops |= hal::AttachmentOps::STORE,
+ StoreOp::Discard => (),
+ };
+ ops
+ }
+}
+
+/// Describes a color attachment to a render pass.
+#[repr(C)]
+#[derive(Clone, Debug, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct RenderPassColorAttachment {
+ /// The view to use as an attachment.
+ pub view: id::TextureViewId,
+ /// The view that will receive the resolved output if multisampling is used.
+ pub resolve_target: Option<id::TextureViewId>,
+ /// What operations will be performed on this color attachment.
+ pub channel: PassChannel<Color>,
+}
+
+/// Describes a depth/stencil attachment to a render pass.
+#[repr(C)]
+#[derive(Clone, Debug, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct RenderPassDepthStencilAttachment {
+ /// The view to use as an attachment.
+ pub view: id::TextureViewId,
+ /// What operations will be performed on the depth part of the attachment.
+ pub depth: PassChannel<f32>,
+ /// What operations will be performed on the stencil part of the attachment.
+ pub stencil: PassChannel<u32>,
+}
+
+impl RenderPassDepthStencilAttachment {
+ /// Validate the given aspects' read-only flags against their load
+ /// and store ops.
+ ///
+ /// When an aspect is read-only, its load and store ops must be
+ /// `LoadOp::Load` and `StoreOp::Store`.
+ ///
+ /// On success, return a pair `(depth, stencil)` indicating
+ /// whether the depth and stencil passes are read-only.
+ fn depth_stencil_read_only(
+ &self,
+ aspects: hal::FormatAspects,
+ ) -> Result<(bool, bool), RenderPassErrorInner> {
+ let mut depth_read_only = true;
+ let mut stencil_read_only = true;
+
+ if aspects.contains(hal::FormatAspects::DEPTH) {
+ if self.depth.read_only
+ && (self.depth.load_op, self.depth.store_op) != (LoadOp::Load, StoreOp::Store)
+ {
+ return Err(RenderPassErrorInner::InvalidDepthOps);
+ }
+ depth_read_only = self.depth.read_only;
+ }
+
+ if aspects.contains(hal::FormatAspects::STENCIL) {
+ if self.stencil.read_only
+ && (self.stencil.load_op, self.stencil.store_op) != (LoadOp::Load, StoreOp::Store)
+ {
+ return Err(RenderPassErrorInner::InvalidStencilOps);
+ }
+ stencil_read_only = self.stencil.read_only;
+ }
+
+ Ok((depth_read_only, stencil_read_only))
+ }
+}
+
+/// Location to write a timestamp to (beginning or end of the pass).
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[cfg_attr(feature = "serde", serde(rename_all = "kebab-case"))]
+pub enum RenderPassTimestampLocation {
+ Beginning = 0,
+ End = 1,
+}
+
+/// Describes the writing of timestamp values in a render pass.
+#[repr(C)]
+#[derive(Clone, Debug, PartialEq, Eq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct RenderPassTimestampWrites {
+ /// The query set to write the timestamp to.
+ pub query_set: id::QuerySetId,
+ /// The index of the query set at which a start timestamp of this pass is written, if any.
+ pub beginning_of_pass_write_index: Option<u32>,
+ /// The index of the query set at which an end timestamp of this pass is written, if any.
+ pub end_of_pass_write_index: Option<u32>,
+}
+
+/// Describes the attachments of a render pass.
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct RenderPassDescriptor<'a> {
+ pub label: Label<'a>,
+ /// The color attachments of the render pass.
+ pub color_attachments: Cow<'a, [Option<RenderPassColorAttachment>]>,
+ /// The depth and stencil attachment of the render pass, if any.
+ pub depth_stencil_attachment: Option<&'a RenderPassDepthStencilAttachment>,
+ /// Defines where and when timestamp values will be written for this pass.
+ pub timestamp_writes: Option<&'a RenderPassTimestampWrites>,
+ /// Defines where the occlusion query results will be stored for this pass.
+ pub occlusion_query_set: Option<id::QuerySetId>,
+}
+
+#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
+pub struct RenderPass {
+ base: BasePass<RenderCommand>,
+ parent_id: id::CommandEncoderId,
+ color_targets: ArrayVec<Option<RenderPassColorAttachment>, { hal::MAX_COLOR_ATTACHMENTS }>,
+ depth_stencil_target: Option<RenderPassDepthStencilAttachment>,
+ timestamp_writes: Option<RenderPassTimestampWrites>,
+ occlusion_query_set_id: Option<id::QuerySetId>,
+
+ // Resource binding dedupe state.
+ #[cfg_attr(feature = "serde", serde(skip))]
+ current_bind_groups: BindGroupStateChange,
+ #[cfg_attr(feature = "serde", serde(skip))]
+ current_pipeline: StateChange<id::RenderPipelineId>,
+}
+
+impl RenderPass {
+ pub fn new(parent_id: id::CommandEncoderId, desc: &RenderPassDescriptor) -> Self {
+ Self {
+ base: BasePass::new(&desc.label),
+ parent_id,
+ color_targets: desc.color_attachments.iter().cloned().collect(),
+ depth_stencil_target: desc.depth_stencil_attachment.cloned(),
+ timestamp_writes: desc.timestamp_writes.cloned(),
+ occlusion_query_set_id: desc.occlusion_query_set,
+
+ current_bind_groups: BindGroupStateChange::new(),
+ current_pipeline: StateChange::new(),
+ }
+ }
+
+ pub fn parent_id(&self) -> id::CommandEncoderId {
+ self.parent_id
+ }
+
+ #[cfg(feature = "trace")]
+ pub fn into_command(self) -> crate::device::trace::Command {
+ crate::device::trace::Command::RunRenderPass {
+ base: self.base,
+ target_colors: self.color_targets.into_iter().collect(),
+ target_depth_stencil: self.depth_stencil_target,
+ timestamp_writes: self.timestamp_writes,
+ occlusion_query_set_id: self.occlusion_query_set_id,
+ }
+ }
+
+ pub fn set_index_buffer(
+ &mut self,
+ buffer_id: id::BufferId,
+ index_format: IndexFormat,
+ offset: BufferAddress,
+ size: Option<BufferSize>,
+ ) {
+ self.base.commands.push(RenderCommand::SetIndexBuffer {
+ buffer_id,
+ index_format,
+ offset,
+ size,
+ });
+ }
+}
+
+impl fmt::Debug for RenderPass {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("RenderPass")
+ .field("encoder_id", &self.parent_id)
+ .field("color_targets", &self.color_targets)
+ .field("depth_stencil_target", &self.depth_stencil_target)
+ .field("command count", &self.base.commands.len())
+ .field("dynamic offset count", &self.base.dynamic_offsets.len())
+ .field(
+ "push constant u32 count",
+ &self.base.push_constant_data.len(),
+ )
+ .finish()
+ }
+}
+
+#[derive(Debug, PartialEq)]
+enum OptionalState {
+ Unused,
+ Required,
+ Set,
+}
+
+impl OptionalState {
+ fn require(&mut self, require: bool) {
+ if require && *self == Self::Unused {
+ *self = Self::Required;
+ }
+ }
+}
+
+#[derive(Debug, Default)]
+struct IndexState {
+ bound_buffer_view: Option<(id::BufferId, Range<BufferAddress>)>,
+ format: Option<IndexFormat>,
+ pipeline_format: Option<IndexFormat>,
+ limit: u64,
+}
+
+impl IndexState {
+ fn update_limit(&mut self) {
+ self.limit = match self.bound_buffer_view {
+ Some((_, ref range)) => {
+ let format = self
+ .format
+ .expect("IndexState::update_limit must be called after a index buffer is set");
+ let shift = match format {
+ IndexFormat::Uint16 => 1,
+ IndexFormat::Uint32 => 2,
+ };
+
+ (range.end - range.start) >> shift
+ }
+ None => 0,
+ }
+ }
+
+ fn reset(&mut self) {
+ self.bound_buffer_view = None;
+ self.limit = 0;
+ }
+}
+
+#[derive(Clone, Copy, Debug)]
+struct VertexBufferState {
+ total_size: BufferAddress,
+ step: pipeline::VertexStep,
+ bound: bool,
+}
+
+impl VertexBufferState {
+ const EMPTY: Self = Self {
+ total_size: 0,
+ step: pipeline::VertexStep {
+ stride: 0,
+ last_stride: 0,
+ mode: VertexStepMode::Vertex,
+ },
+ bound: false,
+ };
+}
+
+#[derive(Debug, Default)]
+struct VertexState {
+ inputs: ArrayVec<VertexBufferState, { hal::MAX_VERTEX_BUFFERS }>,
+ /// Length of the shortest vertex rate vertex buffer
+ vertex_limit: u64,
+ /// Buffer slot which the shortest vertex rate vertex buffer is bound to
+ vertex_limit_slot: u32,
+ /// Length of the shortest instance rate vertex buffer
+ instance_limit: u64,
+ /// Buffer slot which the shortest instance rate vertex buffer is bound to
+ instance_limit_slot: u32,
+ /// Total amount of buffers required by the pipeline.
+ buffers_required: u32,
+}
+
+impl VertexState {
+ fn update_limits(&mut self) {
+ // Implements the validation from https://gpuweb.github.io/gpuweb/#dom-gpurendercommandsmixin-draw
+ // Except that the formula is shuffled to extract the number of vertices in order
+ // to carry the bulk of the computation when changing states instead of when producing
+ // draws. Draw calls tend to happen at a higher frequency. Here we determine vertex
+ // limits that can be cheaply checked for each draw call.
+ self.vertex_limit = u32::MAX as u64;
+ self.instance_limit = u32::MAX as u64;
+ for (idx, vbs) in self.inputs.iter().enumerate() {
+ if !vbs.bound {
+ continue;
+ }
+
+ let limit = if vbs.total_size < vbs.step.last_stride {
+ // The buffer cannot fit the last vertex.
+ 0
+ } else {
+ if vbs.step.stride == 0 {
+ // We already checked that the last stride fits, the same
+ // vertex will be repeated so this slot can accommodate any number of
+ // vertices.
+ continue;
+ }
+
+ // The general case.
+ (vbs.total_size - vbs.step.last_stride) / vbs.step.stride + 1
+ };
+
+ match vbs.step.mode {
+ VertexStepMode::Vertex => {
+ if limit < self.vertex_limit {
+ self.vertex_limit = limit;
+ self.vertex_limit_slot = idx as _;
+ }
+ }
+ VertexStepMode::Instance => {
+ if limit < self.instance_limit {
+ self.instance_limit = limit;
+ self.instance_limit_slot = idx as _;
+ }
+ }
+ }
+ }
+ }
+
+ fn reset(&mut self) {
+ self.inputs.clear();
+ self.vertex_limit = 0;
+ self.instance_limit = 0;
+ }
+}
+
+#[derive(Debug)]
+struct State<A: HalApi> {
+ pipeline_flags: PipelineFlags,
+ binder: Binder<A>,
+ blend_constant: OptionalState,
+ stencil_reference: u32,
+ pipeline: Option<id::RenderPipelineId>,
+ index: IndexState,
+ vertex: VertexState,
+ debug_scope_depth: u32,
+}
+
+impl<A: HalApi> State<A> {
+ fn is_ready(&self, indexed: bool) -> Result<(), DrawError> {
+ // Determine how many vertex buffers have already been bound
+ let vertex_buffer_count = self.vertex.inputs.iter().take_while(|v| v.bound).count() as u32;
+ // Compare with the needed quantity
+ if vertex_buffer_count < self.vertex.buffers_required {
+ return Err(DrawError::MissingVertexBuffer {
+ index: vertex_buffer_count,
+ });
+ }
+
+ let bind_mask = self.binder.invalid_mask();
+ if bind_mask != 0 {
+ //let (expected, provided) = self.binder.entries[index as usize].info();
+ return Err(DrawError::IncompatibleBindGroup {
+ index: bind_mask.trailing_zeros(),
+ diff: self.binder.bgl_diff(),
+ });
+ }
+ if self.pipeline.is_none() {
+ return Err(DrawError::MissingPipeline);
+ }
+ if self.blend_constant == OptionalState::Required {
+ return Err(DrawError::MissingBlendConstant);
+ }
+
+ if indexed {
+ // Pipeline expects an index buffer
+ if let Some(pipeline_index_format) = self.index.pipeline_format {
+ // We have a buffer bound
+ let buffer_index_format = self.index.format.ok_or(DrawError::MissingIndexBuffer)?;
+
+ // The buffers are different formats
+ if pipeline_index_format != buffer_index_format {
+ return Err(DrawError::UnmatchedIndexFormats {
+ pipeline: pipeline_index_format,
+ buffer: buffer_index_format,
+ });
+ }
+ }
+ }
+
+ self.binder.check_late_buffer_bindings()?;
+
+ Ok(())
+ }
+
+ /// Reset the `RenderBundle`-related states.
+ fn reset_bundle(&mut self) {
+ self.binder.reset();
+ self.pipeline = None;
+ self.index.reset();
+ self.vertex.reset();
+ }
+}
+
+/// Describes an attachment location in words.
+///
+/// Can be used as "the {loc} has..." or "{loc} has..."
+#[derive(Debug, Copy, Clone)]
+pub enum AttachmentErrorLocation {
+ Color { index: usize, resolve: bool },
+ Depth,
+}
+
+impl fmt::Display for AttachmentErrorLocation {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match *self {
+ AttachmentErrorLocation::Color {
+ index,
+ resolve: false,
+ } => write!(f, "color attachment at index {index}'s texture view"),
+ AttachmentErrorLocation::Color {
+ index,
+ resolve: true,
+ } => write!(
+ f,
+ "color attachment at index {index}'s resolve texture view"
+ ),
+ AttachmentErrorLocation::Depth => write!(f, "depth attachment's texture view"),
+ }
+ }
+}
+
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum ColorAttachmentError {
+ #[error("Attachment format {0:?} is not a color format")]
+ InvalidFormat(wgt::TextureFormat),
+ #[error("The number of color attachments {given} exceeds the limit {limit}")]
+ TooMany { given: usize, limit: usize },
+}
+
+/// Error encountered when performing a render pass.
+#[derive(Clone, Debug, Error)]
+pub enum RenderPassErrorInner {
+ #[error(transparent)]
+ Device(DeviceError),
+ #[error(transparent)]
+ ColorAttachment(#[from] ColorAttachmentError),
+ #[error(transparent)]
+ Encoder(#[from] CommandEncoderError),
+ #[error("Attachment texture view {0:?} is invalid")]
+ InvalidAttachment(id::TextureViewId),
+ #[error("Attachment texture view {0:?} is invalid")]
+ InvalidResolveTarget(id::TextureViewId),
+ #[error("The format of the depth-stencil attachment ({0:?}) is not a depth-stencil format")]
+ InvalidDepthStencilAttachmentFormat(wgt::TextureFormat),
+ #[error("The format of the {location} ({format:?}) is not resolvable")]
+ UnsupportedResolveTargetFormat {
+ location: AttachmentErrorLocation,
+ format: wgt::TextureFormat,
+ },
+ #[error("No color attachments or depth attachments were provided, at least one attachment of any kind must be provided")]
+ MissingAttachments,
+ #[error("The {location} is not renderable:")]
+ TextureViewIsNotRenderable {
+ location: AttachmentErrorLocation,
+ #[source]
+ reason: TextureViewNotRenderableReason,
+ },
+ #[error("Attachments have differing sizes: the {expected_location} has extent {expected_extent:?} but is followed by the {actual_location} which has {actual_extent:?}")]
+ AttachmentsDimensionMismatch {
+ expected_location: AttachmentErrorLocation,
+ expected_extent: wgt::Extent3d,
+ actual_location: AttachmentErrorLocation,
+ actual_extent: wgt::Extent3d,
+ },
+ #[error("Attachments have differing sample counts: the {expected_location} has count {expected_samples:?} but is followed by the {actual_location} which has count {actual_samples:?}")]
+ AttachmentSampleCountMismatch {
+ expected_location: AttachmentErrorLocation,
+ expected_samples: u32,
+ actual_location: AttachmentErrorLocation,
+ actual_samples: u32,
+ },
+ #[error("The resolve source, {location}, must be multi-sampled (has {src} samples) while the resolve destination must not be multisampled (has {dst} samples)")]
+ InvalidResolveSampleCounts {
+ location: AttachmentErrorLocation,
+ src: u32,
+ dst: u32,
+ },
+ #[error(
+ "Resource source, {location}, format ({src:?}) must match the resolve destination format ({dst:?})"
+ )]
+ MismatchedResolveTextureFormat {
+ location: AttachmentErrorLocation,
+ src: wgt::TextureFormat,
+ dst: wgt::TextureFormat,
+ },
+ #[error("Surface texture is dropped before the render pass is finished")]
+ SurfaceTextureDropped,
+ #[error("Not enough memory left for render pass")]
+ OutOfMemory,
+ #[error("The bind group at index {0:?} is invalid")]
+ InvalidBindGroup(usize),
+ #[error("Unable to clear non-present/read-only depth")]
+ InvalidDepthOps,
+ #[error("Unable to clear non-present/read-only stencil")]
+ InvalidStencilOps,
+ #[error("Setting `values_offset` to be `None` is only for internal use in render bundles")]
+ InvalidValuesOffset,
+ #[error(transparent)]
+ MissingFeatures(#[from] MissingFeatures),
+ #[error(transparent)]
+ MissingDownlevelFlags(#[from] MissingDownlevelFlags),
+ #[error("Indirect draw uses bytes {offset}..{end_offset} {} which overruns indirect buffer of size {buffer_size}",
+ count.map_or_else(String::new, |v| format!("(using count {v})")))]
+ IndirectBufferOverrun {
+ count: Option<NonZeroU32>,
+ offset: u64,
+ end_offset: u64,
+ buffer_size: u64,
+ },
+ #[error("Indirect draw uses bytes {begin_count_offset}..{end_count_offset} which overruns indirect buffer of size {count_buffer_size}")]
+ IndirectCountBufferOverrun {
+ begin_count_offset: u64,
+ end_count_offset: u64,
+ count_buffer_size: u64,
+ },
+ #[error("Cannot pop debug group, because number of pushed debug groups is zero")]
+ InvalidPopDebugGroup,
+ #[error(transparent)]
+ ResourceUsageConflict(#[from] UsageConflict),
+ #[error("Render bundle has incompatible targets, {0}")]
+ IncompatibleBundleTargets(#[from] RenderPassCompatibilityError),
+ #[error(
+ "Render bundle has incompatible read-only flags: \
+ bundle has flags depth = {bundle_depth} and stencil = {bundle_stencil}, \
+ while the pass has flags depth = {pass_depth} and stencil = {pass_stencil}. \
+ Read-only renderpasses are only compatible with read-only bundles for that aspect."
+ )]
+ IncompatibleBundleReadOnlyDepthStencil {
+ pass_depth: bool,
+ pass_stencil: bool,
+ bundle_depth: bool,
+ bundle_stencil: bool,
+ },
+ #[error(transparent)]
+ RenderCommand(#[from] RenderCommandError),
+ #[error(transparent)]
+ Draw(#[from] DrawError),
+ #[error(transparent)]
+ Bind(#[from] BindError),
+ #[error(transparent)]
+ QueryUse(#[from] QueryUseError),
+ #[error("Multiview layer count must match")]
+ MultiViewMismatch,
+ #[error(
+ "Multiview pass texture views with more than one array layer must have D2Array dimension"
+ )]
+ MultiViewDimensionMismatch,
+ #[error("QuerySet {0:?} is invalid")]
+ InvalidQuerySet(id::QuerySetId),
+ #[error("missing occlusion query set")]
+ MissingOcclusionQuerySet,
+}
+
+impl PrettyError for RenderPassErrorInner {
+ fn fmt_pretty(&self, fmt: &mut ErrorFormatter) {
+ fmt.error(self);
+ if let Self::InvalidAttachment(id) = *self {
+ fmt.texture_view_label_with_key(&id, "attachment");
+ };
+ if let Self::Draw(DrawError::IncompatibleBindGroup { diff, .. }) = self {
+ for d in diff {
+ fmt.note(&d);
+ }
+ };
+ }
+}
+
+impl From<MissingBufferUsageError> for RenderPassErrorInner {
+ fn from(error: MissingBufferUsageError) -> Self {
+ Self::RenderCommand(error.into())
+ }
+}
+
+impl From<MissingTextureUsageError> for RenderPassErrorInner {
+ fn from(error: MissingTextureUsageError) -> Self {
+ Self::RenderCommand(error.into())
+ }
+}
+
+impl From<DeviceError> for RenderPassErrorInner {
+ fn from(error: DeviceError) -> Self {
+ Self::Device(error)
+ }
+}
+
+/// Error encountered when performing a render pass.
+#[derive(Clone, Debug, Error)]
+#[error("{scope}")]
+pub struct RenderPassError {
+ pub scope: PassErrorScope,
+ #[source]
+ inner: RenderPassErrorInner,
+}
+impl PrettyError for RenderPassError {
+ fn fmt_pretty(&self, fmt: &mut ErrorFormatter) {
+ // This error is wrapper for the inner error,
+ // but the scope has useful labels
+ fmt.error(self);
+ self.scope.fmt_pretty(fmt);
+ }
+}
+
+impl<T, E> MapPassErr<T, RenderPassError> for Result<T, E>
+where
+ E: Into<RenderPassErrorInner>,
+{
+ fn map_pass_err(self, scope: PassErrorScope) -> Result<T, RenderPassError> {
+ self.map_err(|inner| RenderPassError {
+ scope,
+ inner: inner.into(),
+ })
+ }
+}
+
+struct RenderAttachment<'a, A: HalApi> {
+ texture: Arc<Texture<A>>,
+ selector: &'a TextureSelector,
+ usage: hal::TextureUses,
+}
+
+impl<A: HalApi> TextureView<A> {
+ fn to_render_attachment(&self, usage: hal::TextureUses) -> RenderAttachment<A> {
+ RenderAttachment {
+ texture: self.parent.clone(),
+ selector: &self.selector,
+ usage,
+ }
+ }
+}
+
+const MAX_TOTAL_ATTACHMENTS: usize = hal::MAX_COLOR_ATTACHMENTS + hal::MAX_COLOR_ATTACHMENTS + 1;
+type AttachmentDataVec<T> = ArrayVec<T, MAX_TOTAL_ATTACHMENTS>;
+
+struct RenderPassInfo<'a, A: HalApi> {
+ context: RenderPassContext,
+ usage_scope: UsageScope<A>,
+ /// All render attachments, including depth/stencil
+ render_attachments: AttachmentDataVec<RenderAttachment<'a, A>>,
+ is_depth_read_only: bool,
+ is_stencil_read_only: bool,
+ extent: wgt::Extent3d,
+ _phantom: PhantomData<A>,
+
+ pending_discard_init_fixups: SurfacesInDiscardState<A>,
+ divergent_discarded_depth_stencil_aspect: Option<(wgt::TextureAspect, &'a TextureView<A>)>,
+ multiview: Option<NonZeroU32>,
+}
+
+impl<'a, A: HalApi> RenderPassInfo<'a, A> {
+ fn add_pass_texture_init_actions<V>(
+ channel: &PassChannel<V>,
+ texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>,
+ view: &TextureView<A>,
+ pending_discard_init_fixups: &mut SurfacesInDiscardState<A>,
+ ) {
+ if channel.load_op == LoadOp::Load {
+ pending_discard_init_fixups.extend(texture_memory_actions.register_init_action(
+ &TextureInitTrackerAction {
+ texture: view.parent.clone(),
+ range: TextureInitRange::from(view.selector.clone()),
+ // Note that this is needed even if the target is discarded,
+ kind: MemoryInitKind::NeedsInitializedMemory,
+ },
+ ));
+ } else if channel.store_op == StoreOp::Store {
+ // Clear + Store
+ texture_memory_actions.register_implicit_init(
+ &view.parent,
+ TextureInitRange::from(view.selector.clone()),
+ );
+ }
+ if channel.store_op == StoreOp::Discard {
+ // the discard happens at the *end* of a pass, but recording the
+ // discard right away be alright since the texture can't be used
+ // during the pass anyways
+ texture_memory_actions.discard(TextureSurfaceDiscard {
+ texture: view.parent.clone(),
+ mip_level: view.selector.mips.start,
+ layer: view.selector.layers.start,
+ });
+ }
+ }
+
+ fn start(
+ device: &Device<A>,
+ label: Option<&str>,
+ color_attachments: &[Option<RenderPassColorAttachment>],
+ depth_stencil_attachment: Option<&RenderPassDepthStencilAttachment>,
+ timestamp_writes: Option<&RenderPassTimestampWrites>,
+ occlusion_query_set: Option<id::QuerySetId>,
+ encoder: &mut CommandEncoder<A>,
+ trackers: &mut Tracker<A>,
+ texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>,
+ pending_query_resets: &mut QueryResetMap<A>,
+ view_guard: &'a Storage<TextureView<A>>,
+ buffer_guard: &'a Storage<Buffer<A>>,
+ texture_guard: &'a Storage<Texture<A>>,
+ query_set_guard: &'a Storage<QuerySet<A>>,
+ snatch_guard: &SnatchGuard<'a>,
+ ) -> Result<Self, RenderPassErrorInner> {
+ profiling::scope!("RenderPassInfo::start");
+
+ // We default to false intentionally, even if depth-stencil isn't used at all.
+ // This allows us to use the primary raw pipeline in `RenderPipeline`,
+ // instead of the special read-only one, which would be `None`.
+ let mut is_depth_read_only = false;
+ let mut is_stencil_read_only = false;
+
+ let mut render_attachments = AttachmentDataVec::<RenderAttachment<A>>::new();
+ let mut discarded_surfaces = AttachmentDataVec::new();
+ let mut pending_discard_init_fixups = SurfacesInDiscardState::new();
+ let mut divergent_discarded_depth_stencil_aspect = None;
+
+ let mut attachment_location = AttachmentErrorLocation::Color {
+ index: usize::MAX,
+ resolve: false,
+ };
+ let mut extent = None;
+ let mut sample_count = 0;
+
+ let mut detected_multiview: Option<Option<NonZeroU32>> = None;
+
+ let mut check_multiview = |view: &TextureView<A>| {
+ // Get the multiview configuration for this texture view
+ let layers = view.selector.layers.end - view.selector.layers.start;
+ let this_multiview = if layers >= 2 {
+ // Trivially proven by the if above
+ Some(unsafe { NonZeroU32::new_unchecked(layers) })
+ } else {
+ None
+ };
+
+ // Make sure that if this view is a multiview, it is set to be an array
+ if this_multiview.is_some() && view.desc.dimension != TextureViewDimension::D2Array {
+ return Err(RenderPassErrorInner::MultiViewDimensionMismatch);
+ }
+
+ // Validate matching first, or store the first one
+ if let Some(multiview) = detected_multiview {
+ if multiview != this_multiview {
+ return Err(RenderPassErrorInner::MultiViewMismatch);
+ }
+ } else {
+ // Multiview is only supported if the feature is enabled
+ if this_multiview.is_some() {
+ device.require_features(wgt::Features::MULTIVIEW)?;
+ }
+
+ detected_multiview = Some(this_multiview);
+ }
+
+ Ok(())
+ };
+ let mut add_view = |view: &TextureView<A>, location| {
+ let render_extent = view.render_extent.map_err(|reason| {
+ RenderPassErrorInner::TextureViewIsNotRenderable { location, reason }
+ })?;
+ if let Some(ex) = extent {
+ if ex != render_extent {
+ return Err(RenderPassErrorInner::AttachmentsDimensionMismatch {
+ expected_location: attachment_location,
+ expected_extent: ex,
+ actual_location: location,
+ actual_extent: render_extent,
+ });
+ }
+ } else {
+ extent = Some(render_extent);
+ }
+ if sample_count == 0 {
+ sample_count = view.samples;
+ } else if sample_count != view.samples {
+ return Err(RenderPassErrorInner::AttachmentSampleCountMismatch {
+ expected_location: attachment_location,
+ expected_samples: sample_count,
+ actual_location: location,
+ actual_samples: view.samples,
+ });
+ }
+ attachment_location = location;
+ Ok(())
+ };
+
+ let mut colors =
+ ArrayVec::<Option<hal::ColorAttachment<A>>, { hal::MAX_COLOR_ATTACHMENTS }>::new();
+ let mut depth_stencil = None;
+
+ if let Some(at) = depth_stencil_attachment {
+ let view: &TextureView<A> = trackers
+ .views
+ .add_single(view_guard, at.view)
+ .ok_or(RenderPassErrorInner::InvalidAttachment(at.view))?;
+ check_multiview(view)?;
+ add_view(view, AttachmentErrorLocation::Depth)?;
+
+ let ds_aspects = view.desc.aspects();
+ if ds_aspects.contains(hal::FormatAspects::COLOR) {
+ return Err(RenderPassErrorInner::InvalidDepthStencilAttachmentFormat(
+ view.desc.format,
+ ));
+ }
+
+ if !ds_aspects.contains(hal::FormatAspects::STENCIL)
+ || (at.stencil.load_op == at.depth.load_op
+ && at.stencil.store_op == at.depth.store_op)
+ {
+ Self::add_pass_texture_init_actions(
+ &at.depth,
+ texture_memory_actions,
+ view,
+ &mut pending_discard_init_fixups,
+ );
+ } else if !ds_aspects.contains(hal::FormatAspects::DEPTH) {
+ Self::add_pass_texture_init_actions(
+ &at.stencil,
+ texture_memory_actions,
+ view,
+ &mut pending_discard_init_fixups,
+ );
+ } else {
+ // This is the only place (anywhere in wgpu) where Stencil &
+ // Depth init state can diverge.
+ //
+ // To safe us the overhead of tracking init state of texture
+ // aspects everywhere, we're going to cheat a little bit in
+ // order to keep the init state of both Stencil and Depth
+ // aspects in sync. The expectation is that we hit this path
+ // extremely rarely!
+ //
+ // Diverging LoadOp, i.e. Load + Clear:
+ //
+ // Record MemoryInitKind::NeedsInitializedMemory for the entire
+ // surface, a bit wasteful on unit but no negative effect!
+ //
+ // Rationale: If the loaded channel is uninitialized it needs
+ // clearing, the cleared channel doesn't care. (If everything is
+ // already initialized nothing special happens)
+ //
+ // (possible minor optimization: Clear caused by
+ // NeedsInitializedMemory should know that it doesn't need to
+ // clear the aspect that was set to C)
+ let need_init_beforehand =
+ at.depth.load_op == LoadOp::Load || at.stencil.load_op == LoadOp::Load;
+ if need_init_beforehand {
+ pending_discard_init_fixups.extend(
+ texture_memory_actions.register_init_action(&TextureInitTrackerAction {
+ texture: view.parent.clone(),
+ range: TextureInitRange::from(view.selector.clone()),
+ kind: MemoryInitKind::NeedsInitializedMemory,
+ }),
+ );
+ }
+
+ // Diverging Store, i.e. Discard + Store:
+ //
+ // Immediately zero out channel that is set to discard after
+ // we're done with the render pass. This allows us to set the
+ // entire surface to MemoryInitKind::ImplicitlyInitialized (if
+ // it isn't already set to NeedsInitializedMemory).
+ //
+ // (possible optimization: Delay and potentially drop this zeroing)
+ if at.depth.store_op != at.stencil.store_op {
+ if !need_init_beforehand {
+ texture_memory_actions.register_implicit_init(
+ &view.parent,
+ TextureInitRange::from(view.selector.clone()),
+ );
+ }
+ divergent_discarded_depth_stencil_aspect = Some((
+ if at.depth.store_op == StoreOp::Discard {
+ wgt::TextureAspect::DepthOnly
+ } else {
+ wgt::TextureAspect::StencilOnly
+ },
+ view,
+ ));
+ } else if at.depth.store_op == StoreOp::Discard {
+ // Both are discarded using the regular path.
+ discarded_surfaces.push(TextureSurfaceDiscard {
+ texture: view.parent.clone(),
+ mip_level: view.selector.mips.start,
+ layer: view.selector.layers.start,
+ });
+ }
+ }
+
+ (is_depth_read_only, is_stencil_read_only) = at.depth_stencil_read_only(ds_aspects)?;
+
+ let usage = if is_depth_read_only
+ && is_stencil_read_only
+ && device
+ .downlevel
+ .flags
+ .contains(wgt::DownlevelFlags::READ_ONLY_DEPTH_STENCIL)
+ {
+ hal::TextureUses::DEPTH_STENCIL_READ | hal::TextureUses::RESOURCE
+ } else {
+ hal::TextureUses::DEPTH_STENCIL_WRITE
+ };
+ render_attachments.push(view.to_render_attachment(usage));
+
+ depth_stencil = Some(hal::DepthStencilAttachment {
+ target: hal::Attachment {
+ view: view
+ .raw(snatch_guard)
+ .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(view.info.id()))?,
+ usage,
+ },
+ depth_ops: at.depth.hal_ops(),
+ stencil_ops: at.stencil.hal_ops(),
+ clear_value: (at.depth.clear_value, at.stencil.clear_value),
+ });
+ }
+
+ for (index, attachment) in color_attachments.iter().enumerate() {
+ let at = if let Some(attachment) = attachment.as_ref() {
+ attachment
+ } else {
+ colors.push(None);
+ continue;
+ };
+ let color_view: &TextureView<A> = trackers
+ .views
+ .add_single(view_guard, at.view)
+ .ok_or(RenderPassErrorInner::InvalidAttachment(at.view))?;
+ check_multiview(color_view)?;
+ add_view(
+ color_view,
+ AttachmentErrorLocation::Color {
+ index,
+ resolve: false,
+ },
+ )?;
+
+ if !color_view
+ .desc
+ .aspects()
+ .contains(hal::FormatAspects::COLOR)
+ {
+ return Err(RenderPassErrorInner::ColorAttachment(
+ ColorAttachmentError::InvalidFormat(color_view.desc.format),
+ ));
+ }
+
+ Self::add_pass_texture_init_actions(
+ &at.channel,
+ texture_memory_actions,
+ color_view,
+ &mut pending_discard_init_fixups,
+ );
+ render_attachments
+ .push(color_view.to_render_attachment(hal::TextureUses::COLOR_TARGET));
+
+ let mut hal_resolve_target = None;
+ if let Some(resolve_target) = at.resolve_target {
+ let resolve_view: &TextureView<A> = trackers
+ .views
+ .add_single(view_guard, resolve_target)
+ .ok_or(RenderPassErrorInner::InvalidAttachment(resolve_target))?;
+
+ check_multiview(resolve_view)?;
+
+ let resolve_location = AttachmentErrorLocation::Color {
+ index,
+ resolve: true,
+ };
+
+ let render_extent = resolve_view.render_extent.map_err(|reason| {
+ RenderPassErrorInner::TextureViewIsNotRenderable {
+ location: resolve_location,
+ reason,
+ }
+ })?;
+ if color_view.render_extent.unwrap() != render_extent {
+ return Err(RenderPassErrorInner::AttachmentsDimensionMismatch {
+ expected_location: attachment_location,
+ expected_extent: extent.unwrap_or_default(),
+ actual_location: resolve_location,
+ actual_extent: render_extent,
+ });
+ }
+ if color_view.samples == 1 || resolve_view.samples != 1 {
+ return Err(RenderPassErrorInner::InvalidResolveSampleCounts {
+ location: resolve_location,
+ src: color_view.samples,
+ dst: resolve_view.samples,
+ });
+ }
+ if color_view.desc.format != resolve_view.desc.format {
+ return Err(RenderPassErrorInner::MismatchedResolveTextureFormat {
+ location: resolve_location,
+ src: color_view.desc.format,
+ dst: resolve_view.desc.format,
+ });
+ }
+ if !resolve_view
+ .format_features
+ .flags
+ .contains(wgt::TextureFormatFeatureFlags::MULTISAMPLE_RESOLVE)
+ {
+ return Err(RenderPassErrorInner::UnsupportedResolveTargetFormat {
+ location: resolve_location,
+ format: resolve_view.desc.format,
+ });
+ }
+
+ texture_memory_actions.register_implicit_init(
+ &resolve_view.parent,
+ TextureInitRange::from(resolve_view.selector.clone()),
+ );
+ render_attachments
+ .push(resolve_view.to_render_attachment(hal::TextureUses::COLOR_TARGET));
+
+ hal_resolve_target = Some(hal::Attachment {
+ view: resolve_view.raw(snatch_guard).ok_or_else(|| {
+ RenderPassErrorInner::InvalidResolveTarget(resolve_view.info.id())
+ })?,
+ usage: hal::TextureUses::COLOR_TARGET,
+ });
+ }
+
+ colors.push(Some(hal::ColorAttachment {
+ target: hal::Attachment {
+ view: color_view.raw(snatch_guard).ok_or_else(|| {
+ RenderPassErrorInner::InvalidAttachment(color_view.info.id())
+ })?,
+ usage: hal::TextureUses::COLOR_TARGET,
+ },
+ resolve_target: hal_resolve_target,
+ ops: at.channel.hal_ops(),
+ clear_value: at.channel.clear_value,
+ }));
+ }
+
+ let extent = extent.ok_or(RenderPassErrorInner::MissingAttachments)?;
+ let multiview = detected_multiview.expect("Multiview was not detected, no attachments");
+
+ let view_data = AttachmentData {
+ colors: color_attachments
+ .iter()
+ .map(|at| at.as_ref().map(|at| view_guard.get(at.view).unwrap()))
+ .collect(),
+ resolves: color_attachments
+ .iter()
+ .filter_map(|at| match *at {
+ Some(RenderPassColorAttachment {
+ resolve_target: Some(resolve),
+ ..
+ }) => Some(view_guard.get(resolve).unwrap()),
+ _ => None,
+ })
+ .collect(),
+ depth_stencil: depth_stencil_attachment.map(|at| view_guard.get(at.view).unwrap()),
+ };
+
+ let context = RenderPassContext {
+ attachments: view_data.map(|view| view.desc.format),
+ sample_count,
+ multiview,
+ };
+
+ let timestamp_writes = if let Some(tw) = timestamp_writes {
+ let query_set = trackers
+ .query_sets
+ .add_single(query_set_guard, tw.query_set)
+ .ok_or(RenderPassErrorInner::InvalidQuerySet(tw.query_set))?;
+
+ if let Some(index) = tw.beginning_of_pass_write_index {
+ pending_query_resets.use_query_set(tw.query_set, query_set, index);
+ }
+ if let Some(index) = tw.end_of_pass_write_index {
+ pending_query_resets.use_query_set(tw.query_set, query_set, index);
+ }
+
+ Some(hal::RenderPassTimestampWrites {
+ query_set: query_set.raw.as_ref().unwrap(),
+ beginning_of_pass_write_index: tw.beginning_of_pass_write_index,
+ end_of_pass_write_index: tw.end_of_pass_write_index,
+ })
+ } else {
+ None
+ };
+
+ let occlusion_query_set = if let Some(occlusion_query_set) = occlusion_query_set {
+ let query_set = trackers
+ .query_sets
+ .add_single(query_set_guard, occlusion_query_set)
+ .ok_or(RenderPassErrorInner::InvalidQuerySet(occlusion_query_set))?;
+
+ Some(query_set.raw.as_ref().unwrap())
+ } else {
+ None
+ };
+
+ let hal_desc = hal::RenderPassDescriptor {
+ label: hal_label(label, device.instance_flags),
+ extent,
+ sample_count,
+ color_attachments: &colors,
+ depth_stencil_attachment: depth_stencil,
+ multiview,
+ timestamp_writes,
+ occlusion_query_set,
+ };
+ unsafe {
+ encoder.raw.begin_render_pass(&hal_desc);
+ };
+
+ Ok(Self {
+ context,
+ usage_scope: UsageScope::new(buffer_guard, texture_guard),
+ render_attachments,
+ is_depth_read_only,
+ is_stencil_read_only,
+ extent,
+ _phantom: PhantomData,
+ pending_discard_init_fixups,
+ divergent_discarded_depth_stencil_aspect,
+ multiview,
+ })
+ }
+
+ fn finish(
+ mut self,
+ raw: &mut A::CommandEncoder,
+ snatch_guard: &SnatchGuard,
+ ) -> Result<(UsageScope<A>, SurfacesInDiscardState<A>), RenderPassErrorInner> {
+ profiling::scope!("RenderPassInfo::finish");
+ unsafe {
+ raw.end_render_pass();
+ }
+
+ for ra in self.render_attachments {
+ let texture = &ra.texture;
+ check_texture_usage(texture.desc.usage, TextureUsages::RENDER_ATTACHMENT)?;
+
+ // the tracker set of the pass is always in "extend" mode
+ unsafe {
+ self.usage_scope
+ .textures
+ .merge_single(texture, Some(ra.selector.clone()), ra.usage)
+ .map_err(UsageConflict::from)?
+ };
+ }
+
+ // If either only stencil or depth was discarded, we put in a special
+ // clear pass to keep the init status of the aspects in sync. We do this
+ // so we don't need to track init state for depth/stencil aspects
+ // individually.
+ //
+ // Note that we don't go the usual route of "brute force" initializing
+ // the texture when need arises here, since this path is actually
+ // something a user may genuinely want (where as the other cases are
+ // more seen along the lines as gracefully handling a user error).
+ if let Some((aspect, view)) = self.divergent_discarded_depth_stencil_aspect {
+ let (depth_ops, stencil_ops) = if aspect == wgt::TextureAspect::DepthOnly {
+ (
+ hal::AttachmentOps::STORE, // clear depth
+ hal::AttachmentOps::LOAD | hal::AttachmentOps::STORE, // unchanged stencil
+ )
+ } else {
+ (
+ hal::AttachmentOps::LOAD | hal::AttachmentOps::STORE, // unchanged stencil
+ hal::AttachmentOps::STORE, // clear depth
+ )
+ };
+ let desc = hal::RenderPassDescriptor {
+ label: Some("(wgpu internal) Zero init discarded depth/stencil aspect"),
+ extent: view.render_extent.unwrap(),
+ sample_count: view.samples,
+ color_attachments: &[],
+ depth_stencil_attachment: Some(hal::DepthStencilAttachment {
+ target: hal::Attachment {
+ view: view.raw(snatch_guard).ok_or_else(|| {
+ RenderPassErrorInner::InvalidAttachment(view.info.id())
+ })?,
+ usage: hal::TextureUses::DEPTH_STENCIL_WRITE,
+ },
+ depth_ops,
+ stencil_ops,
+ clear_value: (0.0, 0),
+ }),
+ multiview: self.multiview,
+ timestamp_writes: None,
+ occlusion_query_set: None,
+ };
+ unsafe {
+ raw.begin_render_pass(&desc);
+ raw.end_render_pass();
+ }
+ }
+
+ Ok((self.usage_scope, self.pending_discard_init_fixups))
+ }
+}
+
+// Common routines between render/compute
+
+impl Global {
+ pub fn command_encoder_run_render_pass<A: HalApi>(
+ &self,
+ encoder_id: id::CommandEncoderId,
+ pass: &RenderPass,
+ ) -> Result<(), RenderPassError> {
+ self.command_encoder_run_render_pass_impl::<A>(
+ encoder_id,
+ pass.base.as_ref(),
+ &pass.color_targets,
+ pass.depth_stencil_target.as_ref(),
+ pass.timestamp_writes.as_ref(),
+ pass.occlusion_query_set_id,
+ )
+ }
+
+ #[doc(hidden)]
+ pub fn command_encoder_run_render_pass_impl<A: HalApi>(
+ &self,
+ encoder_id: id::CommandEncoderId,
+ base: BasePassRef<RenderCommand>,
+ color_attachments: &[Option<RenderPassColorAttachment>],
+ depth_stencil_attachment: Option<&RenderPassDepthStencilAttachment>,
+ timestamp_writes: Option<&RenderPassTimestampWrites>,
+ occlusion_query_set_id: Option<id::QuerySetId>,
+ ) -> Result<(), RenderPassError> {
+ profiling::scope!(
+ "CommandEncoder::run_render_pass {}",
+ base.label.unwrap_or("")
+ );
+
+ let discard_hal_labels = self
+ .instance
+ .flags
+ .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS);
+ let label = hal_label(base.label, self.instance.flags);
+
+ let pass_scope = PassErrorScope::Pass(encoder_id);
+
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, encoder_id).map_pass_err(pass_scope)?;
+ let device = &cmd_buf.device;
+ let snatch_guard = device.snatchable_lock.read();
+
+ let (scope, pending_discard_init_fixups) = {
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(crate::device::trace::Command::RunRenderPass {
+ base: BasePass::from_ref(base),
+ target_colors: color_attachments.to_vec(),
+ target_depth_stencil: depth_stencil_attachment.cloned(),
+ timestamp_writes: timestamp_writes.cloned(),
+ occlusion_query_set_id,
+ });
+ }
+
+ if !device.is_valid() {
+ return Err(DeviceError::Lost).map_pass_err(pass_scope);
+ }
+
+ let encoder = &mut cmd_buf_data.encoder;
+ let status = &mut cmd_buf_data.status;
+ let tracker = &mut cmd_buf_data.trackers;
+ let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions;
+ let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions;
+ let pending_query_resets = &mut cmd_buf_data.pending_query_resets;
+
+ // We automatically keep extending command buffers over time, and because
+ // we want to insert a command buffer _before_ what we're about to record,
+ // we need to make sure to close the previous one.
+ encoder.close().map_pass_err(pass_scope)?;
+ // We will reset this to `Recording` if we succeed, acts as a fail-safe.
+ *status = CommandEncoderStatus::Error;
+ encoder.open_pass(label).map_pass_err(pass_scope)?;
+
+ let bundle_guard = hub.render_bundles.read();
+ let bind_group_guard = hub.bind_groups.read();
+ let render_pipeline_guard = hub.render_pipelines.read();
+ let query_set_guard = hub.query_sets.read();
+ let buffer_guard = hub.buffers.read();
+ let texture_guard = hub.textures.read();
+ let view_guard = hub.texture_views.read();
+
+ log::trace!(
+ "Encoding render pass begin in command buffer {:?}",
+ encoder_id
+ );
+
+ let mut info = RenderPassInfo::start(
+ device,
+ label,
+ color_attachments,
+ depth_stencil_attachment,
+ timestamp_writes,
+ occlusion_query_set_id,
+ encoder,
+ tracker,
+ texture_memory_actions,
+ pending_query_resets,
+ &*view_guard,
+ &*buffer_guard,
+ &*texture_guard,
+ &*query_set_guard,
+ &snatch_guard,
+ )
+ .map_pass_err(pass_scope)?;
+
+ tracker.set_size(
+ Some(&*buffer_guard),
+ Some(&*texture_guard),
+ Some(&*view_guard),
+ None,
+ Some(&*bind_group_guard),
+ None,
+ Some(&*render_pipeline_guard),
+ Some(&*bundle_guard),
+ Some(&*query_set_guard),
+ );
+
+ let raw = &mut encoder.raw;
+
+ let mut state = State {
+ pipeline_flags: PipelineFlags::empty(),
+ binder: Binder::new(),
+ blend_constant: OptionalState::Unused,
+ stencil_reference: 0,
+ pipeline: None,
+ index: IndexState::default(),
+ vertex: VertexState::default(),
+ debug_scope_depth: 0,
+ };
+ let mut temp_offsets = Vec::new();
+ let mut dynamic_offset_count = 0;
+ let mut string_offset = 0;
+ let mut active_query = None;
+
+ for command in base.commands {
+ match *command {
+ RenderCommand::SetBindGroup {
+ index,
+ num_dynamic_offsets,
+ bind_group_id,
+ } => {
+ api_log!("RenderPass::set_bind_group {index} {bind_group_id:?}");
+
+ let scope = PassErrorScope::SetBindGroup(bind_group_id);
+ let max_bind_groups = device.limits.max_bind_groups;
+ if index >= max_bind_groups {
+ return Err(RenderCommandError::BindGroupIndexOutOfRange {
+ index,
+ max: max_bind_groups,
+ })
+ .map_pass_err(scope);
+ }
+
+ temp_offsets.clear();
+ temp_offsets.extend_from_slice(
+ &base.dynamic_offsets
+ [dynamic_offset_count..dynamic_offset_count + num_dynamic_offsets],
+ );
+ dynamic_offset_count += num_dynamic_offsets;
+
+ let bind_group = tracker
+ .bind_groups
+ .add_single(&*bind_group_guard, bind_group_id)
+ .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id))
+ .map_pass_err(scope)?;
+
+ if bind_group.device.as_info().id() != device.as_info().id() {
+ return Err(DeviceError::WrongDevice).map_pass_err(scope);
+ }
+
+ bind_group
+ .validate_dynamic_bindings(index, &temp_offsets, &cmd_buf.limits)
+ .map_pass_err(scope)?;
+
+ // merge the resource tracker in
+ unsafe {
+ info.usage_scope
+ .merge_bind_group(&bind_group.used)
+ .map_pass_err(scope)?;
+ }
+ //Note: stateless trackers are not merged: the lifetime reference
+ // is held to the bind group itself.
+
+ buffer_memory_init_actions.extend(
+ bind_group.used_buffer_ranges.iter().filter_map(|action| {
+ action
+ .buffer
+ .initialization_status
+ .read()
+ .check_action(action)
+ }),
+ );
+ for action in bind_group.used_texture_ranges.iter() {
+ info.pending_discard_init_fixups
+ .extend(texture_memory_actions.register_init_action(action));
+ }
+
+ let pipeline_layout = state.binder.pipeline_layout.clone();
+ let entries =
+ state
+ .binder
+ .assign_group(index as usize, bind_group, &temp_offsets);
+ if !entries.is_empty() && pipeline_layout.is_some() {
+ let pipeline_layout = pipeline_layout.as_ref().unwrap().raw();
+ for (i, e) in entries.iter().enumerate() {
+ if let Some(group) = e.group.as_ref() {
+ let raw_bg = group
+ .raw(&snatch_guard)
+ .ok_or(RenderPassErrorInner::InvalidBindGroup(i))
+ .map_pass_err(scope)?;
+ unsafe {
+ raw.set_bind_group(
+ pipeline_layout,
+ index + i as u32,
+ raw_bg,
+ &e.dynamic_offsets,
+ );
+ }
+ }
+ }
+ }
+ }
+ RenderCommand::SetPipeline(pipeline_id) => {
+ api_log!("RenderPass::set_pipeline {pipeline_id:?}");
+
+ let scope = PassErrorScope::SetPipelineRender(pipeline_id);
+ state.pipeline = Some(pipeline_id);
+
+ let pipeline: &pipeline::RenderPipeline<A> = tracker
+ .render_pipelines
+ .add_single(&*render_pipeline_guard, pipeline_id)
+ .ok_or(RenderCommandError::InvalidPipeline(pipeline_id))
+ .map_pass_err(scope)?;
+
+ if pipeline.device.as_info().id() != device.as_info().id() {
+ return Err(DeviceError::WrongDevice).map_pass_err(scope);
+ }
+
+ info.context
+ .check_compatible(
+ &pipeline.pass_context,
+ RenderPassCompatibilityCheckType::RenderPipeline,
+ )
+ .map_err(RenderCommandError::IncompatiblePipelineTargets)
+ .map_pass_err(scope)?;
+
+ state.pipeline_flags = pipeline.flags;
+
+ if (pipeline.flags.contains(PipelineFlags::WRITES_DEPTH)
+ && info.is_depth_read_only)
+ || (pipeline.flags.contains(PipelineFlags::WRITES_STENCIL)
+ && info.is_stencil_read_only)
+ {
+ return Err(RenderCommandError::IncompatiblePipelineRods)
+ .map_pass_err(scope);
+ }
+
+ state
+ .blend_constant
+ .require(pipeline.flags.contains(PipelineFlags::BLEND_CONSTANT));
+
+ unsafe {
+ raw.set_render_pipeline(pipeline.raw());
+ }
+
+ if pipeline.flags.contains(PipelineFlags::STENCIL_REFERENCE) {
+ unsafe {
+ raw.set_stencil_reference(state.stencil_reference);
+ }
+ }
+
+ // Rebind resource
+ if state.binder.pipeline_layout.is_none()
+ || !state
+ .binder
+ .pipeline_layout
+ .as_ref()
+ .unwrap()
+ .is_equal(&pipeline.layout)
+ {
+ let (start_index, entries) = state.binder.change_pipeline_layout(
+ &pipeline.layout,
+ &pipeline.late_sized_buffer_groups,
+ );
+ if !entries.is_empty() {
+ for (i, e) in entries.iter().enumerate() {
+ if let Some(group) = e.group.as_ref() {
+ let raw_bg = group
+ .raw(&snatch_guard)
+ .ok_or(RenderPassErrorInner::InvalidBindGroup(i))
+ .map_pass_err(scope)?;
+ unsafe {
+ raw.set_bind_group(
+ pipeline.layout.raw(),
+ start_index as u32 + i as u32,
+ raw_bg,
+ &e.dynamic_offsets,
+ );
+ }
+ }
+ }
+ }
+
+ // Clear push constant ranges
+ let non_overlapping = super::bind::compute_nonoverlapping_ranges(
+ &pipeline.layout.push_constant_ranges,
+ );
+ for range in non_overlapping {
+ let offset = range.range.start;
+ let size_bytes = range.range.end - offset;
+ super::push_constant_clear(
+ offset,
+ size_bytes,
+ |clear_offset, clear_data| unsafe {
+ raw.set_push_constants(
+ pipeline.layout.raw(),
+ range.stages,
+ clear_offset,
+ clear_data,
+ );
+ },
+ );
+ }
+ }
+
+ state.index.pipeline_format = pipeline.strip_index_format;
+
+ let vertex_steps_len = pipeline.vertex_steps.len();
+ state.vertex.buffers_required = vertex_steps_len as u32;
+
+ // Initialize each `vertex.inputs[i].step` from
+ // `pipeline.vertex_steps[i]`. Enlarge `vertex.inputs`
+ // as necessary to accommodate all slots in the
+ // pipeline. If `vertex.inputs` is longer, fill the
+ // extra entries with default `VertexStep`s.
+ while state.vertex.inputs.len() < vertex_steps_len {
+ state.vertex.inputs.push(VertexBufferState::EMPTY);
+ }
+
+ // This is worse as a `zip`, but it's close.
+ let mut steps = pipeline.vertex_steps.iter();
+ for input in state.vertex.inputs.iter_mut() {
+ input.step = steps.next().cloned().unwrap_or_default();
+ }
+
+ // Update vertex buffer limits.
+ state.vertex.update_limits();
+ }
+ RenderCommand::SetIndexBuffer {
+ buffer_id,
+ index_format,
+ offset,
+ size,
+ } => {
+ api_log!("RenderPass::set_index_buffer {buffer_id:?}");
+
+ let scope = PassErrorScope::SetIndexBuffer(buffer_id);
+ let buffer = info
+ .usage_scope
+ .buffers
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDEX)
+ .map_pass_err(scope)?;
+
+ if buffer.device.as_info().id() != device.as_info().id() {
+ return Err(DeviceError::WrongDevice).map_pass_err(scope);
+ }
+
+ check_buffer_usage(buffer.usage, BufferUsages::INDEX)
+ .map_pass_err(scope)?;
+ let buf_raw = buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(RenderCommandError::DestroyedBuffer(buffer_id))
+ .map_pass_err(scope)?;
+
+ let end = match size {
+ Some(s) => offset + s.get(),
+ None => buffer.size,
+ };
+ state.index.bound_buffer_view = Some((buffer_id, offset..end));
+
+ state.index.format = Some(index_format);
+ state.index.update_limit();
+
+ buffer_memory_init_actions.extend(
+ buffer.initialization_status.read().create_action(
+ buffer,
+ offset..end,
+ MemoryInitKind::NeedsInitializedMemory,
+ ),
+ );
+
+ let bb = hal::BufferBinding {
+ buffer: buf_raw,
+ offset,
+ size,
+ };
+ unsafe {
+ raw.set_index_buffer(bb, index_format);
+ }
+ }
+ RenderCommand::SetVertexBuffer {
+ slot,
+ buffer_id,
+ offset,
+ size,
+ } => {
+ api_log!("RenderPass::set_vertex_buffer {slot} {buffer_id:?}");
+
+ let scope = PassErrorScope::SetVertexBuffer(buffer_id);
+ let buffer = info
+ .usage_scope
+ .buffers
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX)
+ .map_pass_err(scope)?;
+
+ if buffer.device.as_info().id() != device.as_info().id() {
+ return Err(DeviceError::WrongDevice).map_pass_err(scope);
+ }
+
+ let max_vertex_buffers = device.limits.max_vertex_buffers;
+ if slot >= max_vertex_buffers {
+ return Err(RenderCommandError::VertexBufferIndexOutOfRange {
+ index: slot,
+ max: max_vertex_buffers,
+ })
+ .map_pass_err(scope);
+ }
+
+ check_buffer_usage(buffer.usage, BufferUsages::VERTEX)
+ .map_pass_err(scope)?;
+ let buf_raw = buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(RenderCommandError::DestroyedBuffer(buffer_id))
+ .map_pass_err(scope)?;
+
+ let empty_slots =
+ (1 + slot as usize).saturating_sub(state.vertex.inputs.len());
+ state
+ .vertex
+ .inputs
+ .extend(iter::repeat(VertexBufferState::EMPTY).take(empty_slots));
+ let vertex_state = &mut state.vertex.inputs[slot as usize];
+ //TODO: where are we checking that the offset is in bound?
+ vertex_state.total_size = match size {
+ Some(s) => s.get(),
+ None => buffer.size - offset,
+ };
+ vertex_state.bound = true;
+
+ buffer_memory_init_actions.extend(
+ buffer.initialization_status.read().create_action(
+ buffer,
+ offset..(offset + vertex_state.total_size),
+ MemoryInitKind::NeedsInitializedMemory,
+ ),
+ );
+
+ let bb = hal::BufferBinding {
+ buffer: buf_raw,
+ offset,
+ size,
+ };
+ unsafe {
+ raw.set_vertex_buffer(slot, bb);
+ }
+ state.vertex.update_limits();
+ }
+ RenderCommand::SetBlendConstant(ref color) => {
+ api_log!("RenderPass::set_blend_constant");
+
+ state.blend_constant = OptionalState::Set;
+ let array = [
+ color.r as f32,
+ color.g as f32,
+ color.b as f32,
+ color.a as f32,
+ ];
+ unsafe {
+ raw.set_blend_constants(&array);
+ }
+ }
+ RenderCommand::SetStencilReference(value) => {
+ api_log!("RenderPass::set_stencil_reference {value}");
+
+ state.stencil_reference = value;
+ if state
+ .pipeline_flags
+ .contains(PipelineFlags::STENCIL_REFERENCE)
+ {
+ unsafe {
+ raw.set_stencil_reference(value);
+ }
+ }
+ }
+ RenderCommand::SetViewport {
+ ref rect,
+ depth_min,
+ depth_max,
+ } => {
+ api_log!("RenderPass::set_viewport {rect:?}");
+
+ let scope = PassErrorScope::SetViewport;
+ if rect.x < 0.0
+ || rect.y < 0.0
+ || rect.w <= 0.0
+ || rect.h <= 0.0
+ || rect.x + rect.w > info.extent.width as f32
+ || rect.y + rect.h > info.extent.height as f32
+ {
+ return Err(RenderCommandError::InvalidViewportRect(
+ *rect,
+ info.extent,
+ ))
+ .map_pass_err(scope);
+ }
+ if !(0.0..=1.0).contains(&depth_min) || !(0.0..=1.0).contains(&depth_max) {
+ return Err(RenderCommandError::InvalidViewportDepth(
+ depth_min, depth_max,
+ ))
+ .map_pass_err(scope);
+ }
+ let r = hal::Rect {
+ x: rect.x,
+ y: rect.y,
+ w: rect.w,
+ h: rect.h,
+ };
+ unsafe {
+ raw.set_viewport(&r, depth_min..depth_max);
+ }
+ }
+ RenderCommand::SetPushConstant {
+ stages,
+ offset,
+ size_bytes,
+ values_offset,
+ } => {
+ api_log!("RenderPass::set_push_constants");
+
+ let scope = PassErrorScope::SetPushConstant;
+ let values_offset = values_offset
+ .ok_or(RenderPassErrorInner::InvalidValuesOffset)
+ .map_pass_err(scope)?;
+
+ let end_offset_bytes = offset + size_bytes;
+ let values_end_offset =
+ (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize;
+ let data_slice =
+ &base.push_constant_data[(values_offset as usize)..values_end_offset];
+
+ let pipeline_layout = state
+ .binder
+ .pipeline_layout
+ .as_ref()
+ .ok_or(DrawError::MissingPipeline)
+ .map_pass_err(scope)?;
+
+ pipeline_layout
+ .validate_push_constant_ranges(stages, offset, end_offset_bytes)
+ .map_err(RenderCommandError::from)
+ .map_pass_err(scope)?;
+
+ unsafe {
+ raw.set_push_constants(
+ pipeline_layout.raw(),
+ stages,
+ offset,
+ data_slice,
+ )
+ }
+ }
+ RenderCommand::SetScissor(ref rect) => {
+ api_log!("RenderPass::set_scissor_rect {rect:?}");
+
+ let scope = PassErrorScope::SetScissorRect;
+ if rect.x + rect.w > info.extent.width
+ || rect.y + rect.h > info.extent.height
+ {
+ return Err(RenderCommandError::InvalidScissorRect(*rect, info.extent))
+ .map_pass_err(scope);
+ }
+ let r = hal::Rect {
+ x: rect.x,
+ y: rect.y,
+ w: rect.w,
+ h: rect.h,
+ };
+ unsafe {
+ raw.set_scissor_rect(&r);
+ }
+ }
+ RenderCommand::Draw {
+ vertex_count,
+ instance_count,
+ first_vertex,
+ first_instance,
+ } => {
+ api_log!(
+ "RenderPass::draw {vertex_count} {instance_count} {first_vertex} {first_instance}"
+ );
+
+ let indexed = false;
+ let scope = PassErrorScope::Draw {
+ indexed,
+ indirect: false,
+ pipeline: state.pipeline,
+ };
+ state.is_ready(indexed).map_pass_err(scope)?;
+
+ let last_vertex = first_vertex as u64 + vertex_count as u64;
+ let vertex_limit = state.vertex.vertex_limit;
+ if last_vertex > vertex_limit {
+ return Err(DrawError::VertexBeyondLimit {
+ last_vertex,
+ vertex_limit,
+ slot: state.vertex.vertex_limit_slot,
+ })
+ .map_pass_err(scope);
+ }
+ let last_instance = first_instance as u64 + instance_count as u64;
+ let instance_limit = state.vertex.instance_limit;
+ if last_instance > instance_limit {
+ return Err(DrawError::InstanceBeyondLimit {
+ last_instance,
+ instance_limit,
+ slot: state.vertex.instance_limit_slot,
+ })
+ .map_pass_err(scope);
+ }
+
+ unsafe {
+ if instance_count > 0 && vertex_count > 0 {
+ raw.draw(
+ first_vertex,
+ vertex_count,
+ first_instance,
+ instance_count,
+ );
+ }
+ }
+ }
+ RenderCommand::DrawIndexed {
+ index_count,
+ instance_count,
+ first_index,
+ base_vertex,
+ first_instance,
+ } => {
+ api_log!("RenderPass::draw_indexed {index_count} {instance_count} {first_index} {base_vertex} {first_instance}");
+
+ let indexed = true;
+ let scope = PassErrorScope::Draw {
+ indexed,
+ indirect: false,
+ pipeline: state.pipeline,
+ };
+ state.is_ready(indexed).map_pass_err(scope)?;
+
+ let last_index = first_index as u64 + index_count as u64;
+ let index_limit = state.index.limit;
+ if last_index > index_limit {
+ return Err(DrawError::IndexBeyondLimit {
+ last_index,
+ index_limit,
+ })
+ .map_pass_err(scope);
+ }
+ let last_instance = first_instance as u64 + instance_count as u64;
+ let instance_limit = state.vertex.instance_limit;
+ if last_instance > instance_limit {
+ return Err(DrawError::InstanceBeyondLimit {
+ last_instance,
+ instance_limit,
+ slot: state.vertex.instance_limit_slot,
+ })
+ .map_pass_err(scope);
+ }
+
+ unsafe {
+ if instance_count > 0 && index_count > 0 {
+ raw.draw_indexed(
+ first_index,
+ index_count,
+ base_vertex,
+ first_instance,
+ instance_count,
+ );
+ }
+ }
+ }
+ RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count,
+ indexed,
+ } => {
+ api_log!("RenderPass::draw_indirect (indexed:{indexed}) {buffer_id:?} {offset} {count:?}");
+
+ let scope = PassErrorScope::Draw {
+ indexed,
+ indirect: true,
+ pipeline: state.pipeline,
+ };
+ state.is_ready(indexed).map_pass_err(scope)?;
+
+ let stride = match indexed {
+ false => mem::size_of::<wgt::DrawIndirectArgs>(),
+ true => mem::size_of::<wgt::DrawIndexedIndirectArgs>(),
+ };
+
+ if count.is_some() {
+ device
+ .require_features(wgt::Features::MULTI_DRAW_INDIRECT)
+ .map_pass_err(scope)?;
+ }
+ device
+ .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)
+ .map_pass_err(scope)?;
+
+ let indirect_buffer = info
+ .usage_scope
+ .buffers
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT)
+ .map_pass_err(scope)?;
+ check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT)
+ .map_pass_err(scope)?;
+ let indirect_raw = indirect_buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(RenderCommandError::DestroyedBuffer(buffer_id))
+ .map_pass_err(scope)?;
+
+ let actual_count = count.map_or(1, |c| c.get());
+
+ let end_offset = offset + stride as u64 * actual_count as u64;
+ if end_offset > indirect_buffer.size {
+ return Err(RenderPassErrorInner::IndirectBufferOverrun {
+ count,
+ offset,
+ end_offset,
+ buffer_size: indirect_buffer.size,
+ })
+ .map_pass_err(scope);
+ }
+
+ buffer_memory_init_actions.extend(
+ indirect_buffer.initialization_status.read().create_action(
+ indirect_buffer,
+ offset..end_offset,
+ MemoryInitKind::NeedsInitializedMemory,
+ ),
+ );
+
+ match indexed {
+ false => unsafe {
+ raw.draw_indirect(indirect_raw, offset, actual_count);
+ },
+ true => unsafe {
+ raw.draw_indexed_indirect(indirect_raw, offset, actual_count);
+ },
+ }
+ }
+ RenderCommand::MultiDrawIndirectCount {
+ buffer_id,
+ offset,
+ count_buffer_id,
+ count_buffer_offset,
+ max_count,
+ indexed,
+ } => {
+ api_log!("RenderPass::multi_draw_indirect_count (indexed:{indexed}) {buffer_id:?} {offset} {count_buffer_id:?} {count_buffer_offset:?} {max_count:?}");
+
+ let scope = PassErrorScope::Draw {
+ indexed,
+ indirect: true,
+ pipeline: state.pipeline,
+ };
+ state.is_ready(indexed).map_pass_err(scope)?;
+
+ let stride = match indexed {
+ false => mem::size_of::<wgt::DrawIndirectArgs>(),
+ true => mem::size_of::<wgt::DrawIndexedIndirectArgs>(),
+ } as u64;
+
+ device
+ .require_features(wgt::Features::MULTI_DRAW_INDIRECT_COUNT)
+ .map_pass_err(scope)?;
+ device
+ .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)
+ .map_pass_err(scope)?;
+
+ let indirect_buffer = info
+ .usage_scope
+ .buffers
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT)
+ .map_pass_err(scope)?;
+ check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT)
+ .map_pass_err(scope)?;
+ let indirect_raw = indirect_buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(RenderCommandError::DestroyedBuffer(buffer_id))
+ .map_pass_err(scope)?;
+
+ let count_buffer = info
+ .usage_scope
+ .buffers
+ .merge_single(
+ &*buffer_guard,
+ count_buffer_id,
+ hal::BufferUses::INDIRECT,
+ )
+ .map_pass_err(scope)?;
+ check_buffer_usage(count_buffer.usage, BufferUsages::INDIRECT)
+ .map_pass_err(scope)?;
+ let count_raw = count_buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(RenderCommandError::DestroyedBuffer(count_buffer_id))
+ .map_pass_err(scope)?;
+
+ let end_offset = offset + stride * max_count as u64;
+ if end_offset > indirect_buffer.size {
+ return Err(RenderPassErrorInner::IndirectBufferOverrun {
+ count: None,
+ offset,
+ end_offset,
+ buffer_size: indirect_buffer.size,
+ })
+ .map_pass_err(scope);
+ }
+ buffer_memory_init_actions.extend(
+ indirect_buffer.initialization_status.read().create_action(
+ indirect_buffer,
+ offset..end_offset,
+ MemoryInitKind::NeedsInitializedMemory,
+ ),
+ );
+
+ let begin_count_offset = count_buffer_offset;
+ let end_count_offset = count_buffer_offset + 4;
+ if end_count_offset > count_buffer.size {
+ return Err(RenderPassErrorInner::IndirectCountBufferOverrun {
+ begin_count_offset,
+ end_count_offset,
+ count_buffer_size: count_buffer.size,
+ })
+ .map_pass_err(scope);
+ }
+ buffer_memory_init_actions.extend(
+ count_buffer.initialization_status.read().create_action(
+ count_buffer,
+ count_buffer_offset..end_count_offset,
+ MemoryInitKind::NeedsInitializedMemory,
+ ),
+ );
+
+ match indexed {
+ false => unsafe {
+ raw.draw_indirect_count(
+ indirect_raw,
+ offset,
+ count_raw,
+ count_buffer_offset,
+ max_count,
+ );
+ },
+ true => unsafe {
+ raw.draw_indexed_indirect_count(
+ indirect_raw,
+ offset,
+ count_raw,
+ count_buffer_offset,
+ max_count,
+ );
+ },
+ }
+ }
+ RenderCommand::PushDebugGroup { color: _, len } => {
+ state.debug_scope_depth += 1;
+ if !discard_hal_labels {
+ let label = str::from_utf8(
+ &base.string_data[string_offset..string_offset + len],
+ )
+ .unwrap();
+
+ api_log!("RenderPass::push_debug_group {label:?}");
+ unsafe {
+ raw.begin_debug_marker(label);
+ }
+ }
+ string_offset += len;
+ }
+ RenderCommand::PopDebugGroup => {
+ api_log!("RenderPass::pop_debug_group");
+
+ let scope = PassErrorScope::PopDebugGroup;
+ if state.debug_scope_depth == 0 {
+ return Err(RenderPassErrorInner::InvalidPopDebugGroup)
+ .map_pass_err(scope);
+ }
+ state.debug_scope_depth -= 1;
+ if !discard_hal_labels {
+ unsafe {
+ raw.end_debug_marker();
+ }
+ }
+ }
+ RenderCommand::InsertDebugMarker { color: _, len } => {
+ if !discard_hal_labels {
+ let label = str::from_utf8(
+ &base.string_data[string_offset..string_offset + len],
+ )
+ .unwrap();
+ api_log!("RenderPass::insert_debug_marker {label:?}");
+ unsafe {
+ raw.insert_debug_marker(label);
+ }
+ }
+ string_offset += len;
+ }
+ RenderCommand::WriteTimestamp {
+ query_set_id,
+ query_index,
+ } => {
+ api_log!("RenderPass::write_timestamps {query_set_id:?} {query_index}");
+ let scope = PassErrorScope::WriteTimestamp;
+
+ device
+ .require_features(wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES)
+ .map_pass_err(scope)?;
+
+ let query_set = tracker
+ .query_sets
+ .add_single(&*query_set_guard, query_set_id)
+ .ok_or(RenderCommandError::InvalidQuerySet(query_set_id))
+ .map_pass_err(scope)?;
+
+ query_set
+ .validate_and_write_timestamp(
+ raw,
+ query_set_id,
+ query_index,
+ Some(&mut cmd_buf_data.pending_query_resets),
+ )
+ .map_pass_err(scope)?;
+ }
+ RenderCommand::BeginOcclusionQuery { query_index } => {
+ api_log!("RenderPass::begin_occlusion_query {query_index}");
+ let scope = PassErrorScope::BeginOcclusionQuery;
+
+ let query_set_id = occlusion_query_set_id
+ .ok_or(RenderPassErrorInner::MissingOcclusionQuerySet)
+ .map_pass_err(scope)?;
+
+ let query_set = tracker
+ .query_sets
+ .add_single(&*query_set_guard, query_set_id)
+ .ok_or(RenderCommandError::InvalidQuerySet(query_set_id))
+ .map_pass_err(scope)?;
+
+ query_set
+ .validate_and_begin_occlusion_query(
+ raw,
+ query_set_id,
+ query_index,
+ Some(&mut cmd_buf_data.pending_query_resets),
+ &mut active_query,
+ )
+ .map_pass_err(scope)?;
+ }
+ RenderCommand::EndOcclusionQuery => {
+ api_log!("RenderPass::end_occlusion_query");
+ let scope = PassErrorScope::EndOcclusionQuery;
+
+ end_occlusion_query(raw, &*query_set_guard, &mut active_query)
+ .map_pass_err(scope)?;
+ }
+ RenderCommand::BeginPipelineStatisticsQuery {
+ query_set_id,
+ query_index,
+ } => {
+ api_log!("RenderPass::begin_pipeline_statistics_query {query_set_id:?} {query_index}");
+ let scope = PassErrorScope::BeginPipelineStatisticsQuery;
+
+ let query_set = tracker
+ .query_sets
+ .add_single(&*query_set_guard, query_set_id)
+ .ok_or(RenderCommandError::InvalidQuerySet(query_set_id))
+ .map_pass_err(scope)?;
+
+ query_set
+ .validate_and_begin_pipeline_statistics_query(
+ raw,
+ query_set_id,
+ query_index,
+ Some(&mut cmd_buf_data.pending_query_resets),
+ &mut active_query,
+ )
+ .map_pass_err(scope)?;
+ }
+ RenderCommand::EndPipelineStatisticsQuery => {
+ api_log!("RenderPass::end_pipeline_statistics_query");
+ let scope = PassErrorScope::EndPipelineStatisticsQuery;
+
+ end_pipeline_statistics_query(raw, &*query_set_guard, &mut active_query)
+ .map_pass_err(scope)?;
+ }
+ RenderCommand::ExecuteBundle(bundle_id) => {
+ api_log!("RenderPass::execute_bundle {bundle_id:?}");
+ let scope = PassErrorScope::ExecuteBundle;
+ let bundle: &command::RenderBundle<A> = tracker
+ .bundles
+ .add_single(&*bundle_guard, bundle_id)
+ .ok_or(RenderCommandError::InvalidRenderBundle(bundle_id))
+ .map_pass_err(scope)?;
+
+ if bundle.device.as_info().id() != device.as_info().id() {
+ return Err(DeviceError::WrongDevice).map_pass_err(scope);
+ }
+
+ info.context
+ .check_compatible(
+ &bundle.context,
+ RenderPassCompatibilityCheckType::RenderBundle,
+ )
+ .map_err(RenderPassErrorInner::IncompatibleBundleTargets)
+ .map_pass_err(scope)?;
+
+ if (info.is_depth_read_only && !bundle.is_depth_read_only)
+ || (info.is_stencil_read_only && !bundle.is_stencil_read_only)
+ {
+ return Err(
+ RenderPassErrorInner::IncompatibleBundleReadOnlyDepthStencil {
+ pass_depth: info.is_depth_read_only,
+ pass_stencil: info.is_stencil_read_only,
+ bundle_depth: bundle.is_depth_read_only,
+ bundle_stencil: bundle.is_stencil_read_only,
+ },
+ )
+ .map_pass_err(scope);
+ }
+
+ buffer_memory_init_actions.extend(
+ bundle
+ .buffer_memory_init_actions
+ .iter()
+ .filter_map(|action| {
+ action
+ .buffer
+ .initialization_status
+ .read()
+ .check_action(action)
+ }),
+ );
+ for action in bundle.texture_memory_init_actions.iter() {
+ info.pending_discard_init_fixups
+ .extend(texture_memory_actions.register_init_action(action));
+ }
+
+ unsafe { bundle.execute(raw) }
+ .map_err(|e| match e {
+ ExecutionError::DestroyedBuffer(id) => {
+ RenderCommandError::DestroyedBuffer(id)
+ }
+ ExecutionError::InvalidBindGroup(id) => {
+ RenderCommandError::InvalidBindGroup(id)
+ }
+ ExecutionError::Unimplemented(what) => {
+ RenderCommandError::Unimplemented(what)
+ }
+ })
+ .map_pass_err(scope)?;
+
+ unsafe {
+ info.usage_scope
+ .merge_render_bundle(&bundle.used)
+ .map_pass_err(scope)?;
+ tracker
+ .add_from_render_bundle(&bundle.used)
+ .map_pass_err(scope)?;
+ };
+ state.reset_bundle();
+ }
+ }
+ }
+
+ log::trace!("Merging renderpass into cmd_buf {:?}", encoder_id);
+ let (trackers, pending_discard_init_fixups) =
+ info.finish(raw, &snatch_guard).map_pass_err(pass_scope)?;
+
+ encoder.close().map_pass_err(pass_scope)?;
+ (trackers, pending_discard_init_fixups)
+ };
+
+ let cmd_buf = hub.command_buffers.get(encoder_id.transmute()).unwrap();
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ let query_set_guard = hub.query_sets.read();
+
+ let encoder = &mut cmd_buf_data.encoder;
+ let status = &mut cmd_buf_data.status;
+ let tracker = &mut cmd_buf_data.trackers;
+
+ {
+ let transit = encoder.open().map_pass_err(pass_scope)?;
+
+ fixup_discarded_surfaces(
+ pending_discard_init_fixups.into_iter(),
+ transit,
+ &mut tracker.textures,
+ &cmd_buf.device,
+ );
+
+ cmd_buf_data
+ .pending_query_resets
+ .reset_queries(
+ transit,
+ &query_set_guard,
+ cmd_buf.device.info.id().backend(),
+ )
+ .map_err(RenderCommandError::InvalidQuerySet)
+ .map_pass_err(PassErrorScope::QueryReset)?;
+
+ super::CommandBuffer::insert_barriers_from_scope(
+ transit,
+ tracker,
+ &scope,
+ &snatch_guard,
+ );
+ }
+
+ *status = CommandEncoderStatus::Recording;
+ encoder.close_and_swap().map_pass_err(pass_scope)?;
+
+ Ok(())
+ }
+}
+
+pub mod render_ffi {
+ use super::{
+ super::{Rect, RenderCommand},
+ RenderPass,
+ };
+ use crate::{id, RawString};
+ use std::{convert::TryInto, ffi, num::NonZeroU32, slice};
+ use wgt::{BufferAddress, BufferSize, Color, DynamicOffset, IndexFormat};
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given pointer is
+ /// valid for `offset_length` elements.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_render_pass_set_bind_group(
+ pass: &mut RenderPass,
+ index: u32,
+ bind_group_id: id::BindGroupId,
+ offsets: *const DynamicOffset,
+ offset_length: usize,
+ ) {
+ let redundant = unsafe {
+ pass.current_bind_groups.set_and_check_redundant(
+ bind_group_id,
+ index,
+ &mut pass.base.dynamic_offsets,
+ offsets,
+ offset_length,
+ )
+ };
+
+ if redundant {
+ return;
+ }
+
+ pass.base.commands.push(RenderCommand::SetBindGroup {
+ index,
+ num_dynamic_offsets: offset_length,
+ bind_group_id,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_set_pipeline(
+ pass: &mut RenderPass,
+ pipeline_id: id::RenderPipelineId,
+ ) {
+ if pass.current_pipeline.set_and_check_redundant(pipeline_id) {
+ return;
+ }
+
+ pass.base
+ .commands
+ .push(RenderCommand::SetPipeline(pipeline_id));
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_set_vertex_buffer(
+ pass: &mut RenderPass,
+ slot: u32,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ size: Option<BufferSize>,
+ ) {
+ pass.base.commands.push(RenderCommand::SetVertexBuffer {
+ slot,
+ buffer_id,
+ offset,
+ size,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_set_index_buffer(
+ pass: &mut RenderPass,
+ buffer: id::BufferId,
+ index_format: IndexFormat,
+ offset: BufferAddress,
+ size: Option<BufferSize>,
+ ) {
+ pass.set_index_buffer(buffer, index_format, offset, size);
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_set_blend_constant(pass: &mut RenderPass, color: &Color) {
+ pass.base
+ .commands
+ .push(RenderCommand::SetBlendConstant(*color));
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_set_stencil_reference(pass: &mut RenderPass, value: u32) {
+ pass.base
+ .commands
+ .push(RenderCommand::SetStencilReference(value));
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_set_viewport(
+ pass: &mut RenderPass,
+ x: f32,
+ y: f32,
+ w: f32,
+ h: f32,
+ depth_min: f32,
+ depth_max: f32,
+ ) {
+ pass.base.commands.push(RenderCommand::SetViewport {
+ rect: Rect { x, y, w, h },
+ depth_min,
+ depth_max,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_set_scissor_rect(
+ pass: &mut RenderPass,
+ x: u32,
+ y: u32,
+ w: u32,
+ h: u32,
+ ) {
+ pass.base
+ .commands
+ .push(RenderCommand::SetScissor(Rect { x, y, w, h }));
+ }
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given pointer is
+ /// valid for `size_bytes` bytes.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_render_pass_set_push_constants(
+ pass: &mut RenderPass,
+ stages: wgt::ShaderStages,
+ offset: u32,
+ size_bytes: u32,
+ data: *const u8,
+ ) {
+ assert_eq!(
+ offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1),
+ 0,
+ "Push constant offset must be aligned to 4 bytes."
+ );
+ assert_eq!(
+ size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1),
+ 0,
+ "Push constant size must be aligned to 4 bytes."
+ );
+ let data_slice = unsafe { slice::from_raw_parts(data, size_bytes as usize) };
+ let value_offset = pass.base.push_constant_data.len().try_into().expect(
+ "Ran out of push constant space. Don't set 4gb of push constants per RenderPass.",
+ );
+
+ pass.base.push_constant_data.extend(
+ data_slice
+ .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize)
+ .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])),
+ );
+
+ pass.base.commands.push(RenderCommand::SetPushConstant {
+ stages,
+ offset,
+ size_bytes,
+ values_offset: Some(value_offset),
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_draw(
+ pass: &mut RenderPass,
+ vertex_count: u32,
+ instance_count: u32,
+ first_vertex: u32,
+ first_instance: u32,
+ ) {
+ pass.base.commands.push(RenderCommand::Draw {
+ vertex_count,
+ instance_count,
+ first_vertex,
+ first_instance,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_draw_indexed(
+ pass: &mut RenderPass,
+ index_count: u32,
+ instance_count: u32,
+ first_index: u32,
+ base_vertex: i32,
+ first_instance: u32,
+ ) {
+ pass.base.commands.push(RenderCommand::DrawIndexed {
+ index_count,
+ instance_count,
+ first_index,
+ base_vertex,
+ first_instance,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_draw_indirect(
+ pass: &mut RenderPass,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ ) {
+ pass.base.commands.push(RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count: None,
+ indexed: false,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_draw_indexed_indirect(
+ pass: &mut RenderPass,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ ) {
+ pass.base.commands.push(RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count: None,
+ indexed: true,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_multi_draw_indirect(
+ pass: &mut RenderPass,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ count: u32,
+ ) {
+ pass.base.commands.push(RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count: NonZeroU32::new(count),
+ indexed: false,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_multi_draw_indexed_indirect(
+ pass: &mut RenderPass,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ count: u32,
+ ) {
+ pass.base.commands.push(RenderCommand::MultiDrawIndirect {
+ buffer_id,
+ offset,
+ count: NonZeroU32::new(count),
+ indexed: true,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_multi_draw_indirect_count(
+ pass: &mut RenderPass,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ count_buffer_id: id::BufferId,
+ count_buffer_offset: BufferAddress,
+ max_count: u32,
+ ) {
+ pass.base
+ .commands
+ .push(RenderCommand::MultiDrawIndirectCount {
+ buffer_id,
+ offset,
+ count_buffer_id,
+ count_buffer_offset,
+ max_count,
+ indexed: false,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_multi_draw_indexed_indirect_count(
+ pass: &mut RenderPass,
+ buffer_id: id::BufferId,
+ offset: BufferAddress,
+ count_buffer_id: id::BufferId,
+ count_buffer_offset: BufferAddress,
+ max_count: u32,
+ ) {
+ pass.base
+ .commands
+ .push(RenderCommand::MultiDrawIndirectCount {
+ buffer_id,
+ offset,
+ count_buffer_id,
+ count_buffer_offset,
+ max_count,
+ indexed: true,
+ });
+ }
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given `label`
+ /// is a valid null-terminated string.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_render_pass_push_debug_group(
+ pass: &mut RenderPass,
+ label: RawString,
+ color: u32,
+ ) {
+ let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes();
+ pass.base.string_data.extend_from_slice(bytes);
+
+ pass.base.commands.push(RenderCommand::PushDebugGroup {
+ color,
+ len: bytes.len(),
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_pop_debug_group(pass: &mut RenderPass) {
+ pass.base.commands.push(RenderCommand::PopDebugGroup);
+ }
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given `label`
+ /// is a valid null-terminated string.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_render_pass_insert_debug_marker(
+ pass: &mut RenderPass,
+ label: RawString,
+ color: u32,
+ ) {
+ let bytes = unsafe { ffi::CStr::from_ptr(label) }.to_bytes();
+ pass.base.string_data.extend_from_slice(bytes);
+
+ pass.base.commands.push(RenderCommand::InsertDebugMarker {
+ color,
+ len: bytes.len(),
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_write_timestamp(
+ pass: &mut RenderPass,
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ ) {
+ pass.base.commands.push(RenderCommand::WriteTimestamp {
+ query_set_id,
+ query_index,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_begin_occlusion_query(
+ pass: &mut RenderPass,
+ query_index: u32,
+ ) {
+ pass.base
+ .commands
+ .push(RenderCommand::BeginOcclusionQuery { query_index });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_end_occlusion_query(pass: &mut RenderPass) {
+ pass.base.commands.push(RenderCommand::EndOcclusionQuery);
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_begin_pipeline_statistics_query(
+ pass: &mut RenderPass,
+ query_set_id: id::QuerySetId,
+ query_index: u32,
+ ) {
+ pass.base
+ .commands
+ .push(RenderCommand::BeginPipelineStatisticsQuery {
+ query_set_id,
+ query_index,
+ });
+ }
+
+ #[no_mangle]
+ pub extern "C" fn wgpu_render_pass_end_pipeline_statistics_query(pass: &mut RenderPass) {
+ pass.base
+ .commands
+ .push(RenderCommand::EndPipelineStatisticsQuery);
+ }
+
+ /// # Safety
+ ///
+ /// This function is unsafe as there is no guarantee that the given pointer is
+ /// valid for `render_bundle_ids_length` elements.
+ #[no_mangle]
+ pub unsafe extern "C" fn wgpu_render_pass_execute_bundles(
+ pass: &mut RenderPass,
+ render_bundle_ids: *const id::RenderBundleId,
+ render_bundle_ids_length: usize,
+ ) {
+ for &bundle_id in
+ unsafe { slice::from_raw_parts(render_bundle_ids, render_bundle_ids_length) }
+ {
+ pass.base
+ .commands
+ .push(RenderCommand::ExecuteBundle(bundle_id));
+ }
+ pass.current_pipeline.reset();
+ pass.current_bind_groups.reset();
+ }
+}
diff --git a/third_party/rust/wgpu-core/src/command/transfer.rs b/third_party/rust/wgpu-core/src/command/transfer.rs
new file mode 100644
index 0000000000..0a952dfc84
--- /dev/null
+++ b/third_party/rust/wgpu-core/src/command/transfer.rs
@@ -0,0 +1,1229 @@
+#[cfg(feature = "trace")]
+use crate::device::trace::Command as TraceCommand;
+use crate::{
+ api_log,
+ command::{clear_texture, CommandBuffer, CommandEncoderError},
+ conv,
+ device::{Device, DeviceError, MissingDownlevelFlags},
+ error::{ErrorFormatter, PrettyError},
+ global::Global,
+ hal_api::HalApi,
+ id::{BufferId, CommandEncoderId, DeviceId, TextureId},
+ init_tracker::{
+ has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
+ TextureInitTrackerAction,
+ },
+ resource::{Resource, Texture, TextureErrorDimension},
+ track::{TextureSelector, Tracker},
+};
+
+use arrayvec::ArrayVec;
+use hal::CommandEncoder as _;
+use thiserror::Error;
+use wgt::{BufferAddress, BufferUsages, Extent3d, TextureUsages};
+
+use std::{iter, sync::Arc};
+
+use super::{memory_init::CommandBufferTextureMemoryActions, ClearError, CommandEncoder};
+
+pub type ImageCopyBuffer = wgt::ImageCopyBuffer<BufferId>;
+pub type ImageCopyTexture = wgt::ImageCopyTexture<TextureId>;
+pub type ImageCopyTextureTagged = wgt::ImageCopyTextureTagged<TextureId>;
+
+#[derive(Clone, Copy, Debug)]
+pub enum CopySide {
+ Source,
+ Destination,
+}
+
+/// Error encountered while attempting a data transfer.
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum TransferError {
+ #[error("Device {0:?} is invalid")]
+ InvalidDevice(DeviceId),
+ #[error("Buffer {0:?} is invalid or destroyed")]
+ InvalidBuffer(BufferId),
+ #[error("Texture {0:?} is invalid or destroyed")]
+ InvalidTexture(TextureId),
+ #[error("Source and destination cannot be the same buffer")]
+ SameSourceDestinationBuffer,
+ #[error("Source buffer/texture is missing the `COPY_SRC` usage flag")]
+ MissingCopySrcUsageFlag,
+ #[error("Destination buffer/texture is missing the `COPY_DST` usage flag")]
+ MissingCopyDstUsageFlag(Option<BufferId>, Option<TextureId>),
+ #[error("Destination texture is missing the `RENDER_ATTACHMENT` usage flag")]
+ MissingRenderAttachmentUsageFlag(TextureId),
+ #[error("Copy of {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}")]
+ BufferOverrun {
+ start_offset: BufferAddress,
+ end_offset: BufferAddress,
+ buffer_size: BufferAddress,
+ side: CopySide,
+ },
+ #[error("Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")]
+ TextureOverrun {
+ start_offset: u32,
+ end_offset: u32,
+ texture_size: u32,
+ dimension: TextureErrorDimension,
+ side: CopySide,
+ },
+ #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
+ InvalidTextureAspect {
+ format: wgt::TextureFormat,
+ aspect: wgt::TextureAspect,
+ },
+ #[error("Unable to select texture mip level {level} out of {total}")]
+ InvalidTextureMipLevel { level: u32, total: u32 },
+ #[error("Texture dimension must be 2D when copying from an external texture")]
+ InvalidDimensionExternal(TextureId),
+ #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
+ UnalignedBufferOffset(BufferAddress),
+ #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
+ UnalignedCopySize(BufferAddress),
+ #[error("Copy width is not a multiple of block width")]
+ UnalignedCopyWidth,
+ #[error("Copy height is not a multiple of block height")]
+ UnalignedCopyHeight,
+ #[error("Copy origin's x component is not a multiple of block width")]
+ UnalignedCopyOriginX,
+ #[error("Copy origin's y component is not a multiple of block height")]
+ UnalignedCopyOriginY,
+ #[error("Bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")]
+ UnalignedBytesPerRow,
+ #[error("Number of bytes per row needs to be specified since more than one row is copied")]
+ UnspecifiedBytesPerRow,
+ #[error("Number of rows per image needs to be specified since more than one image is copied")]
+ UnspecifiedRowsPerImage,
+ #[error("Number of bytes per row is less than the number of bytes in a complete row")]
+ InvalidBytesPerRow,
+ #[error("Image is 1D and the copy height and depth are not both set to 1")]
+ InvalidCopySize,
+ #[error("Number of rows per image is invalid")]
+ InvalidRowsPerImage,
+ #[error("Copy source aspects must refer to all aspects of the source texture format")]
+ CopySrcMissingAspects,
+ #[error(
+ "Copy destination aspects must refer to all aspects of the destination texture format"
+ )]
+ CopyDstMissingAspects,
+ #[error("Copy aspect must refer to a single aspect of texture format")]
+ CopyAspectNotOne,
+ #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
+ CopyFromForbiddenTextureFormat {
+ format: wgt::TextureFormat,
+ aspect: wgt::TextureAspect,
+ },
+ #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
+ CopyToForbiddenTextureFormat {
+ format: wgt::TextureFormat,
+ aspect: wgt::TextureAspect,
+ },
+ #[error(
+ "Copying to textures with format {0:?} is forbidden when copying from external texture"
+ )]
+ ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
+ #[error("The entire texture must be copied when copying from depth texture")]
+ InvalidDepthTextureExtent,
+ #[error(
+ "Source format ({src_format:?}) and destination format ({dst_format:?}) are not copy-compatible (they may only differ in srgb-ness)"
+ )]
+ TextureFormatsNotCopyCompatible {
+ src_format: wgt::TextureFormat,
+ dst_format: wgt::TextureFormat,
+ },
+ #[error(transparent)]
+ MemoryInitFailure(#[from] ClearError),
+ #[error("Cannot encode this copy because of a missing downelevel flag")]
+ MissingDownlevelFlags(#[from] MissingDownlevelFlags),
+ #[error("Source texture sample count must be 1, got {sample_count}")]
+ InvalidSampleCount { sample_count: u32 },
+ #[error("Requested mip level {requested} does no exist (count: {count})")]
+ InvalidMipLevel { requested: u32, count: u32 },
+}
+
+impl PrettyError for TransferError {
+ fn fmt_pretty(&self, fmt: &mut ErrorFormatter) {
+ fmt.error(self);
+ match *self {
+ Self::InvalidBuffer(id) => {
+ fmt.buffer_label(&id);
+ }
+ Self::InvalidTexture(id) => {
+ fmt.texture_label(&id);
+ }
+ // Self::MissingCopySrcUsageFlag(buf_opt, tex_opt) => {
+ // if let Some(buf) = buf_opt {
+ // let name = crate::gfx_select!(buf => global.buffer_label(buf));
+ // ret.push_str(&format_label_line("source", &name));
+ // }
+ // if let Some(tex) = tex_opt {
+ // let name = crate::gfx_select!(tex => global.texture_label(tex));
+ // ret.push_str(&format_label_line("source", &name));
+ // }
+ // }
+ Self::MissingCopyDstUsageFlag(buf_opt, tex_opt) => {
+ if let Some(buf) = buf_opt {
+ fmt.buffer_label_with_key(&buf, "destination");
+ }
+ if let Some(tex) = tex_opt {
+ fmt.texture_label_with_key(&tex, "destination");
+ }
+ }
+ _ => {}
+ };
+ }
+}
+/// Error encountered while attempting to do a copy on a command encoder.
+#[derive(Clone, Debug, Error)]
+#[non_exhaustive]
+pub enum CopyError {
+ #[error(transparent)]
+ Encoder(#[from] CommandEncoderError),
+ #[error("Copy error")]
+ Transfer(#[from] TransferError),
+}
+
+impl From<DeviceError> for CopyError {
+ fn from(err: DeviceError) -> Self {
+ CopyError::Encoder(CommandEncoderError::Device(err))
+ }
+}
+
+pub(crate) fn extract_texture_selector<A: HalApi>(
+ copy_texture: &ImageCopyTexture,
+ copy_size: &Extent3d,
+ texture: &Texture<A>,
+) -> Result<(TextureSelector, hal::TextureCopyBase), TransferError> {
+ let format = texture.desc.format;
+ let copy_aspect = hal::FormatAspects::new(format, copy_texture.aspect);
+ if copy_aspect.is_empty() {
+ return Err(TransferError::InvalidTextureAspect {
+ format,
+ aspect: copy_texture.aspect,
+ });
+ }
+
+ let (layers, origin_z) = match texture.desc.dimension {
+ wgt::TextureDimension::D1 => (0..1, 0),
+ wgt::TextureDimension::D2 => (
+ copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers,
+ 0,
+ ),
+ wgt::TextureDimension::D3 => (0..1, copy_texture.origin.z),
+ };
+ let base = hal::TextureCopyBase {
+ origin: wgt::Origin3d {
+ x: copy_texture.origin.x,
+ y: copy_texture.origin.y,
+ z: origin_z,
+ },
+ // this value will be incremented per copied layer
+ array_layer: layers.start,
+ mip_level: copy_texture.mip_level,
+ aspect: copy_aspect,
+ };
+ let selector = TextureSelector {
+ mips: copy_texture.mip_level..copy_texture.mip_level + 1,
+ layers,
+ };
+
+ Ok((selector, base))
+}
+
+/// WebGPU's [validating linear texture data][vltd] algorithm.
+///
+/// Copied with some modifications from WebGPU standard.
+///
+/// If successful, returns a pair `(bytes, stride)`, where:
+/// - `bytes` is the number of buffer bytes required for this copy, and
+/// - `stride` number of bytes between array layers.
+///
+/// [vltd]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-linear-texture-data
+pub(crate) fn validate_linear_texture_data(
+ layout: &wgt::ImageDataLayout,
+ format: wgt::TextureFormat,
+ aspect: wgt::TextureAspect,
+ buffer_size: BufferAddress,
+ buffer_side: CopySide,
+ copy_size: &Extent3d,
+ need_copy_aligned_rows: bool,
+) -> Result<(BufferAddress, BufferAddress), TransferError> {
+ // Convert all inputs to BufferAddress (u64) to avoid some of the overflow issues
+ // Note: u64 is not always enough to prevent overflow, especially when multiplying
+ // something with a potentially large depth value, so it is preferable to validate
+ // the copy size before calling this function (for example via `validate_texture_copy_range`).
+ let copy_width = copy_size.width as BufferAddress;
+ let copy_height = copy_size.height as BufferAddress;
+ let copy_depth = copy_size.depth_or_array_layers as BufferAddress;
+
+ let offset = layout.offset;
+
+ let block_size = format.block_copy_size(Some(aspect)).unwrap() as BufferAddress;
+ let (block_width, block_height) = format.block_dimensions();
+ let block_width = block_width as BufferAddress;
+ let block_height = block_height as BufferAddress;
+
+ if copy_width % block_width != 0 {
+ return Err(TransferError::UnalignedCopyWidth);
+ }
+ if copy_height % block_height != 0 {
+ return Err(TransferError::UnalignedCopyHeight);
+ }
+
+ let width_in_blocks = copy_width / block_width;
+ let height_in_blocks = copy_height / block_height;
+
+ let bytes_in_last_row = width_in_blocks * block_size;
+
+ let bytes_per_row = if let Some(bytes_per_row) = layout.bytes_per_row {
+ let bytes_per_row = bytes_per_row as BufferAddress;
+ if bytes_per_row < bytes_in_last_row {
+ return Err(TransferError::InvalidBytesPerRow);
+ }
+ bytes_per_row
+ } else {
+ if copy_depth > 1 || height_in_blocks > 1 {
+ return Err(TransferError::UnspecifiedBytesPerRow);
+ }
+ 0
+ };
+ let block_rows_per_image = if let Some(rows_per_image) = layout.rows_per_image {
+ let rows_per_image = rows_per_image as BufferAddress;
+ if rows_per_image < height_in_blocks {
+ return Err(TransferError::InvalidRowsPerImage);
+ }
+ rows_per_image
+ } else {
+ if copy_depth > 1 {
+ return Err(TransferError::UnspecifiedRowsPerImage);
+ }
+ 0
+ };
+
+ if need_copy_aligned_rows {
+ let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT as BufferAddress;
+
+ let mut offset_alignment = block_size;
+ if format.is_depth_stencil_format() {
+ offset_alignment = 4
+ }
+ if offset % offset_alignment != 0 {
+ return Err(TransferError::UnalignedBufferOffset(offset));
+ }
+
+ if bytes_per_row % bytes_per_row_alignment != 0 {
+ return Err(TransferError::UnalignedBytesPerRow);
+ }
+ }
+
+ let bytes_per_image = bytes_per_row * block_rows_per_image;
+
+ let required_bytes_in_copy = if copy_depth == 0 {
+ 0
+ } else {
+ let mut required_bytes_in_copy = bytes_per_image * (copy_depth - 1);
+ if height_in_blocks > 0 {
+ required_bytes_in_copy += bytes_per_row * (height_in_blocks - 1) + bytes_in_last_row;
+ }
+ required_bytes_in_copy
+ };
+
+ if offset + required_bytes_in_copy > buffer_size {
+ return Err(TransferError::BufferOverrun {
+ start_offset: offset,
+ end_offset: offset + required_bytes_in_copy,
+ buffer_size,
+ side: buffer_side,
+ });
+ }
+
+ Ok((required_bytes_in_copy, bytes_per_image))
+}
+
+/// WebGPU's [validating texture copy range][vtcr] algorithm.
+///
+/// Copied with minor modifications from WebGPU standard.
+///
+/// Returns the HAL copy extent and the layer count.
+///
+/// [vtcr]: https://gpuweb.github.io/gpuweb/#validating-texture-copy-range
+pub(crate) fn validate_texture_copy_range(
+ texture_copy_view: &ImageCopyTexture,
+ desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
+ texture_side: CopySide,
+ copy_size: &Extent3d,
+) -> Result<(hal::CopyExtent, u32), TransferError> {
+ let (block_width, block_height) = desc.format.block_dimensions();
+
+ let extent_virtual = desc.mip_level_size(texture_copy_view.mip_level).ok_or(
+ TransferError::InvalidTextureMipLevel {
+ level: texture_copy_view.mip_level,
+ total: desc.mip_level_count,
+ },
+ )?;
+ // physical size can be larger than the virtual
+ let extent = extent_virtual.physical_size(desc.format);
+
+ if desc.format.is_depth_stencil_format() && *copy_size != extent {
+ return Err(TransferError::InvalidDepthTextureExtent);
+ }
+
+ /// Return `Ok` if a run `size` texels long starting at `start_offset` falls
+ /// entirely within `texture_size`. Otherwise, return an appropriate a`Err`.
+ fn check_dimension(
+ dimension: TextureErrorDimension,
+ side: CopySide,
+ start_offset: u32,
+ size: u32,
+ texture_size: u32,
+ ) -> Result<(), TransferError> {
+ // Avoid underflow in the subtraction by checking start_offset against
+ // texture_size first.
+ if start_offset <= texture_size && size <= texture_size - start_offset {
+ Ok(())
+ } else {
+ Err(TransferError::TextureOverrun {
+ start_offset,
+ end_offset: start_offset.wrapping_add(size),
+ texture_size,
+ dimension,
+ side,
+ })
+ }
+ }
+
+ check_dimension(
+ TextureErrorDimension::X,
+ texture_side,
+ texture_copy_view.origin.x,
+ copy_size.width,
+ extent.width,
+ )?;
+ check_dimension(
+ TextureErrorDimension::Y,
+ texture_side,
+ texture_copy_view.origin.y,
+ copy_size.height,
+ extent.height,
+ )?;
+ check_dimension(
+ TextureErrorDimension::Z,
+ texture_side,
+ texture_copy_view.origin.z,
+ copy_size.depth_or_array_layers,
+ extent.depth_or_array_layers,
+ )?;
+
+ if texture_copy_view.origin.x % block_width != 0 {
+ return Err(TransferError::UnalignedCopyOriginX);
+ }
+ if texture_copy_view.origin.y % block_height != 0 {
+ return Err(TransferError::UnalignedCopyOriginY);
+ }
+ if copy_size.width % block_width != 0 {
+ return Err(TransferError::UnalignedCopyWidth);
+ }
+ if copy_size.height % block_height != 0 {
+ return Err(TransferError::UnalignedCopyHeight);
+ }
+
+ let (depth, array_layer_count) = match desc.dimension {
+ wgt::TextureDimension::D1 => (1, 1),
+ wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
+ wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
+ };
+
+ let copy_extent = hal::CopyExtent {
+ width: copy_size.width,
+ height: copy_size.height,
+ depth,
+ };
+ Ok((copy_extent, array_layer_count))
+}
+
+fn handle_texture_init<A: HalApi>(
+ init_kind: MemoryInitKind,
+ encoder: &mut CommandEncoder<A>,
+ trackers: &mut Tracker<A>,
+ texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>,
+ device: &Device<A>,
+ copy_texture: &ImageCopyTexture,
+ copy_size: &Extent3d,
+ texture: &Arc<Texture<A>>,
+) -> Result<(), ClearError> {
+ let init_action = TextureInitTrackerAction {
+ texture: texture.clone(),
+ range: TextureInitRange {
+ mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
+ layer_range: copy_texture.origin.z
+ ..(copy_texture.origin.z + copy_size.depth_or_array_layers),
+ },
+ kind: init_kind,
+ };
+
+ // Register the init action.
+ let immediate_inits = texture_memory_actions.register_init_action(&{ init_action });
+
+ // In rare cases we may need to insert an init operation immediately onto the command buffer.
+ if !immediate_inits.is_empty() {
+ let cmd_buf_raw = encoder.open()?;
+ for init in immediate_inits {
+ clear_texture(
+ &init.texture,
+ TextureInitRange {
+ mip_range: init.mip_level..(init.mip_level + 1),
+ layer_range: init.layer..(init.layer + 1),
+ },
+ cmd_buf_raw,
+ &mut trackers.textures,
+ &device.alignments,
+ device.zero_buffer.as_ref().unwrap(),
+ )?;
+ }
+ }
+
+ Ok(())
+}
+
+/// Prepare a transfer's source texture.
+///
+/// Ensure the source texture of a transfer is in the right initialization
+/// state, and record the state for after the transfer operation.
+fn handle_src_texture_init<A: HalApi>(
+ encoder: &mut CommandEncoder<A>,
+ trackers: &mut Tracker<A>,
+ texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>,
+ device: &Device<A>,
+ source: &ImageCopyTexture,
+ copy_size: &Extent3d,
+ texture: &Arc<Texture<A>>,
+) -> Result<(), TransferError> {
+ handle_texture_init(
+ MemoryInitKind::NeedsInitializedMemory,
+ encoder,
+ trackers,
+ texture_memory_actions,
+ device,
+ source,
+ copy_size,
+ texture,
+ )?;
+ Ok(())
+}
+
+/// Prepare a transfer's destination texture.
+///
+/// Ensure the destination texture of a transfer is in the right initialization
+/// state, and record the state for after the transfer operation.
+fn handle_dst_texture_init<A: HalApi>(
+ encoder: &mut CommandEncoder<A>,
+ trackers: &mut Tracker<A>,
+ texture_memory_actions: &mut CommandBufferTextureMemoryActions<A>,
+ device: &Device<A>,
+ destination: &ImageCopyTexture,
+ copy_size: &Extent3d,
+ texture: &Arc<Texture<A>>,
+) -> Result<(), TransferError> {
+ // Attention: If we don't write full texture subresources, we need to a full
+ // clear first since we don't track subrects. This means that in rare cases
+ // even a *destination* texture of a transfer may need an immediate texture
+ // init.
+ let dst_init_kind = if has_copy_partial_init_tracker_coverage(
+ copy_size,
+ destination.mip_level,
+ &texture.desc,
+ ) {
+ MemoryInitKind::NeedsInitializedMemory
+ } else {
+ MemoryInitKind::ImplicitlyInitialized
+ };
+
+ handle_texture_init(
+ dst_init_kind,
+ encoder,
+ trackers,
+ texture_memory_actions,
+ device,
+ destination,
+ copy_size,
+ texture,
+ )?;
+ Ok(())
+}
+
+impl Global {
+ pub fn command_encoder_copy_buffer_to_buffer<A: HalApi>(
+ &self,
+ command_encoder_id: CommandEncoderId,
+ source: BufferId,
+ source_offset: BufferAddress,
+ destination: BufferId,
+ destination_offset: BufferAddress,
+ size: BufferAddress,
+ ) -> Result<(), CopyError> {
+ profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
+ api_log!(
+ "CommandEncoder::copy_buffer_to_buffer {source:?} -> {destination:?} {size:?}bytes"
+ );
+
+ if source == destination {
+ return Err(TransferError::SameSourceDestinationBuffer.into());
+ }
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?;
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ let device = &cmd_buf.device;
+ if !device.is_valid() {
+ return Err(TransferError::InvalidDevice(cmd_buf.device.as_info().id()).into());
+ }
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::CopyBufferToBuffer {
+ src: source,
+ src_offset: source_offset,
+ dst: destination,
+ dst_offset: destination_offset,
+ size,
+ });
+ }
+
+ let snatch_guard = device.snatchable_lock.read();
+
+ let (src_buffer, src_pending) = {
+ let buffer_guard = hub.buffers.read();
+ let src_buffer = buffer_guard
+ .get(source)
+ .map_err(|_| TransferError::InvalidBuffer(source))?;
+ cmd_buf_data
+ .trackers
+ .buffers
+ .set_single(src_buffer, hal::BufferUses::COPY_SRC)
+ .ok_or(TransferError::InvalidBuffer(source))?
+ };
+ let src_raw = src_buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(TransferError::InvalidBuffer(source))?;
+ if !src_buffer.usage.contains(BufferUsages::COPY_SRC) {
+ return Err(TransferError::MissingCopySrcUsageFlag.into());
+ }
+ // expecting only a single barrier
+ let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
+
+ let (dst_buffer, dst_pending) = {
+ let buffer_guard = hub.buffers.read();
+ let dst_buffer = buffer_guard
+ .get(destination)
+ .map_err(|_| TransferError::InvalidBuffer(destination))?;
+ cmd_buf_data
+ .trackers
+ .buffers
+ .set_single(dst_buffer, hal::BufferUses::COPY_DST)
+ .ok_or(TransferError::InvalidBuffer(destination))?
+ };
+ let dst_raw = dst_buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(TransferError::InvalidBuffer(destination))?;
+ if !dst_buffer.usage.contains(BufferUsages::COPY_DST) {
+ return Err(TransferError::MissingCopyDstUsageFlag(Some(destination), None).into());
+ }
+ let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
+
+ if size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
+ return Err(TransferError::UnalignedCopySize(size).into());
+ }
+ if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
+ return Err(TransferError::UnalignedBufferOffset(source_offset).into());
+ }
+ if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
+ return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
+ }
+ if !device
+ .downlevel
+ .flags
+ .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
+ && (src_buffer.usage.contains(wgt::BufferUsages::INDEX)
+ || dst_buffer.usage.contains(wgt::BufferUsages::INDEX))
+ {
+ let forbidden_usages = wgt::BufferUsages::VERTEX
+ | wgt::BufferUsages::UNIFORM
+ | wgt::BufferUsages::INDIRECT
+ | wgt::BufferUsages::STORAGE;
+ if src_buffer.usage.intersects(forbidden_usages)
+ || dst_buffer.usage.intersects(forbidden_usages)
+ {
+ return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
+ wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
+ ))
+ .into());
+ }
+ }
+
+ let source_end_offset = source_offset + size;
+ let destination_end_offset = destination_offset + size;
+ if source_end_offset > src_buffer.size {
+ return Err(TransferError::BufferOverrun {
+ start_offset: source_offset,
+ end_offset: source_end_offset,
+ buffer_size: src_buffer.size,
+ side: CopySide::Source,
+ }
+ .into());
+ }
+ if destination_end_offset > dst_buffer.size {
+ return Err(TransferError::BufferOverrun {
+ start_offset: destination_offset,
+ end_offset: destination_end_offset,
+ buffer_size: dst_buffer.size,
+ side: CopySide::Destination,
+ }
+ .into());
+ }
+
+ if size == 0 {
+ log::trace!("Ignoring copy_buffer_to_buffer of size 0");
+ return Ok(());
+ }
+
+ // Make sure source is initialized memory and mark dest as initialized.
+ cmd_buf_data.buffer_memory_init_actions.extend(
+ dst_buffer.initialization_status.read().create_action(
+ &dst_buffer,
+ destination_offset..(destination_offset + size),
+ MemoryInitKind::ImplicitlyInitialized,
+ ),
+ );
+ cmd_buf_data.buffer_memory_init_actions.extend(
+ src_buffer.initialization_status.read().create_action(
+ &src_buffer,
+ source_offset..(source_offset + size),
+ MemoryInitKind::NeedsInitializedMemory,
+ ),
+ );
+
+ let region = hal::BufferCopy {
+ src_offset: source_offset,
+ dst_offset: destination_offset,
+ size: wgt::BufferSize::new(size).unwrap(),
+ };
+ let cmd_buf_raw = cmd_buf_data.encoder.open()?;
+ unsafe {
+ cmd_buf_raw.transition_buffers(src_barrier.into_iter().chain(dst_barrier));
+ cmd_buf_raw.copy_buffer_to_buffer(src_raw, dst_raw, iter::once(region));
+ }
+ Ok(())
+ }
+
+ pub fn command_encoder_copy_buffer_to_texture<A: HalApi>(
+ &self,
+ command_encoder_id: CommandEncoderId,
+ source: &ImageCopyBuffer,
+ destination: &ImageCopyTexture,
+ copy_size: &Extent3d,
+ ) -> Result<(), CopyError> {
+ profiling::scope!("CommandEncoder::copy_buffer_to_texture");
+ api_log!(
+ "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
+ source.buffer,
+ destination.texture
+ );
+
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?;
+ let device = &cmd_buf.device;
+ if !device.is_valid() {
+ return Err(TransferError::InvalidDevice(cmd_buf.device.as_info().id()).into());
+ }
+
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::CopyBufferToTexture {
+ src: *source,
+ dst: *destination,
+ size: *copy_size,
+ });
+ }
+
+ let encoder = &mut cmd_buf_data.encoder;
+ let tracker = &mut cmd_buf_data.trackers;
+ let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions;
+ let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions;
+
+ if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
+ log::trace!("Ignoring copy_buffer_to_texture of size 0");
+ return Ok(());
+ }
+
+ let dst_texture = hub
+ .textures
+ .get(destination.texture)
+ .map_err(|_| TransferError::InvalidTexture(destination.texture))?;
+
+ let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
+ destination,
+ &dst_texture.desc,
+ CopySide::Destination,
+ copy_size,
+ )?;
+
+ let (dst_range, dst_base) = extract_texture_selector(destination, copy_size, &dst_texture)?;
+
+ // Handle texture init *before* dealing with barrier transitions so we
+ // have an easier time inserting "immediate-inits" that may be required
+ // by prior discards in rare cases.
+ handle_dst_texture_init(
+ encoder,
+ tracker,
+ texture_memory_actions,
+ device,
+ destination,
+ copy_size,
+ &dst_texture,
+ )?;
+
+ let snatch_guard = device.snatchable_lock.read();
+
+ let (src_buffer, src_pending) = {
+ let buffer_guard = hub.buffers.read();
+ let src_buffer = buffer_guard
+ .get(source.buffer)
+ .map_err(|_| TransferError::InvalidBuffer(source.buffer))?;
+ tracker
+ .buffers
+ .set_single(src_buffer, hal::BufferUses::COPY_SRC)
+ .ok_or(TransferError::InvalidBuffer(source.buffer))?
+ };
+ let src_raw = src_buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(TransferError::InvalidBuffer(source.buffer))?;
+ if !src_buffer.usage.contains(BufferUsages::COPY_SRC) {
+ return Err(TransferError::MissingCopySrcUsageFlag.into());
+ }
+ let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
+
+ let dst_pending = tracker
+ .textures
+ .set_single(&dst_texture, dst_range, hal::TextureUses::COPY_DST)
+ .ok_or(TransferError::InvalidTexture(destination.texture))?;
+ let dst_raw = dst_texture
+ .raw(&snatch_guard)
+ .ok_or(TransferError::InvalidTexture(destination.texture))?;
+ if !dst_texture.desc.usage.contains(TextureUsages::COPY_DST) {
+ return Err(
+ TransferError::MissingCopyDstUsageFlag(None, Some(destination.texture)).into(),
+ );
+ }
+ let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_raw));
+
+ if !dst_base.aspect.is_one() {
+ return Err(TransferError::CopyAspectNotOne.into());
+ }
+
+ if !conv::is_valid_copy_dst_texture_format(dst_texture.desc.format, destination.aspect) {
+ return Err(TransferError::CopyToForbiddenTextureFormat {
+ format: dst_texture.desc.format,
+ aspect: destination.aspect,
+ }
+ .into());
+ }
+
+ let (required_buffer_bytes_in_copy, bytes_per_array_layer) = validate_linear_texture_data(
+ &source.layout,
+ dst_texture.desc.format,
+ destination.aspect,
+ src_buffer.size,
+ CopySide::Source,
+ copy_size,
+ true,
+ )?;
+
+ if dst_texture.desc.format.is_depth_stencil_format() {
+ device
+ .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
+ .map_err(TransferError::from)?;
+ }
+
+ buffer_memory_init_actions.extend(src_buffer.initialization_status.read().create_action(
+ &src_buffer,
+ source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy),
+ MemoryInitKind::NeedsInitializedMemory,
+ ));
+
+ let regions = (0..array_layer_count).map(|rel_array_layer| {
+ let mut texture_base = dst_base.clone();
+ texture_base.array_layer += rel_array_layer;
+ let mut buffer_layout = source.layout;
+ buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
+ hal::BufferTextureCopy {
+ buffer_layout,
+ texture_base,
+ size: hal_copy_size,
+ }
+ });
+
+ let cmd_buf_raw = encoder.open()?;
+ unsafe {
+ cmd_buf_raw.transition_textures(dst_barrier.into_iter());
+ cmd_buf_raw.transition_buffers(src_barrier.into_iter());
+ cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, regions);
+ }
+ Ok(())
+ }
+
+ pub fn command_encoder_copy_texture_to_buffer<A: HalApi>(
+ &self,
+ command_encoder_id: CommandEncoderId,
+ source: &ImageCopyTexture,
+ destination: &ImageCopyBuffer,
+ copy_size: &Extent3d,
+ ) -> Result<(), CopyError> {
+ profiling::scope!("CommandEncoder::copy_texture_to_buffer");
+ api_log!(
+ "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
+ source.texture,
+ destination.buffer
+ );
+
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?;
+ let device = &cmd_buf.device;
+ if !device.is_valid() {
+ return Err(TransferError::InvalidDevice(cmd_buf.device.as_info().id()).into());
+ }
+
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::CopyTextureToBuffer {
+ src: *source,
+ dst: *destination,
+ size: *copy_size,
+ });
+ }
+ let encoder = &mut cmd_buf_data.encoder;
+ let tracker = &mut cmd_buf_data.trackers;
+ let buffer_memory_init_actions = &mut cmd_buf_data.buffer_memory_init_actions;
+ let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions;
+
+ if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
+ log::trace!("Ignoring copy_texture_to_buffer of size 0");
+ return Ok(());
+ }
+
+ let src_texture = hub
+ .textures
+ .get(source.texture)
+ .map_err(|_| TransferError::InvalidTexture(source.texture))?;
+
+ let (hal_copy_size, array_layer_count) =
+ validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
+
+ let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?;
+
+ // Handle texture init *before* dealing with barrier transitions so we
+ // have an easier time inserting "immediate-inits" that may be required
+ // by prior discards in rare cases.
+ handle_src_texture_init(
+ encoder,
+ tracker,
+ texture_memory_actions,
+ device,
+ source,
+ copy_size,
+ &src_texture,
+ )?;
+
+ let snatch_guard = device.snatchable_lock.read();
+
+ let src_pending = tracker
+ .textures
+ .set_single(&src_texture, src_range, hal::TextureUses::COPY_SRC)
+ .ok_or(TransferError::InvalidTexture(source.texture))?;
+ let src_raw = src_texture
+ .raw(&snatch_guard)
+ .ok_or(TransferError::InvalidTexture(source.texture))?;
+ if !src_texture.desc.usage.contains(TextureUsages::COPY_SRC) {
+ return Err(TransferError::MissingCopySrcUsageFlag.into());
+ }
+ if src_texture.desc.sample_count != 1 {
+ return Err(TransferError::InvalidSampleCount {
+ sample_count: src_texture.desc.sample_count,
+ }
+ .into());
+ }
+ if source.mip_level >= src_texture.desc.mip_level_count {
+ return Err(TransferError::InvalidMipLevel {
+ requested: source.mip_level,
+ count: src_texture.desc.mip_level_count,
+ }
+ .into());
+ }
+ let src_barrier = src_pending.map(|pending| pending.into_hal(src_raw));
+
+ let (dst_buffer, dst_pending) = {
+ let buffer_guard = hub.buffers.read();
+ let dst_buffer = buffer_guard
+ .get(destination.buffer)
+ .map_err(|_| TransferError::InvalidBuffer(destination.buffer))?;
+ tracker
+ .buffers
+ .set_single(dst_buffer, hal::BufferUses::COPY_DST)
+ .ok_or(TransferError::InvalidBuffer(destination.buffer))?
+ };
+ let dst_raw = dst_buffer
+ .raw
+ .get(&snatch_guard)
+ .ok_or(TransferError::InvalidBuffer(destination.buffer))?;
+ if !dst_buffer.usage.contains(BufferUsages::COPY_DST) {
+ return Err(
+ TransferError::MissingCopyDstUsageFlag(Some(destination.buffer), None).into(),
+ );
+ }
+ let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
+
+ if !src_base.aspect.is_one() {
+ return Err(TransferError::CopyAspectNotOne.into());
+ }
+
+ if !conv::is_valid_copy_src_texture_format(src_texture.desc.format, source.aspect) {
+ return Err(TransferError::CopyFromForbiddenTextureFormat {
+ format: src_texture.desc.format,
+ aspect: source.aspect,
+ }
+ .into());
+ }
+
+ let (required_buffer_bytes_in_copy, bytes_per_array_layer) = validate_linear_texture_data(
+ &destination.layout,
+ src_texture.desc.format,
+ source.aspect,
+ dst_buffer.size,
+ CopySide::Destination,
+ copy_size,
+ true,
+ )?;
+
+ if src_texture.desc.format.is_depth_stencil_format() {
+ device
+ .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
+ .map_err(TransferError::from)?;
+ }
+
+ buffer_memory_init_actions.extend(dst_buffer.initialization_status.read().create_action(
+ &dst_buffer,
+ destination.layout.offset..(destination.layout.offset + required_buffer_bytes_in_copy),
+ MemoryInitKind::ImplicitlyInitialized,
+ ));
+
+ let regions = (0..array_layer_count).map(|rel_array_layer| {
+ let mut texture_base = src_base.clone();
+ texture_base.array_layer += rel_array_layer;
+ let mut buffer_layout = destination.layout;
+ buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
+ hal::BufferTextureCopy {
+ buffer_layout,
+ texture_base,
+ size: hal_copy_size,
+ }
+ });
+ let cmd_buf_raw = encoder.open()?;
+ unsafe {
+ cmd_buf_raw.transition_buffers(dst_barrier.into_iter());
+ cmd_buf_raw.transition_textures(src_barrier.into_iter());
+ cmd_buf_raw.copy_texture_to_buffer(
+ src_raw,
+ hal::TextureUses::COPY_SRC,
+ dst_raw,
+ regions,
+ );
+ }
+ Ok(())
+ }
+
+ pub fn command_encoder_copy_texture_to_texture<A: HalApi>(
+ &self,
+ command_encoder_id: CommandEncoderId,
+ source: &ImageCopyTexture,
+ destination: &ImageCopyTexture,
+ copy_size: &Extent3d,
+ ) -> Result<(), CopyError> {
+ profiling::scope!("CommandEncoder::copy_texture_to_texture");
+ api_log!(
+ "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
+ source.texture,
+ destination.texture
+ );
+
+ let hub = A::hub(self);
+
+ let cmd_buf = CommandBuffer::get_encoder(hub, command_encoder_id)?;
+ let device = &cmd_buf.device;
+ if !device.is_valid() {
+ return Err(TransferError::InvalidDevice(cmd_buf.device.as_info().id()).into());
+ }
+
+ let snatch_guard = device.snatchable_lock.read();
+
+ let mut cmd_buf_data = cmd_buf.data.lock();
+ let cmd_buf_data = cmd_buf_data.as_mut().unwrap();
+
+ #[cfg(feature = "trace")]
+ if let Some(ref mut list) = cmd_buf_data.commands {
+ list.push(TraceCommand::CopyTextureToTexture {
+ src: *source,
+ dst: *destination,
+ size: *copy_size,
+ });
+ }
+ let encoder = &mut cmd_buf_data.encoder;
+ let tracker = &mut cmd_buf_data.trackers;
+ let texture_memory_actions = &mut cmd_buf_data.texture_memory_actions;
+
+ if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
+ log::trace!("Ignoring copy_texture_to_texture of size 0");
+ return Ok(());
+ }
+
+ let src_texture = hub
+ .textures
+ .get(source.texture)
+ .map_err(|_| TransferError::InvalidTexture(source.texture))?;
+ let dst_texture = hub
+ .textures
+ .get(destination.texture)
+ .map_err(|_| TransferError::InvalidTexture(source.texture))?;
+
+ // src and dst texture format must be copy-compatible
+ // https://gpuweb.github.io/gpuweb/#copy-compatible
+ if src_texture.desc.format.remove_srgb_suffix()
+ != dst_texture.desc.format.remove_srgb_suffix()
+ {
+ return Err(TransferError::TextureFormatsNotCopyCompatible {
+ src_format: src_texture.desc.format,
+ dst_format: dst_texture.desc.format,
+ }
+ .into());
+ }
+
+ let (src_copy_size, array_layer_count) =
+ validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
+ let (dst_copy_size, _) = validate_texture_copy_range(
+ destination,
+ &dst_texture.desc,
+ CopySide::Destination,
+ copy_size,
+ )?;
+
+ let (src_range, src_tex_base) = extract_texture_selector(source, copy_size, &src_texture)?;
+ let (dst_range, dst_tex_base) =
+ extract_texture_selector(destination, copy_size, &dst_texture)?;
+ let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
+ let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
+ if src_tex_base.aspect != src_texture_aspects {
+ return Err(TransferError::CopySrcMissingAspects.into());
+ }
+ if dst_tex_base.aspect != dst_texture_aspects {
+ return Err(TransferError::CopyDstMissingAspects.into());
+ }
+
+ // Handle texture init *before* dealing with barrier transitions so we
+ // have an easier time inserting "immediate-inits" that may be required
+ // by prior discards in rare cases.
+ handle_src_texture_init(
+ encoder,
+ tracker,
+ texture_memory_actions,
+ device,
+ source,
+ copy_size,
+ &src_texture,
+ )?;
+ handle_dst_texture_init(
+ encoder,
+ tracker,
+ texture_memory_actions,
+ device,
+ destination,
+ copy_size,
+ &dst_texture,
+ )?;
+
+ let src_pending = cmd_buf_data
+ .trackers
+ .textures
+ .set_single(&src_texture, src_range, hal::TextureUses::COPY_SRC)
+ .ok_or(TransferError::InvalidTexture(source.texture))?;
+ let src_raw = src_texture
+ .raw(&snatch_guard)
+ .ok_or(TransferError::InvalidTexture(source.texture))?;
+ if !src_texture.desc.usage.contains(TextureUsages::COPY_SRC) {
+ return Err(TransferError::MissingCopySrcUsageFlag.into());
+ }
+
+ //TODO: try to avoid this the collection. It's needed because both
+ // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably.
+ let mut barriers: ArrayVec<_, 2> = src_pending
+ .map(|pending| pending.into_hal(src_raw))
+ .collect();
+
+ let dst_pending = cmd_buf_data
+ .trackers
+ .textures
+ .set_single(&dst_texture, dst_range, hal::TextureUses::COPY_DST)
+ .ok_or(TransferError::InvalidTexture(destination.texture))?;
+ let dst_raw = dst_texture
+ .raw(&snatch_guard)
+ .ok_or(TransferError::InvalidTexture(destination.texture))?;
+ if !dst_texture.desc.usage.contains(TextureUsages::COPY_DST) {
+ return Err(
+ TransferError::MissingCopyDstUsageFlag(None, Some(destination.texture)).into(),
+ );
+ }
+
+ barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
+
+ let hal_copy_size = hal::CopyExtent {
+ width: src_copy_size.width.min(dst_copy_size.width),
+ height: src_copy_size.height.min(dst_copy_size.height),
+ depth: src_copy_size.depth.min(dst_copy_size.depth),
+ };
+ let regions = (0..array_layer_count).map(|rel_array_layer| {
+ let mut src_base = src_tex_base.clone();
+ let mut dst_base = dst_tex_base.clone();
+ src_base.array_layer += rel_array_layer;
+ dst_base.array_layer += rel_array_layer;
+ hal::TextureCopy {
+ src_base,
+ dst_base,
+ size: hal_copy_size,
+ }
+ });
+ let cmd_buf_raw = cmd_buf_data.encoder.open()?;
+ unsafe {
+ cmd_buf_raw.transition_textures(barriers.into_iter());
+ cmd_buf_raw.copy_texture_to_texture(
+ src_raw,
+ hal::TextureUses::COPY_SRC,
+ dst_raw,
+ regions,
+ );
+ }
+
+ Ok(())
+ }
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-08-23 08:39:22 +0000