diff options
Diffstat (limited to 'gfx/wgpu/wgpu-core/src/command')
| -rw-r--r-- | gfx/wgpu/wgpu-core/src/command/allocator.rs | 268 | ||||
| -rw-r--r-- | gfx/wgpu/wgpu-core/src/command/bind.rs | 295 | ||||
| -rw-r--r-- | gfx/wgpu/wgpu-core/src/command/bundle.rs | 1230 | ||||
| -rw-r--r-- | gfx/wgpu/wgpu-core/src/command/compute.rs | 657 | ||||
| -rw-r--r-- | gfx/wgpu/wgpu-core/src/command/draw.rs | 180 | ||||
| -rw-r--r-- | gfx/wgpu/wgpu-core/src/command/mod.rs | 362 | ||||
| -rw-r--r-- | gfx/wgpu/wgpu-core/src/command/render.rs | 2078 | ||||
| -rw-r--r-- | gfx/wgpu/wgpu-core/src/command/transfer.rs | 789 |
8 files changed, 5859 insertions, 0 deletions
diff --git a/gfx/wgpu/wgpu-core/src/command/allocator.rs b/gfx/wgpu/wgpu-core/src/command/allocator.rs new file mode 100644 index 0000000000..cfaa6258c2 --- /dev/null +++ b/gfx/wgpu/wgpu-core/src/command/allocator.rs @@ -0,0 +1,268 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use super::CommandBuffer; +use crate::{ + device::DeviceError, hub::GfxBackend, id::DeviceId, track::TrackerSet, FastHashMap, + PrivateFeatures, Stored, SubmissionIndex, +}; + +#[cfg(debug_assertions)] +use crate::LabelHelpers; + +use hal::{command::CommandBuffer as _, device::Device as _, pool::CommandPool as _}; +use parking_lot::Mutex; +use thiserror::Error; + +use std::thread; + +const GROW_AMOUNT: usize = 20; + +#[derive(Debug)] +struct CommandPool<B: hal::Backend> { + raw: B::CommandPool, + total: usize, + available: Vec<B::CommandBuffer>, + pending: Vec<(B::CommandBuffer, SubmissionIndex)>, +} + +impl<B: hal::Backend> CommandPool<B> { + fn maintain(&mut self, last_done_index: SubmissionIndex) { + for i in (0..self.pending.len()).rev() { + if self.pending[i].1 <= last_done_index { + let (cmd_buf, index) = self.pending.swap_remove(i); + tracing::trace!( + "recycling cmdbuf submitted in {} when {} is last done", + index, + last_done_index, + ); + self.recycle(cmd_buf); + } + } + } + + fn recycle(&mut self, mut raw: B::CommandBuffer) { + unsafe { + raw.reset(false); + } + self.available.push(raw); + } + + fn allocate(&mut self) -> B::CommandBuffer { + if self.available.is_empty() { + self.total += GROW_AMOUNT; + unsafe { + self.raw.allocate( + GROW_AMOUNT, + hal::command::Level::Primary, + &mut self.available, + ) + }; + } + self.available.pop().unwrap() + } +} + +#[derive(Debug)] +struct Inner<B: hal::Backend> { + pools: FastHashMap<thread::ThreadId, CommandPool<B>>, +} + +#[derive(Debug)] +pub struct CommandAllocator<B: hal::Backend> { + queue_family: hal::queue::QueueFamilyId, + internal_thread_id: thread::ThreadId, + inner: Mutex<Inner<B>>, +} + +impl<B: GfxBackend> CommandAllocator<B> { + pub(crate) fn allocate( + &self, + device_id: Stored<DeviceId>, + device: &B::Device, + limits: wgt::Limits, + private_features: PrivateFeatures, + label: &crate::Label, + #[cfg(feature = "trace")] enable_tracing: bool, + ) -> Result<CommandBuffer<B>, CommandAllocatorError> { + //debug_assert_eq!(device_id.backend(), B::VARIANT); + let _ = label; // silence warning on release + let thread_id = thread::current().id(); + let mut inner = self.inner.lock(); + + use std::collections::hash_map::Entry; + let pool = match inner.pools.entry(thread_id) { + Entry::Occupied(e) => e.into_mut(), + Entry::Vacant(e) => { + tracing::info!("Starting on thread {:?}", thread_id); + let raw = unsafe { + device + .create_command_pool( + self.queue_family, + hal::pool::CommandPoolCreateFlags::RESET_INDIVIDUAL, + ) + .or(Err(DeviceError::OutOfMemory))? + }; + let pool = CommandPool { + raw, + total: 0, + available: Vec::new(), + pending: Vec::new(), + }; + e.insert(pool) + } + }; + + let init = pool.allocate(); + + Ok(CommandBuffer { + raw: vec![init], + is_recording: true, + recorded_thread_id: thread_id, + device_id, + trackers: TrackerSet::new(B::VARIANT), + used_swap_chain: None, + limits, + private_features, + #[cfg(feature = "trace")] + commands: if enable_tracing { + Some(Vec::new()) + } else { + None + }, + #[cfg(debug_assertions)] + label: label.to_string_or_default(), + }) + } +} + +impl<B: hal::Backend> CommandAllocator<B> { + pub fn new( + queue_family: hal::queue::QueueFamilyId, + device: &B::Device, + ) -> Result<Self, CommandAllocatorError> { + let internal_thread_id = thread::current().id(); + tracing::info!("Starting on (internal) thread {:?}", internal_thread_id); + let mut pools = FastHashMap::default(); + pools.insert( + internal_thread_id, + CommandPool { + raw: unsafe { + device + .create_command_pool( + queue_family, + hal::pool::CommandPoolCreateFlags::RESET_INDIVIDUAL, + ) + .or(Err(DeviceError::OutOfMemory))? + }, + total: 0, + available: Vec::new(), + pending: Vec::new(), + }, + ); + Ok(Self { + queue_family, + internal_thread_id, + inner: Mutex::new(Inner { pools }), + }) + } + + fn allocate_for_thread_id(&self, thread_id: thread::ThreadId) -> B::CommandBuffer { + let mut inner = self.inner.lock(); + inner.pools.get_mut(&thread_id).unwrap().allocate() + } + + pub fn allocate_internal(&self) -> B::CommandBuffer { + self.allocate_for_thread_id(self.internal_thread_id) + } + + pub fn extend(&self, cmd_buf: &CommandBuffer<B>) -> B::CommandBuffer { + self.allocate_for_thread_id(cmd_buf.recorded_thread_id) + } + + pub fn discard_internal(&self, raw: B::CommandBuffer) { + let mut inner = self.inner.lock(); + inner + .pools + .get_mut(&self.internal_thread_id) + .unwrap() + .recycle(raw); + } + + pub fn discard(&self, mut cmd_buf: CommandBuffer<B>) { + cmd_buf.trackers.clear(); + let mut inner = self.inner.lock(); + let pool = inner.pools.get_mut(&cmd_buf.recorded_thread_id).unwrap(); + for raw in cmd_buf.raw { + pool.recycle(raw); + } + } + + pub fn after_submit_internal(&self, raw: B::CommandBuffer, submit_index: SubmissionIndex) { + let mut inner = self.inner.lock(); + inner + .pools + .get_mut(&self.internal_thread_id) + .unwrap() + .pending + .push((raw, submit_index)); + } + + pub fn after_submit(&self, cmd_buf: CommandBuffer<B>, submit_index: SubmissionIndex) { + // Record this command buffer as pending + let mut inner = self.inner.lock(); + inner + .pools + .get_mut(&cmd_buf.recorded_thread_id) + .unwrap() + .pending + .extend(cmd_buf.raw.into_iter().map(|raw| (raw, submit_index))); + } + + pub fn maintain(&self, device: &B::Device, last_done_index: SubmissionIndex) { + let mut inner = self.inner.lock(); + let mut remove_threads = Vec::new(); + for (&thread_id, pool) in inner.pools.iter_mut() { + pool.maintain(last_done_index); + if pool.total == pool.available.len() && thread_id != self.internal_thread_id { + assert!(pool.pending.is_empty()); + remove_threads.push(thread_id); + } + } + for thread_id in remove_threads { + tracing::info!("Removing from thread {:?}", thread_id); + let mut pool = inner.pools.remove(&thread_id).unwrap(); + unsafe { + pool.raw.free(pool.available); + device.destroy_command_pool(pool.raw); + } + } + } + + pub fn destroy(self, device: &B::Device) { + let mut inner = self.inner.lock(); + for (_, mut pool) in inner.pools.drain() { + while let Some((raw, _)) = pool.pending.pop() { + pool.recycle(raw); + } + if pool.total != pool.available.len() { + tracing::error!( + "Some command buffers are still recorded, only tracking {} / {}", + pool.available.len(), + pool.total + ); + } + unsafe { + pool.raw.free(pool.available); + device.destroy_command_pool(pool.raw); + } + } + } +} + +#[derive(Clone, Debug, Error)] +pub enum CommandAllocatorError { + #[error(transparent)] + Device(#[from] DeviceError), +} diff --git a/gfx/wgpu/wgpu-core/src/command/bind.rs b/gfx/wgpu/wgpu-core/src/command/bind.rs new file mode 100644 index 0000000000..a62b38d5b7 --- /dev/null +++ b/gfx/wgpu/wgpu-core/src/command/bind.rs @@ -0,0 +1,295 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::{ + binding_model::{BindGroup, PipelineLayout}, + device::SHADER_STAGE_COUNT, + hub::{GfxBackend, Storage}, + id::{BindGroupId, BindGroupLayoutId, PipelineLayoutId, Valid}, + Stored, MAX_BIND_GROUPS, +}; + +use arrayvec::ArrayVec; +use std::slice; +use wgt::DynamicOffset; + +type BindGroupMask = u8; + +#[derive(Clone, Debug)] +pub(super) struct BindGroupPair { + layout_id: Valid<BindGroupLayoutId>, + group_id: Stored<BindGroupId>, +} + +#[derive(Debug)] +pub(super) enum LayoutChange<'a> { + Unchanged, + Match(Valid<BindGroupId>, &'a [DynamicOffset]), + Mismatch, +} + +#[derive(Debug)] +pub enum Provision { + Unchanged, + Changed { was_compatible: bool }, +} + +#[derive(Clone)] +pub(super) struct FollowUpIter<'a> { + iter: slice::Iter<'a, BindGroupEntry>, +} +impl<'a> Iterator for FollowUpIter<'a> { + type Item = (Valid<BindGroupId>, &'a [DynamicOffset]); + fn next(&mut self) -> Option<Self::Item> { + self.iter + .next() + .and_then(|entry| Some((entry.actual_value()?, entry.dynamic_offsets.as_slice()))) + } +} + +#[derive(Clone, Default, Debug)] +pub(super) struct BindGroupEntry { + expected_layout_id: Option<Valid<BindGroupLayoutId>>, + provided: Option<BindGroupPair>, + dynamic_offsets: Vec<DynamicOffset>, +} + +impl BindGroupEntry { + fn provide<B: GfxBackend>( + &mut self, + bind_group_id: Valid<BindGroupId>, + bind_group: &BindGroup<B>, + offsets: &[DynamicOffset], + ) -> Provision { + debug_assert_eq!(B::VARIANT, bind_group_id.0.backend()); + + let was_compatible = match self.provided { + Some(BindGroupPair { + layout_id, + ref group_id, + }) => { + if group_id.value == bind_group_id && offsets == self.dynamic_offsets.as_slice() { + assert_eq!(layout_id, bind_group.layout_id); + return Provision::Unchanged; + } + self.expected_layout_id == Some(layout_id) + } + None => false, + }; + + self.provided = Some(BindGroupPair { + layout_id: bind_group.layout_id, + group_id: Stored { + value: bind_group_id, + ref_count: bind_group.life_guard.add_ref(), + }, + }); + self.dynamic_offsets.clear(); + self.dynamic_offsets.extend_from_slice(offsets); + + Provision::Changed { was_compatible } + } + + pub fn expect_layout( + &mut self, + bind_group_layout_id: Valid<BindGroupLayoutId>, + ) -> LayoutChange { + let some = Some(bind_group_layout_id); + if self.expected_layout_id != some { + self.expected_layout_id = some; + match self.provided { + Some(BindGroupPair { + layout_id, + ref group_id, + }) if layout_id == bind_group_layout_id => { + LayoutChange::Match(group_id.value, &self.dynamic_offsets) + } + Some(_) | None => LayoutChange::Mismatch, + } + } else { + LayoutChange::Unchanged + } + } + + fn is_valid(&self) -> Option<bool> { + match (self.expected_layout_id, self.provided.as_ref()) { + (None, None) => Some(true), + (None, Some(_)) => None, + (Some(_), None) => Some(false), + (Some(layout), Some(pair)) => Some(layout == pair.layout_id), + } + } + + fn actual_value(&self) -> Option<Valid<BindGroupId>> { + self.expected_layout_id.and_then(|layout_id| { + self.provided.as_ref().and_then(|pair| { + if pair.layout_id == layout_id { + Some(pair.group_id.value) + } else { + None + } + }) + }) + } +} + +#[derive(Debug)] +pub struct Binder { + pub(super) pipeline_layout_id: Option<Valid<PipelineLayoutId>>, //TODO: strongly `Stored` + pub(super) entries: ArrayVec<[BindGroupEntry; MAX_BIND_GROUPS]>, +} + +impl Binder { + pub(super) fn new(max_bind_groups: u32) -> Self { + Self { + pipeline_layout_id: None, + entries: (0..max_bind_groups) + .map(|_| BindGroupEntry::default()) + .collect(), + } + } + + pub(super) fn reset(&mut self) { + self.pipeline_layout_id = None; + self.entries.clear(); + } + + pub(super) fn change_pipeline_layout<B: GfxBackend>( + &mut self, + guard: &Storage<PipelineLayout<B>, PipelineLayoutId>, + new_id: Valid<PipelineLayoutId>, + ) { + let old_id_opt = self.pipeline_layout_id.replace(new_id); + let new = &guard[new_id]; + + let length = if let Some(old_id) = old_id_opt { + let old = &guard[old_id]; + if old.push_constant_ranges == new.push_constant_ranges { + new.bind_group_layout_ids.len() + } else { + 0 + } + } else { + 0 + }; + + for entry in self.entries[length..].iter_mut() { + entry.expected_layout_id = None; + } + } + + /// Attempt to set the value of the specified bind group index. + /// Returns Some() when the new bind group is ready to be actually bound + /// (i.e. compatible with current expectations). Also returns an iterator + /// of bind group IDs to be bound with it: those are compatible bind groups + /// that were previously blocked because the current one was incompatible. + pub(super) fn provide_entry<'a, B: GfxBackend>( + &'a mut self, + index: usize, + bind_group_id: Valid<BindGroupId>, + bind_group: &BindGroup<B>, + offsets: &[DynamicOffset], + ) -> Option<(Valid<PipelineLayoutId>, FollowUpIter<'a>)> { + tracing::trace!("\tBinding [{}] = group {:?}", index, bind_group_id); + debug_assert_eq!(B::VARIANT, bind_group_id.0.backend()); + + match self.entries[index].provide(bind_group_id, bind_group, offsets) { + Provision::Unchanged => None, + Provision::Changed { was_compatible, .. } => { + let compatible_count = self.compatible_count(); + if index < compatible_count { + let end = compatible_count.min(if was_compatible { + index + 1 + } else { + self.entries.len() + }); + tracing::trace!("\t\tbinding up to {}", end); + Some(( + self.pipeline_layout_id?, + FollowUpIter { + iter: self.entries[index + 1..end].iter(), + }, + )) + } else { + tracing::trace!("\t\tskipping above compatible {}", compatible_count); + None + } + } + } + } + + pub(super) fn list_active(&self) -> impl Iterator<Item = Valid<BindGroupId>> + '_ { + self.entries.iter().filter_map(|e| match e.provided { + Some(ref pair) if e.expected_layout_id.is_some() => Some(pair.group_id.value), + _ => None, + }) + } + + pub(super) fn invalid_mask(&self) -> BindGroupMask { + self.entries.iter().enumerate().fold(0, |mask, (i, entry)| { + if entry.is_valid().unwrap_or(true) { + mask + } else { + mask | 1u8 << i + } + }) + } + + fn compatible_count(&self) -> usize { + self.entries + .iter() + .position(|entry| !entry.is_valid().unwrap_or(false)) + .unwrap_or_else(|| self.entries.len()) + } +} + +struct PushConstantChange { + stages: wgt::ShaderStage, + offset: u32, + enable: bool, +} + +/// Break up possibly overlapping push constant ranges into a set of non-overlapping ranges +/// which contain all the stage flags of the original ranges. This allows us to zero out (or write any value) +/// to every possible value. +pub fn compute_nonoverlapping_ranges( + ranges: &[wgt::PushConstantRange], +) -> ArrayVec<[wgt::PushConstantRange; SHADER_STAGE_COUNT * 2]> { + if ranges.is_empty() { + return ArrayVec::new(); + } + debug_assert!(ranges.len() <= SHADER_STAGE_COUNT); + + let mut breaks: ArrayVec<[PushConstantChange; SHADER_STAGE_COUNT * 2]> = ArrayVec::new(); + for range in ranges { + breaks.push(PushConstantChange { + stages: range.stages, + offset: range.range.start, + enable: true, + }); + breaks.push(PushConstantChange { + stages: range.stages, + offset: range.range.end, + enable: false, + }); + } + breaks.sort_unstable_by_key(|change| change.offset); + + let mut output_ranges = ArrayVec::new(); + let mut position = 0_u32; + let mut stages = wgt::ShaderStage::NONE; + + for bk in breaks { + if bk.offset - position > 0 && !stages.is_empty() { + output_ranges.push(wgt::PushConstantRange { + stages, + range: position..bk.offset, + }) + } + position = bk.offset; + stages.set(bk.stages, bk.enable); + } + + output_ranges +} diff --git a/gfx/wgpu/wgpu-core/src/command/bundle.rs b/gfx/wgpu/wgpu-core/src/command/bundle.rs new file mode 100644 index 0000000000..19c11c7136 --- /dev/null +++ b/gfx/wgpu/wgpu-core/src/command/bundle.rs @@ -0,0 +1,1230 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/*! Render Bundles + + ## Software implementation + + The path from nothing to using a render bundle consists of 3 phases. + + ### Initial command encoding + + User creates a `RenderBundleEncoder` and populates it by issuing commands + from `bundle_ffi` module, just like with `RenderPass`, except that the + set of available commands is reduced. Everything is written into a `RawPass`. + + ### Bundle baking + + Once the commands are encoded, user calls `render_bundle_encoder_finish`. + This is perhaps the most complex part of the logic. It consumes the + commands stored in `RawPass`, while validating everything, tracking the state, + and re-recording the commands into a separate `Vec<RenderCommand>`. It + doesn't actually execute any commands. + + What's more important, is that the produced vector of commands is "normalized", + which means it can be executed verbatim without any state tracking. More + formally, "normalized" command stream guarantees that any state required by + a draw call is set explicitly by one of the commands between the draw call + and the last changing of the pipeline. + + ### Execution + + When the bundle is used in an actual render pass, `RenderBundle::execute` is + called. It goes through the commands and issues them into the native command + buffer. Thanks to the "normalized" property, it doesn't track any bind group + invalidations or index format changes. +!*/ +#![allow(clippy::reversed_empty_ranges)] + +use crate::{ + command::{ + BasePass, DrawError, MapPassErr, PassErrorScope, RenderCommand, RenderCommandError, + StateChange, + }, + conv, + device::{ + AttachmentData, Device, DeviceError, RenderPassContext, MAX_VERTEX_BUFFERS, + SHADER_STAGE_COUNT, + }, + hub::{GfxBackend, GlobalIdentityHandlerFactory, Hub, Resource, Storage, Token}, + id, + resource::BufferUse, + span, + track::{TrackerSet, UsageConflict}, + validation::check_buffer_usage, + Label, LabelHelpers, LifeGuard, Stored, MAX_BIND_GROUPS, +}; +use arrayvec::ArrayVec; +use std::{borrow::Cow, iter, ops::Range}; +use thiserror::Error; + +/// Describes a [`RenderBundleEncoder`]. +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "trace", derive(serde::Serialize))] +#[cfg_attr(feature = "replay", derive(serde::Deserialize))] +pub struct RenderBundleEncoderDescriptor<'a> { + /// Debug label of the render bundle encoder. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// The formats of the color attachments that this render bundle is capable to rendering to. This + /// must match the formats of the color attachments in the renderpass this render bundle is executed in. + pub color_formats: Cow<'a, [wgt::TextureFormat]>, + /// The formats of the depth attachment that this render bundle is capable to rendering to. This + /// must match the formats of the depth attachments in the renderpass this render bundle is executed in. + pub depth_stencil_format: Option<wgt::TextureFormat>, + /// Sample count this render bundle is capable of rendering to. This must match the pipelines and + /// the renderpasses it is used in. + pub sample_count: u32, +} + +#[derive(Debug)] +#[cfg_attr(feature = "serial-pass", derive(serde::Deserialize, serde::Serialize))] +pub struct RenderBundleEncoder { + base: BasePass<RenderCommand>, + parent_id: id::DeviceId, + pub(crate) context: RenderPassContext, +} + +impl RenderBundleEncoder { + pub fn new( + desc: &RenderBundleEncoderDescriptor, + parent_id: id::DeviceId, + base: Option<BasePass<RenderCommand>>, + ) -> Result<Self, CreateRenderBundleError> { + span!(_guard, INFO, "RenderBundleEncoder::new"); + Ok(Self { + base: base.unwrap_or_else(BasePass::new), + parent_id, + context: RenderPassContext { + attachments: AttachmentData { + colors: desc.color_formats.iter().cloned().collect(), + resolves: ArrayVec::new(), + depth_stencil: desc.depth_stencil_format, + }, + sample_count: { + let sc = desc.sample_count; + if sc == 0 || sc > 32 || !conv::is_power_of_two(sc) { + return Err(CreateRenderBundleError::InvalidSampleCount(sc)); + } + sc as u8 + }, + }, + }) + } + + pub fn dummy(parent_id: id::DeviceId) -> Self { + Self { + base: BasePass::new(), + parent_id, + context: RenderPassContext { + attachments: AttachmentData { + colors: ArrayVec::new(), + resolves: ArrayVec::new(), + depth_stencil: None, + }, + sample_count: 0, + }, + } + } + + pub fn parent(&self) -> id::DeviceId { + self.parent_id + } + + pub(crate) fn finish<B: hal::Backend, G: GlobalIdentityHandlerFactory>( + self, + desc: &RenderBundleDescriptor, + device: &Device<B>, + hub: &Hub<B, G>, + token: &mut Token<Device<B>>, + ) -> Result<RenderBundle, RenderBundleError> { + let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(token); + let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); + let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); + let (buffer_guard, _) = hub.buffers.read(&mut token); + + let mut state = State { + trackers: TrackerSet::new(self.parent_id.backend()), + index: IndexState::new(), + vertex: (0..MAX_VERTEX_BUFFERS) + .map(|_| VertexState::new()) + .collect(), + bind: (0..MAX_BIND_GROUPS).map(|_| BindState::new()).collect(), + push_constant_ranges: PushConstantState::new(), + raw_dynamic_offsets: Vec::new(), + flat_dynamic_offsets: Vec::new(), + used_bind_groups: 0, + pipeline: StateChange::new(), + }; + let mut commands = Vec::new(); + let mut base = self.base.as_ref(); + let mut pipeline_layout_id = None::<id::Valid<id::PipelineLayoutId>>; + + for &command in base.commands { + match command { + RenderCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group_id, + } => { + let scope = PassErrorScope::SetBindGroup(bind_group_id); + + let max_bind_groups = device.limits.max_bind_groups; + if (index as u32) >= max_bind_groups { + return Err(RenderCommandError::BindGroupIndexOutOfRange { + index, + max: max_bind_groups, + }) + .map_pass_err(scope); + } + + let offsets = &base.dynamic_offsets[..num_dynamic_offsets as usize]; + base.dynamic_offsets = &base.dynamic_offsets[num_dynamic_offsets as usize..]; + // Check for misaligned offsets. + if let Some(offset) = offsets + .iter() + .map(|offset| *offset as wgt::BufferAddress) + .find(|offset| offset % wgt::BIND_BUFFER_ALIGNMENT != 0) + { + return Err(RenderCommandError::UnalignedBufferOffset(offset)) + .map_pass_err(scope); + } + + let bind_group = state + .trackers + .bind_groups + .use_extend(&*bind_group_guard, bind_group_id, (), ()) + .map_err(|_| RenderCommandError::InvalidBindGroup(bind_group_id)) + .map_pass_err(scope)?; + if bind_group.dynamic_binding_info.len() != offsets.len() { + return Err(RenderCommandError::InvalidDynamicOffsetCount { + actual: offsets.len(), + expected: bind_group.dynamic_binding_info.len(), + }) + .map_pass_err(scope); + } + + state.set_bind_group(index, bind_group_id, bind_group.layout_id, offsets); + state + .trackers + .merge_extend(&bind_group.used) + .map_pass_err(scope)?; + } + RenderCommand::SetPipeline(pipeline_id) => { + let scope = PassErrorScope::SetPipelineRender(pipeline_id); + if state.pipeline.set_and_check_redundant(pipeline_id) { + continue; + } + + let pipeline = state + .trackers + .render_pipes + .use_extend(&*pipeline_guard, pipeline_id, (), ()) + .unwrap(); + + self.context + .check_compatible(&pipeline.pass_context) + .map_err(RenderCommandError::IncompatiblePipeline) + .map_pass_err(scope)?; + + //TODO: check read-only depth + + let layout = &pipeline_layout_guard[pipeline.layout_id.value]; + pipeline_layout_id = Some(pipeline.layout_id.value); + + state.set_pipeline( + pipeline.index_format, + &pipeline.vertex_strides, + &layout.bind_group_layout_ids, + &layout.push_constant_ranges, + ); + commands.push(command); + if let Some(iter) = state.flush_push_constants() { + commands.extend(iter) + } + } + RenderCommand::SetIndexBuffer { + buffer_id, + offset, + size, + } => { + let scope = PassErrorScope::SetIndexBuffer(buffer_id); + let buffer = state + .trackers + .buffers + .use_extend(&*buffer_guard, buffer_id, (), BufferUse::INDEX) + .unwrap(); + check_buffer_usage(buffer.usage, wgt::BufferUsage::INDEX) + .map_pass_err(scope)?; + + let end = match size { + Some(s) => offset + s.get(), + None => buffer.size, + }; + state.index.set_buffer(buffer_id, offset..end); + } + RenderCommand::SetVertexBuffer { + slot, + buffer_id, + offset, + size, + } => { + let scope = PassErrorScope::SetVertexBuffer(buffer_id); + let buffer = state + .trackers + .buffers + .use_extend(&*buffer_guard, buffer_id, (), BufferUse::VERTEX) + .unwrap(); + check_buffer_usage(buffer.usage, wgt::BufferUsage::VERTEX) + .map_pass_err(scope)?; + + let end = match size { + Some(s) => offset + s.get(), + None => buffer.size, + }; + state.vertex[slot as usize].set_buffer(buffer_id, offset..end); + } + RenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset: _, + } => { + let scope = PassErrorScope::SetPushConstant; + let end_offset = offset + size_bytes; + + let pipeline_layout_id = pipeline_layout_id + .ok_or(DrawError::MissingPipeline) + .map_pass_err(scope)?; + let pipeline_layout = &pipeline_layout_guard[pipeline_layout_id]; + + pipeline_layout + .validate_push_constant_ranges(stages, offset, end_offset) + .map_pass_err(scope)?; + + commands.push(command); + } + RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + } => { + let scope = PassErrorScope::Draw; + let (vertex_limit, instance_limit) = state.vertex_limits(); + let last_vertex = first_vertex + vertex_count; + if last_vertex > vertex_limit { + return Err(DrawError::VertexBeyondLimit { + last_vertex, + vertex_limit, + }) + .map_pass_err(scope); + } + let last_instance = first_instance + instance_count; + if last_instance > instance_limit { + return Err(DrawError::InstanceBeyondLimit { + last_instance, + instance_limit, + }) + .map_pass_err(scope); + } + commands.extend(state.flush_vertices()); + commands.extend(state.flush_binds()); + commands.push(command); + } + RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex: _, + first_instance, + } => { + let scope = PassErrorScope::DrawIndexed; + //TODO: validate that base_vertex + max_index() is within the provided range + let (_, instance_limit) = state.vertex_limits(); + let index_limit = state.index.limit(); + let last_index = first_index + index_count; + if last_index > index_limit { + return Err(DrawError::IndexBeyondLimit { + last_index, + index_limit, + }) + .map_pass_err(scope); + } + let last_instance = first_instance + instance_count; + if last_instance > instance_limit { + return Err(DrawError::InstanceBeyondLimit { + last_instance, + instance_limit, + }) + .map_pass_err(scope); + } + commands.extend(state.index.flush()); + commands.extend(state.flush_vertices()); + commands.extend(state.flush_binds()); + commands.push(command); + } + RenderCommand::MultiDrawIndirect { + buffer_id, + offset: _, + count: None, + indexed: false, + } => { + let scope = PassErrorScope::DrawIndirect; + let buffer = state + .trackers + .buffers + .use_extend(&*buffer_guard, buffer_id, (), BufferUse::INDIRECT) + .unwrap(); + check_buffer_usage(buffer.usage, wgt::BufferUsage::INDIRECT) + .map_pass_err(scope)?; + + commands.extend(state.flush_vertices()); + commands.extend(state.flush_binds()); + commands.push(command); + } + RenderCommand::MultiDrawIndirect { + buffer_id, + offset: _, + count: None, + indexed: true, + } => { + let scope = PassErrorScope::DrawIndexedIndirect; + let buffer = state + .trackers + .buffers + .use_extend(&*buffer_guard, buffer_id, (), BufferUse::INDIRECT) + .map_err(|err| RenderCommandError::Buffer(buffer_id, err)) + .map_pass_err(scope)?; + check_buffer_usage(buffer.usage, wgt::BufferUsage::INDIRECT) + .map_pass_err(scope)?; + + commands.extend(state.index.flush()); + commands.extend(state.flush_vertices()); + commands.extend(state.flush_binds()); + commands.push(command); + } + RenderCommand::MultiDrawIndirect { .. } + | RenderCommand::MultiDrawIndirectCount { .. } => unimplemented!(), + RenderCommand::PushDebugGroup { color: _, len: _ } => unimplemented!(), + RenderCommand::InsertDebugMarker { color: _, len: _ } => unimplemented!(), + RenderCommand::PopDebugGroup => unimplemented!(), + RenderCommand::ExecuteBundle(_) + | RenderCommand::SetBlendColor(_) + | RenderCommand::SetStencilReference(_) + | RenderCommand::SetViewport { .. } + | RenderCommand::SetScissor(_) => unreachable!("not supported by a render bundle"), + } + } + + let _ = desc.label; //TODO: actually use + Ok(RenderBundle { + base: BasePass { + commands, + dynamic_offsets: state.flat_dynamic_offsets, + string_data: Vec::new(), + push_constant_data: Vec::new(), + }, + device_id: Stored { + value: id::Valid(self.parent_id), + ref_count: device.life_guard.add_ref(), + }, + used: state.trackers, + context: self.context, + life_guard: LifeGuard::new(desc.label.borrow_or_default()), + }) + } +} + +/// Error type returned from `RenderBundleEncoder::new` if the sample count is invalid. +#[derive(Clone, Debug, Error)] +pub enum CreateRenderBundleError { + #[error("invalid number of samples {0}")] + InvalidSampleCount(u32), +} + +/// Error type returned from `RenderBundleEncoder::new` if the sample count is invalid. +#[derive(Clone, Debug, Error)] +pub enum ExecutionError { + #[error("buffer {0:?} is destroyed")] + DestroyedBuffer(id::BufferId), +} + +pub type RenderBundleDescriptor<'a> = wgt::RenderBundleDescriptor<Label<'a>>; + +//Note: here, `RenderBundle` is just wrapping a raw stream of render commands. +// The plan is to back it by an actual Vulkan secondary buffer, D3D12 Bundle, +// or Metal indirect command buffer. +#[derive(Debug)] +pub struct RenderBundle { + // Normalized command stream. It can be executed verbatim, + // without re-binding anything on the pipeline change. + base: BasePass<RenderCommand>, + pub(crate) device_id: Stored<id::DeviceId>, + pub(crate) used: TrackerSet, + pub(crate) context: RenderPassContext, + pub(crate) life_guard: LifeGuard, +} + +unsafe impl Send for RenderBundle {} +unsafe impl Sync for RenderBundle {} + +impl RenderBundle { + #[cfg(feature = "trace")] + pub(crate) fn to_base_pass(&self) -> BasePass<RenderCommand> { + BasePass::from_ref(self.base.as_ref()) + } + + /// Actually encode the contents into a native command buffer. + /// + /// This is partially duplicating the logic of `command_encoder_run_render_pass`. + /// However the point of this function is to be lighter, since we already had + /// a chance to go through the commands in `render_bundle_encoder_finish`. + /// + /// Note that the function isn't expected to fail, generally. + /// All the validation has already been done by this point. + /// The only failure condition is if some of the used buffers are destroyed. + pub(crate) unsafe fn execute<B: GfxBackend>( + &self, + cmd_buf: &mut B::CommandBuffer, + pipeline_layout_guard: &Storage< + crate::binding_model::PipelineLayout<B>, + id::PipelineLayoutId, + >, + bind_group_guard: &Storage<crate::binding_model::BindGroup<B>, id::BindGroupId>, + pipeline_guard: &Storage<crate::pipeline::RenderPipeline<B>, id::RenderPipelineId>, + buffer_guard: &Storage<crate::resource::Buffer<B>, id::BufferId>, + ) -> Result<(), ExecutionError> { + use hal::command::CommandBuffer as _; + + let mut offsets = self.base.dynamic_offsets.as_slice(); + let mut index_type = hal::IndexType::U16; + let mut pipeline_layout_id = None::<id::Valid<id::PipelineLayoutId>>; + + for command in self.base.commands.iter() { + match *command { + RenderCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group_id, + } => { + let bind_group = bind_group_guard.get(bind_group_id).unwrap(); + cmd_buf.bind_graphics_descriptor_sets( + &pipeline_layout_guard[pipeline_layout_id.unwrap()].raw, + index as usize, + iter::once(bind_group.raw.raw()), + &offsets[..num_dynamic_offsets as usize], + ); + offsets = &offsets[num_dynamic_offsets as usize..]; + } + RenderCommand::SetPipeline(pipeline_id) => { + let pipeline = pipeline_guard.get(pipeline_id).unwrap(); + cmd_buf.bind_graphics_pipeline(&pipeline.raw); + index_type = conv::map_index_format(pipeline.index_format); + pipeline_layout_id = Some(pipeline.layout_id.value); + } + RenderCommand::SetIndexBuffer { + buffer_id, + offset, + size, + } => { + let &(ref buffer, _) = buffer_guard + .get(buffer_id) + .unwrap() + .raw + .as_ref() + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?; + let range = hal::buffer::SubRange { + offset, + size: size.map(|s| s.get()), + }; + cmd_buf.bind_index_buffer(buffer, range, index_type); + } + RenderCommand::SetVertexBuffer { + slot, + buffer_id, + offset, + size, + } => { + let &(ref buffer, _) = buffer_guard + .get(buffer_id) + .unwrap() + .raw + .as_ref() + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?; + let range = hal::buffer::SubRange { + offset, + size: size.map(|s| s.get()), + }; + cmd_buf.bind_vertex_buffers(slot, iter::once((buffer, range))); + } + RenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset, + } => { + let pipeline_layout_id = pipeline_layout_id.unwrap(); + let pipeline_layout = &pipeline_layout_guard[pipeline_layout_id]; + + if let Some(values_offset) = values_offset { + let values_end_offset = + (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; + let data_slice = &self.base.push_constant_data + [(values_offset as usize)..values_end_offset]; + + cmd_buf.push_graphics_constants( + &pipeline_layout.raw, + conv::map_shader_stage_flags(stages), + offset, + &data_slice, + ) + } else { + super::push_constant_clear( + offset, + size_bytes, + |clear_offset, clear_data| { + cmd_buf.push_graphics_constants( + &pipeline_layout.raw, + conv::map_shader_stage_flags(stages), + clear_offset, + clear_data, + ); + }, + ); + } + } + RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + } => { + cmd_buf.draw( + first_vertex..first_vertex + vertex_count, + first_instance..first_instance + instance_count, + ); + } + RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + } => { + cmd_buf.draw_indexed( + first_index..first_index + index_count, + base_vertex, + first_instance..first_instance + instance_count, + ); + } + RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: false, + } => { + let &(ref buffer, _) = buffer_guard + .get(buffer_id) + .unwrap() + .raw + .as_ref() + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?; + cmd_buf.draw_indirect(buffer, offset, 1, 0); + } + RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: true, + } => { + let &(ref buffer, _) = buffer_guard + .get(buffer_id) + .unwrap() + .raw + .as_ref() + .ok_or(ExecutionError::DestroyedBuffer(buffer_id))?; + cmd_buf.draw_indexed_indirect(buffer, offset, 1, 0); + } + RenderCommand::MultiDrawIndirect { .. } + | RenderCommand::MultiDrawIndirectCount { .. } => unimplemented!(), + RenderCommand::PushDebugGroup { color: _, len: _ } => unimplemented!(), + RenderCommand::InsertDebugMarker { color: _, len: _ } => unimplemented!(), + RenderCommand::PopDebugGroup => unimplemented!(), + RenderCommand::ExecuteBundle(_) + | RenderCommand::SetBlendColor(_) + | RenderCommand::SetStencilReference(_) + | RenderCommand::SetViewport { .. } + | RenderCommand::SetScissor(_) => unreachable!(), + } + } + + Ok(()) + } +} + +impl Resource for RenderBundle { + const TYPE: &'static str = "RenderBundle"; + + fn life_guard(&self) -> &LifeGuard { + &self.life_guard + } +} + +#[derive(Debug)] +struct IndexState { + buffer: Option<id::BufferId>, + format: wgt::IndexFormat, + range: Range<wgt::BufferAddress>, + is_dirty: bool, +} + +impl IndexState { + fn new() -> Self { + Self { + buffer: None, + format: wgt::IndexFormat::default(), + range: 0..0, + is_dirty: false, + } + } + + fn limit(&self) -> u32 { + assert!(self.buffer.is_some()); + let bytes_per_index = match self.format { + wgt::IndexFormat::Uint16 => 2, + wgt::IndexFormat::Uint32 => 4, + }; + ((self.range.end - self.range.start) / bytes_per_index) as u32 + } + + fn flush(&mut self) -> Option<RenderCommand> { + if self.is_dirty { + self.is_dirty = false; + Some(RenderCommand::SetIndexBuffer { + buffer_id: self.buffer.unwrap(), + offset: self.range.start, + size: wgt::BufferSize::new(self.range.end - self.range.start), + }) + } else { + None + } + } + + fn set_format(&mut self, format: wgt::IndexFormat) { + if self.format != format { + self.format = format; + self.is_dirty = true; + } + } + + fn set_buffer(&mut self, id: id::BufferId, range: Range<wgt::BufferAddress>) { + self.buffer = Some(id); + self.range = range; + self.is_dirty = true; + } +} + +#[derive(Debug)] +struct VertexState { + buffer: Option<id::BufferId>, + range: Range<wgt::BufferAddress>, + stride: wgt::BufferAddress, + rate: wgt::InputStepMode, + is_dirty: bool, +} + +impl VertexState { + fn new() -> Self { + Self { + buffer: None, + range: 0..0, + stride: 0, + rate: wgt::InputStepMode::Vertex, + is_dirty: false, + } + } + + fn set_buffer(&mut self, buffer_id: id::BufferId, range: Range<wgt::BufferAddress>) { + self.buffer = Some(buffer_id); + self.range = range; + self.is_dirty = true; + } + + fn flush(&mut self, slot: u32) -> Option<RenderCommand> { + if self.is_dirty { + self.is_dirty = false; + Some(RenderCommand::SetVertexBuffer { + slot, + buffer_id: self.buffer.unwrap(), + offset: self.range.start, + size: wgt::BufferSize::new(self.range.end - self.range.start), + }) + } else { + None + } + } +} + +#[derive(Debug)] +struct BindState { + bind_group: Option<(id::BindGroupId, id::BindGroupLayoutId)>, + dynamic_offsets: Range<usize>, + is_dirty: bool, +} + +impl BindState { + fn new() -> Self { + Self { + bind_group: None, + dynamic_offsets: 0..0, + is_dirty: false, + } + } + + fn set_group( + &mut self, + bind_group_id: id::BindGroupId, + layout_id: id::BindGroupLayoutId, + dyn_offset: usize, + dyn_count: usize, + ) -> bool { + match self.bind_group { + Some((bg_id, _)) if bg_id == bind_group_id && dyn_count == 0 => false, + _ => { + self.bind_group = Some((bind_group_id, layout_id)); + self.dynamic_offsets = dyn_offset..dyn_offset + dyn_count; + self.is_dirty = true; + true + } + } + } +} + +#[derive(Debug)] +struct PushConstantState { + ranges: ArrayVec<[wgt::PushConstantRange; SHADER_STAGE_COUNT]>, + is_dirty: bool, +} +impl PushConstantState { + fn new() -> Self { + Self { + ranges: ArrayVec::new(), + is_dirty: false, + } + } + + fn set_push_constants(&mut self, new_ranges: &[wgt::PushConstantRange]) -> bool { + if &*self.ranges != new_ranges { + self.ranges = new_ranges.iter().cloned().collect(); + self.is_dirty = true; + true + } else { + false + } + } +} + +#[derive(Debug)] +struct State { + trackers: TrackerSet, + index: IndexState, + vertex: ArrayVec<[VertexState; MAX_VERTEX_BUFFERS]>, + bind: ArrayVec<[BindState; MAX_BIND_GROUPS]>, + push_constant_ranges: PushConstantState, + raw_dynamic_offsets: Vec<wgt::DynamicOffset>, + flat_dynamic_offsets: Vec<wgt::DynamicOffset>, + used_bind_groups: usize, + pipeline: StateChange<id::RenderPipelineId>, +} + +impl State { + fn vertex_limits(&self) -> (u32, u32) { + let mut vertex_limit = !0; + let mut instance_limit = !0; + for vbs in &self.vertex { + if vbs.stride == 0 { + continue; + } + let limit = ((vbs.range.end - vbs.range.start) / vbs.stride) as u32; + match vbs.rate { + wgt::InputStepMode::Vertex => vertex_limit = vertex_limit.min(limit), + wgt::InputStepMode::Instance => instance_limit = instance_limit.min(limit), + } + } + (vertex_limit, instance_limit) + } + + fn invalidate_group_from(&mut self, slot: usize) { + for bind in self.bind[slot..].iter_mut() { + if bind.bind_group.is_some() { + bind.is_dirty = true; + } + } + } + + fn set_bind_group( + &mut self, + slot: u8, + bind_group_id: id::BindGroupId, + layout_id: id::Valid<id::BindGroupLayoutId>, + offsets: &[wgt::DynamicOffset], + ) { + if self.bind[slot as usize].set_group( + bind_group_id, + layout_id.0, + self.raw_dynamic_offsets.len(), + offsets.len(), + ) { + self.invalidate_group_from(slot as usize + 1); + } + self.raw_dynamic_offsets.extend(offsets); + } + + fn set_pipeline( + &mut self, + index_format: wgt::IndexFormat, + vertex_strides: &[(wgt::BufferAddress, wgt::InputStepMode)], + layout_ids: &[id::Valid<id::BindGroupLayoutId>], + push_constant_layouts: &[wgt::PushConstantRange], + ) { + self.index.set_format(index_format); + for (vs, &(stride, step_mode)) in self.vertex.iter_mut().zip(vertex_strides) { + if vs.stride != stride || vs.rate != step_mode { + vs.stride = stride; + vs.rate = step_mode; + vs.is_dirty = true; + } + } + + let push_constants_changed = self + .push_constant_ranges + .set_push_constants(push_constant_layouts); + + self.used_bind_groups = layout_ids.len(); + let invalid_from = if push_constants_changed { + Some(0) + } else { + self.bind + .iter() + .zip(layout_ids) + .position(|(bs, layout_id)| match bs.bind_group { + Some((_, bgl_id)) => bgl_id != layout_id.0, + None => false, + }) + }; + if let Some(slot) = invalid_from { + self.invalidate_group_from(slot); + } + } + + fn flush_push_constants(&mut self) -> Option<impl Iterator<Item = RenderCommand>> { + let is_dirty = self.push_constant_ranges.is_dirty; + + if is_dirty { + let nonoverlapping_ranges = + super::bind::compute_nonoverlapping_ranges(&self.push_constant_ranges.ranges); + + Some( + nonoverlapping_ranges + .into_iter() + .map(|range| RenderCommand::SetPushConstant { + stages: range.stages, + offset: range.range.start, + size_bytes: range.range.end - range.range.start, + values_offset: None, + }), + ) + } else { + None + } + } + + fn flush_vertices(&mut self) -> impl Iterator<Item = RenderCommand> + '_ { + self.vertex + .iter_mut() + .enumerate() + .flat_map(|(i, vs)| vs.flush(i as u32)) + } + + fn flush_binds(&mut self) -> impl Iterator<Item = RenderCommand> + '_ { + for bs in self.bind[..self.used_bind_groups].iter() { + if bs.is_dirty { + self.flat_dynamic_offsets + .extend_from_slice(&self.raw_dynamic_offsets[bs.dynamic_offsets.clone()]); + } + } + self.bind + .iter_mut() + .take(self.used_bind_groups) + .enumerate() + .flat_map(|(i, bs)| { + if bs.is_dirty { + bs.is_dirty = false; + Some(RenderCommand::SetBindGroup { + index: i as u8, + bind_group_id: bs.bind_group.unwrap().0, + num_dynamic_offsets: (bs.dynamic_offsets.end - bs.dynamic_offsets.start) + as u8, + }) + } else { + None + } + }) + } +} + +/// Error encountered when finishing recording a render bundle. +#[derive(Clone, Debug, Error)] +pub(super) enum RenderBundleErrorInner { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + RenderCommand(RenderCommandError), + #[error(transparent)] + ResourceUsageConflict(#[from] UsageConflict), + #[error(transparent)] + Draw(#[from] DrawError), +} + +impl<T> From<T> for RenderBundleErrorInner +where + T: Into<RenderCommandError>, +{ + fn from(t: T) -> Self { + Self::RenderCommand(t.into()) + } +} + +/// Error encountered when finishing recording a render bundle. +#[derive(Clone, Debug, Error)] +#[error("{scope}")] +pub struct RenderBundleError { + pub scope: PassErrorScope, + #[source] + inner: RenderBundleErrorInner, +} + +impl RenderBundleError { + pub(crate) const INVALID_DEVICE: Self = RenderBundleError { + scope: PassErrorScope::Bundle, + inner: RenderBundleErrorInner::Device(DeviceError::Invalid), + }; +} + +impl<T, E> MapPassErr<T, RenderBundleError> for Result<T, E> +where + E: Into<RenderBundleErrorInner>, +{ + fn map_pass_err(self, scope: PassErrorScope) -> Result<T, RenderBundleError> { + self.map_err(|inner| RenderBundleError { + scope, + inner: inner.into(), + }) + } +} + +pub mod bundle_ffi { + use super::{RenderBundleEncoder, RenderCommand}; + use crate::{id, span, RawString}; + use std::{convert::TryInto, slice}; + use wgt::{BufferAddress, BufferSize, DynamicOffset}; + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `offset_length` elements. + // TODO: There might be other safety issues, such as using the unsafe + // `RawPass::encode` and `RawPass::encode_slice`. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_bundle_set_bind_group( + bundle: &mut RenderBundleEncoder, + index: u32, + bind_group_id: id::BindGroupId, + offsets: *const DynamicOffset, + offset_length: usize, + ) { + span!(_guard, DEBUG, "RenderBundle::set_bind_group"); + bundle.base.commands.push(RenderCommand::SetBindGroup { + index: index.try_into().unwrap(), + num_dynamic_offsets: offset_length.try_into().unwrap(), + bind_group_id, + }); + bundle + .base + .dynamic_offsets + .extend_from_slice(slice::from_raw_parts(offsets, offset_length)); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_set_pipeline( + bundle: &mut RenderBundleEncoder, + pipeline_id: id::RenderPipelineId, + ) { + span!(_guard, DEBUG, "RenderBundle::set_pipeline"); + bundle + .base + .commands + .push(RenderCommand::SetPipeline(pipeline_id)); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_set_index_buffer( + bundle: &mut RenderBundleEncoder, + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option<BufferSize>, + ) { + span!(_guard, DEBUG, "RenderBundle::set_index_buffer"); + bundle.base.commands.push(RenderCommand::SetIndexBuffer { + buffer_id, + offset, + size, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_set_vertex_buffer( + bundle: &mut RenderBundleEncoder, + slot: u32, + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option<BufferSize>, + ) { + span!(_guard, DEBUG, "RenderBundle::set_vertex_buffer"); + bundle.base.commands.push(RenderCommand::SetVertexBuffer { + slot, + buffer_id, + offset, + size, + }); + } + + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_bundle_set_push_constants( + pass: &mut RenderBundleEncoder, + stages: wgt::ShaderStage, + offset: u32, + size_bytes: u32, + data: *const u8, + ) { + span!(_guard, DEBUG, "RenderBundle::set_push_constants"); + assert_eq!( + offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant offset must be aligned to 4 bytes." + ); + assert_eq!( + size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant size must be aligned to 4 bytes." + ); + let data_slice = slice::from_raw_parts(data, size_bytes as usize); + let value_offset = pass.base.push_constant_data.len().try_into().expect( + "Ran out of push constant space. Don't set 4gb of push constants per RenderBundle.", + ); + + pass.base.push_constant_data.extend( + data_slice + .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize) + .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])), + ); + + pass.base.commands.push(RenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset: Some(value_offset), + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_draw( + bundle: &mut RenderBundleEncoder, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, + ) { + span!(_guard, DEBUG, "RenderBundle::draw"); + bundle.base.commands.push(RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_draw_indexed( + bundle: &mut RenderBundleEncoder, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, + ) { + span!(_guard, DEBUG, "RenderBundle::draw_indexed"); + bundle.base.commands.push(RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_bundle_draw_indirect( + bundle: &mut RenderBundleEncoder, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + span!(_guard, DEBUG, "RenderBundle::draw_indirect"); + bundle.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: false, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_bundle_indexed_indirect( + bundle: &mut RenderBundleEncoder, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + span!(_guard, DEBUG, "RenderBundle::draw_indexed_indirect"); + bundle.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: true, + }); + } + + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_bundle_push_debug_group( + _bundle: &mut RenderBundleEncoder, + _label: RawString, + ) { + span!(_guard, DEBUG, "RenderBundle::push_debug_group"); + //TODO + } + + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_bundle_pop_debug_group(_bundle: &mut RenderBundleEncoder) { + span!(_guard, DEBUG, "RenderBundle::pop_debug_group"); + //TODO + } + + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_bundle_insert_debug_marker( + _bundle: &mut RenderBundleEncoder, + _label: RawString, + ) { + span!(_guard, DEBUG, "RenderBundle::insert_debug_marker"); + //TODO + } +} diff --git a/gfx/wgpu/wgpu-core/src/command/compute.rs b/gfx/wgpu/wgpu-core/src/command/compute.rs new file mode 100644 index 0000000000..1ab9df2516 --- /dev/null +++ b/gfx/wgpu/wgpu-core/src/command/compute.rs @@ -0,0 +1,657 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::{ + binding_model::{BindError, BindGroup, PushConstantUploadError}, + command::{ + bind::{Binder, LayoutChange}, + BasePass, BasePassRef, CommandBuffer, CommandEncoderError, MapPassErr, PassErrorScope, + StateChange, + }, + hub::{GfxBackend, Global, GlobalIdentityHandlerFactory, Storage, Token}, + id, + resource::{Buffer, BufferUse, Texture}, + span, + track::{TrackerSet, UsageConflict}, + validation::{check_buffer_usage, MissingBufferUsageError}, + MAX_BIND_GROUPS, +}; + +use arrayvec::ArrayVec; +use hal::command::CommandBuffer as _; +use thiserror::Error; +use wgt::{BufferAddress, BufferUsage, ShaderStage}; + +use std::{fmt, iter, str}; + +#[doc(hidden)] +#[derive(Clone, Copy, Debug)] +#[cfg_attr( + any(feature = "serial-pass", feature = "trace"), + derive(serde::Serialize) +)] +#[cfg_attr( + any(feature = "serial-pass", feature = "replay"), + derive(serde::Deserialize) +)] +pub enum ComputeCommand { + SetBindGroup { + index: u8, + num_dynamic_offsets: u8, + bind_group_id: id::BindGroupId, + }, + SetPipeline(id::ComputePipelineId), + SetPushConstant { + offset: u32, + size_bytes: u32, + values_offset: u32, + }, + Dispatch([u32; 3]), + DispatchIndirect { + buffer_id: id::BufferId, + offset: BufferAddress, + }, + PushDebugGroup { + color: u32, + len: usize, + }, + PopDebugGroup, + InsertDebugMarker { + color: u32, + len: usize, + }, +} + +#[cfg_attr(feature = "serial-pass", derive(serde::Deserialize, serde::Serialize))] +pub struct ComputePass { + base: BasePass<ComputeCommand>, + parent_id: id::CommandEncoderId, +} + +impl ComputePass { + pub fn new(parent_id: id::CommandEncoderId) -> Self { + Self { + base: BasePass::new(), + parent_id, + } + } + + pub fn parent_id(&self) -> id::CommandEncoderId { + self.parent_id + } + + #[cfg(feature = "trace")] + pub fn into_command(self) -> crate::device::trace::Command { + crate::device::trace::Command::RunComputePass { base: self.base } + } +} + +impl fmt::Debug for ComputePass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "ComputePass {{ encoder_id: {:?}, data: {:?} commands and {:?} dynamic offsets }}", + self.parent_id, + self.base.commands.len(), + self.base.dynamic_offsets.len() + ) + } +} + +#[repr(C)] +#[derive(Clone, Debug, Default)] +pub struct ComputePassDescriptor { + pub todo: u32, +} + +#[derive(Clone, Debug, Error, PartialEq)] +pub enum DispatchError { + #[error("compute pipeline must be set")] + MissingPipeline, + #[error("current compute pipeline has a layout which is incompatible with a currently set bind group, first differing at entry index {index}")] + IncompatibleBindGroup { + index: u32, + //expected: BindGroupLayoutId, + //provided: Option<(BindGroupLayoutId, BindGroupId)>, + }, +} + +/// Error encountered when performing a compute pass. +#[derive(Clone, Debug, Error)] +pub enum ComputePassErrorInner { + #[error(transparent)] + Encoder(#[from] CommandEncoderError), + #[error("bind group {0:?} is invalid")] + InvalidBindGroup(id::BindGroupId), + #[error("bind group index {index} is greater than the device's requested `max_bind_group` limit {max}")] + BindGroupIndexOutOfRange { index: u8, max: u32 }, + #[error("compute pipeline {0:?} is invalid")] + InvalidPipeline(id::ComputePipelineId), + #[error("indirect buffer {0:?} is invalid or destroyed")] + InvalidIndirectBuffer(id::BufferId), + #[error(transparent)] + ResourceUsageConflict(#[from] UsageConflict), + #[error(transparent)] + MissingBufferUsage(#[from] MissingBufferUsageError), + #[error("cannot pop debug group, because number of pushed debug groups is zero")] + InvalidPopDebugGroup, + #[error(transparent)] + Dispatch(#[from] DispatchError), + #[error(transparent)] + Bind(#[from] BindError), + #[error(transparent)] + PushConstants(#[from] PushConstantUploadError), +} + +/// Error encountered when performing a compute pass. +#[derive(Clone, Debug, Error)] +#[error("{scope}")] +pub struct ComputePassError { + pub scope: PassErrorScope, + #[source] + inner: ComputePassErrorInner, +} + +impl<T, E> MapPassErr<T, ComputePassError> for Result<T, E> +where + E: Into<ComputePassErrorInner>, +{ + fn map_pass_err(self, scope: PassErrorScope) -> Result<T, ComputePassError> { + self.map_err(|inner| ComputePassError { + scope, + inner: inner.into(), + }) + } +} + +#[derive(Debug)] +struct State { + binder: Binder, + pipeline: StateChange<id::ComputePipelineId>, + trackers: TrackerSet, + debug_scope_depth: u32, +} + +impl State { + fn is_ready(&self) -> Result<(), DispatchError> { + //TODO: vertex buffers + let bind_mask = self.binder.invalid_mask(); + if bind_mask != 0 { + //let (expected, provided) = self.binder.entries[index as usize].info(); + return Err(DispatchError::IncompatibleBindGroup { + index: bind_mask.trailing_zeros(), + }); + } + if self.pipeline.is_unset() { + return Err(DispatchError::MissingPipeline); + } + Ok(()) + } + + fn flush_states<B: GfxBackend>( + &mut self, + raw_cmd_buf: &mut B::CommandBuffer, + base_trackers: &mut TrackerSet, + bind_group_guard: &Storage<BindGroup<B>, id::BindGroupId>, + buffer_guard: &Storage<Buffer<B>, id::BufferId>, + texture_guard: &Storage<Texture<B>, id::TextureId>, + ) -> Result<(), UsageConflict> { + for id in self.binder.list_active() { + self.trackers.merge_extend(&bind_group_guard[id].used)?; + } + + tracing::trace!("Encoding dispatch barriers"); + + CommandBuffer::insert_barriers( + raw_cmd_buf, + base_trackers, + &self.trackers, + buffer_guard, + texture_guard, + ); + + self.trackers.clear(); + Ok(()) + } +} + +// Common routines between render/compute + +impl<G: GlobalIdentityHandlerFactory> Global<G> { + pub fn command_encoder_run_compute_pass<B: GfxBackend>( + &self, + encoder_id: id::CommandEncoderId, + pass: &ComputePass, + ) -> Result<(), ComputePassError> { + self.command_encoder_run_compute_pass_impl::<B>(encoder_id, pass.base.as_ref()) + } + + #[doc(hidden)] + pub fn command_encoder_run_compute_pass_impl<B: GfxBackend>( + &self, + encoder_id: id::CommandEncoderId, + mut base: BasePassRef<ComputeCommand>, + ) -> Result<(), ComputePassError> { + span!(_guard, INFO, "CommandEncoder::run_compute_pass"); + let scope = PassErrorScope::Pass(encoder_id); + + let hub = B::hub(self); + let mut token = Token::root(); + + let (mut cmd_buf_guard, mut token) = hub.command_buffers.write(&mut token); + let cmd_buf = + CommandBuffer::get_encoder(&mut *cmd_buf_guard, encoder_id).map_pass_err(scope)?; + let raw = cmd_buf.raw.last_mut().unwrap(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf.commands { + list.push(crate::device::trace::Command::RunComputePass { + base: BasePass::from_ref(base), + }); + } + + let (_, mut token) = hub.render_bundles.read(&mut token); + let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(&mut token); + let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); + let (pipeline_guard, mut token) = hub.compute_pipelines.read(&mut token); + let (buffer_guard, mut token) = hub.buffers.read(&mut token); + let (texture_guard, _) = hub.textures.read(&mut token); + + let mut state = State { + binder: Binder::new(cmd_buf.limits.max_bind_groups), + pipeline: StateChange::new(), + trackers: TrackerSet::new(B::VARIANT), + debug_scope_depth: 0, + }; + let mut temp_offsets = Vec::new(); + + for command in base.commands { + match *command { + ComputeCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group_id, + } => { + let scope = PassErrorScope::SetBindGroup(bind_group_id); + + let max_bind_groups = cmd_buf.limits.max_bind_groups; + if (index as u32) >= max_bind_groups { + return Err(ComputePassErrorInner::BindGroupIndexOutOfRange { + index, + max: max_bind_groups, + }) + .map_pass_err(scope); + } + + temp_offsets.clear(); + temp_offsets + .extend_from_slice(&base.dynamic_offsets[..num_dynamic_offsets as usize]); + base.dynamic_offsets = &base.dynamic_offsets[num_dynamic_offsets as usize..]; + + let bind_group = cmd_buf + .trackers + .bind_groups + .use_extend(&*bind_group_guard, bind_group_id, (), ()) + .map_err(|_| ComputePassErrorInner::InvalidBindGroup(bind_group_id)) + .map_pass_err(scope)?; + bind_group + .validate_dynamic_bindings(&temp_offsets) + .map_pass_err(scope)?; + + if let Some((pipeline_layout_id, follow_ups)) = state.binder.provide_entry( + index as usize, + id::Valid(bind_group_id), + bind_group, + &temp_offsets, + ) { + let bind_groups = iter::once(bind_group.raw.raw()) + .chain( + follow_ups + .clone() + .map(|(bg_id, _)| bind_group_guard[bg_id].raw.raw()), + ) + .collect::<ArrayVec<[_; MAX_BIND_GROUPS]>>(); + temp_offsets.extend(follow_ups.flat_map(|(_, offsets)| offsets)); + unsafe { + raw.bind_compute_descriptor_sets( + &pipeline_layout_guard[pipeline_layout_id].raw, + index as usize, + bind_groups, + &temp_offsets, + ); + } + } + } + ComputeCommand::SetPipeline(pipeline_id) => { + let scope = PassErrorScope::SetPipelineCompute(pipeline_id); + + if state.pipeline.set_and_check_redundant(pipeline_id) { + continue; + } + + let pipeline = cmd_buf + .trackers + .compute_pipes + .use_extend(&*pipeline_guard, pipeline_id, (), ()) + .map_err(|_| ComputePassErrorInner::InvalidPipeline(pipeline_id)) + .map_pass_err(scope)?; + + unsafe { + raw.bind_compute_pipeline(&pipeline.raw); + } + + // Rebind resources + if state.binder.pipeline_layout_id != Some(pipeline.layout_id.value) { + let pipeline_layout = &pipeline_layout_guard[pipeline.layout_id.value]; + + state.binder.change_pipeline_layout( + &*pipeline_layout_guard, + pipeline.layout_id.value, + ); + + let mut is_compatible = true; + + for (index, (entry, &bgl_id)) in state + .binder + .entries + .iter_mut() + .zip(&pipeline_layout.bind_group_layout_ids) + .enumerate() + { + match entry.expect_layout(bgl_id) { + LayoutChange::Match(bg_id, offsets) if is_compatible => { + let desc_set = bind_group_guard[bg_id].raw.raw(); + unsafe { + raw.bind_compute_descriptor_sets( + &pipeline_layout.raw, + index, + iter::once(desc_set), + offsets.iter().cloned(), + ); + } + } + LayoutChange::Match(..) | LayoutChange::Unchanged => {} + LayoutChange::Mismatch => { + is_compatible = false; + } + } + } + + // Clear push constant ranges + let non_overlapping = super::bind::compute_nonoverlapping_ranges( + &pipeline_layout.push_constant_ranges, + ); + for range in non_overlapping { + let offset = range.range.start; + let size_bytes = range.range.end - offset; + super::push_constant_clear( + offset, + size_bytes, + |clear_offset, clear_data| unsafe { + raw.push_compute_constants( + &pipeline_layout.raw, + clear_offset, + clear_data, + ); + }, + ); + } + } + } + ComputeCommand::SetPushConstant { + offset, + size_bytes, + values_offset, + } => { + let scope = PassErrorScope::SetPushConstant; + + let end_offset_bytes = offset + size_bytes; + let values_end_offset = + (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; + let data_slice = + &base.push_constant_data[(values_offset as usize)..values_end_offset]; + + let pipeline_layout_id = state + .binder + .pipeline_layout_id + //TODO: don't error here, lazily update the push constants + .ok_or(ComputePassErrorInner::Dispatch( + DispatchError::MissingPipeline, + )) + .map_pass_err(scope)?; + let pipeline_layout = &pipeline_layout_guard[pipeline_layout_id]; + + pipeline_layout + .validate_push_constant_ranges( + ShaderStage::COMPUTE, + offset, + end_offset_bytes, + ) + .map_pass_err(scope)?; + + unsafe { raw.push_compute_constants(&pipeline_layout.raw, offset, data_slice) } + } + ComputeCommand::Dispatch(groups) => { + let scope = PassErrorScope::Dispatch; + + state.is_ready().map_pass_err(scope)?; + state + .flush_states( + raw, + &mut cmd_buf.trackers, + &*bind_group_guard, + &*buffer_guard, + &*texture_guard, + ) + .map_pass_err(scope)?; + unsafe { + raw.dispatch(groups); + } + } + ComputeCommand::DispatchIndirect { buffer_id, offset } => { + let scope = PassErrorScope::DispatchIndirect; + + state.is_ready().map_pass_err(scope)?; + + let indirect_buffer = state + .trackers + .buffers + .use_extend(&*buffer_guard, buffer_id, (), BufferUse::INDIRECT) + .map_err(|_| ComputePassErrorInner::InvalidIndirectBuffer(buffer_id)) + .map_pass_err(scope)?; + check_buffer_usage(indirect_buffer.usage, BufferUsage::INDIRECT) + .map_pass_err(scope)?; + let &(ref buf_raw, _) = indirect_buffer + .raw + .as_ref() + .ok_or(ComputePassErrorInner::InvalidIndirectBuffer(buffer_id)) + .map_pass_err(scope)?; + + state + .flush_states( + raw, + &mut cmd_buf.trackers, + &*bind_group_guard, + &*buffer_guard, + &*texture_guard, + ) + .map_pass_err(scope)?; + unsafe { + raw.dispatch_indirect(buf_raw, offset); + } + } + ComputeCommand::PushDebugGroup { color, len } => { + state.debug_scope_depth += 1; + + let label = str::from_utf8(&base.string_data[..len]).unwrap(); + unsafe { + raw.begin_debug_marker(label, color); + } + base.string_data = &base.string_data[len..]; + } + ComputeCommand::PopDebugGroup => { + let scope = PassErrorScope::PopDebugGroup; + + if state.debug_scope_depth == 0 { + return Err(ComputePassErrorInner::InvalidPopDebugGroup) + .map_pass_err(scope); + } + state.debug_scope_depth -= 1; + unsafe { + raw.end_debug_marker(); + } + } + ComputeCommand::InsertDebugMarker { color, len } => { + let label = str::from_utf8(&base.string_data[..len]).unwrap(); + unsafe { raw.insert_debug_marker(label, color) } + base.string_data = &base.string_data[len..]; + } + } + } + + Ok(()) + } +} + +pub mod compute_ffi { + use super::{ComputeCommand, ComputePass}; + use crate::{id, span, RawString}; + use std::{convert::TryInto, ffi, slice}; + use wgt::{BufferAddress, DynamicOffset}; + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `offset_length` elements. + // TODO: There might be other safety issues, such as using the unsafe + // `RawPass::encode` and `RawPass::encode_slice`. + #[no_mangle] + pub unsafe extern "C" fn wgpu_compute_pass_set_bind_group( + pass: &mut ComputePass, + index: u32, + bind_group_id: id::BindGroupId, + offsets: *const DynamicOffset, + offset_length: usize, + ) { + span!(_guard, DEBUG, "ComputePass::set_bind_group"); + pass.base.commands.push(ComputeCommand::SetBindGroup { + index: index.try_into().unwrap(), + num_dynamic_offsets: offset_length.try_into().unwrap(), + bind_group_id, + }); + pass.base + .dynamic_offsets + .extend_from_slice(slice::from_raw_parts(offsets, offset_length)); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_set_pipeline( + pass: &mut ComputePass, + pipeline_id: id::ComputePipelineId, + ) { + span!(_guard, DEBUG, "ComputePass::set_pipeline"); + pass.base + .commands + .push(ComputeCommand::SetPipeline(pipeline_id)); + } + + #[no_mangle] + pub unsafe extern "C" fn wgpu_compute_pass_set_push_constant( + pass: &mut ComputePass, + offset: u32, + size_bytes: u32, + data: *const u8, + ) { + span!(_guard, DEBUG, "ComputePass::set_push_constant"); + assert_eq!( + offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant offset must be aligned to 4 bytes." + ); + assert_eq!( + size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant size must be aligned to 4 bytes." + ); + let data_slice = slice::from_raw_parts(data, size_bytes as usize); + let value_offset = pass.base.push_constant_data.len().try_into().expect( + "Ran out of push constant space. Don't set 4gb of push constants per ComputePass.", + ); + + pass.base.push_constant_data.extend( + data_slice + .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize) + .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])), + ); + + pass.base.commands.push(ComputeCommand::SetPushConstant { + offset, + size_bytes, + values_offset: value_offset, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_dispatch( + pass: &mut ComputePass, + groups_x: u32, + groups_y: u32, + groups_z: u32, + ) { + span!(_guard, DEBUG, "ComputePass::dispatch"); + pass.base + .commands + .push(ComputeCommand::Dispatch([groups_x, groups_y, groups_z])); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_dispatch_indirect( + pass: &mut ComputePass, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + span!(_guard, DEBUG, "ComputePass::dispatch_indirect"); + pass.base + .commands + .push(ComputeCommand::DispatchIndirect { buffer_id, offset }); + } + + #[no_mangle] + pub unsafe extern "C" fn wgpu_compute_pass_push_debug_group( + pass: &mut ComputePass, + label: RawString, + color: u32, + ) { + span!(_guard, DEBUG, "ComputePass::push_debug_group"); + let bytes = ffi::CStr::from_ptr(label).to_bytes(); + pass.base.string_data.extend_from_slice(bytes); + + pass.base.commands.push(ComputeCommand::PushDebugGroup { + color, + len: bytes.len(), + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_compute_pass_pop_debug_group(pass: &mut ComputePass) { + span!(_guard, DEBUG, "ComputePass::pop_debug_group"); + pass.base.commands.push(ComputeCommand::PopDebugGroup); + } + + #[no_mangle] + pub unsafe extern "C" fn wgpu_compute_pass_insert_debug_marker( + pass: &mut ComputePass, + label: RawString, + color: u32, + ) { + span!(_guard, DEBUG, "ComputePass::insert_debug_marker"); + let bytes = ffi::CStr::from_ptr(label).to_bytes(); + pass.base.string_data.extend_from_slice(bytes); + + pass.base.commands.push(ComputeCommand::InsertDebugMarker { + color, + len: bytes.len(), + }); + } +} diff --git a/gfx/wgpu/wgpu-core/src/command/draw.rs b/gfx/wgpu/wgpu-core/src/command/draw.rs new file mode 100644 index 0000000000..30c19fef7f --- /dev/null +++ b/gfx/wgpu/wgpu-core/src/command/draw.rs @@ -0,0 +1,180 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/*! Draw structures - shared between render passes and bundles. +!*/ + +use crate::{ + binding_model::PushConstantUploadError, + id, + resource::BufferUse, + track::UseExtendError, + validation::{MissingBufferUsageError, MissingTextureUsageError}, +}; +use wgt::{BufferAddress, BufferSize, Color}; + +use std::num::NonZeroU32; +use thiserror::Error; + +pub type BufferError = UseExtendError<BufferUse>; + +/// Error validating a draw call. +#[derive(Clone, Debug, Error, PartialEq)] +pub enum DrawError { + #[error("blend color needs to be set")] + MissingBlendColor, + #[error("render pipeline must be set")] + MissingPipeline, + #[error("current render pipeline has a layout which is incompatible with a currently set bind group, first differing at entry index {index}")] + IncompatibleBindGroup { + index: u32, + //expected: BindGroupLayoutId, + //provided: Option<(BindGroupLayoutId, BindGroupId)>, + }, + #[error("vertex {last_vertex} extends beyond limit {vertex_limit}")] + VertexBeyondLimit { last_vertex: u32, vertex_limit: u32 }, + #[error("instance {last_instance} extends beyond limit {instance_limit}")] + InstanceBeyondLimit { + last_instance: u32, + instance_limit: u32, + }, + #[error("index {last_index} extends beyond limit {index_limit}")] + IndexBeyondLimit { last_index: u32, index_limit: u32 }, +} + +/// Error encountered when encoding a render command. +/// This is the shared error set between render bundles and passes. +#[derive(Clone, Debug, Error)] +pub enum RenderCommandError { + #[error("bind group {0:?} is invalid")] + InvalidBindGroup(id::BindGroupId), + #[error("bind group index {index} is greater than the device's requested `max_bind_group` limit {max}")] + BindGroupIndexOutOfRange { index: u8, max: u32 }, + #[error("dynamic buffer offset {0} does not respect `BIND_BUFFER_ALIGNMENT`")] + UnalignedBufferOffset(u64), + #[error("number of buffer offsets ({actual}) does not match the number of dynamic bindings ({expected})")] + InvalidDynamicOffsetCount { actual: usize, expected: usize }, + #[error("render pipeline {0:?} is invalid")] + InvalidPipeline(id::RenderPipelineId), + #[error("Render pipeline is incompatible with render pass")] + IncompatiblePipeline(#[from] crate::device::RenderPassCompatibilityError), + #[error("pipeline is not compatible with the depth-stencil read-only render pass")] + IncompatibleReadOnlyDepthStencil, + #[error("buffer {0:?} is in error {1:?}")] + Buffer(id::BufferId, BufferError), + #[error("buffer {0:?} is destroyed")] + DestroyedBuffer(id::BufferId), + #[error(transparent)] + MissingBufferUsage(#[from] MissingBufferUsageError), + #[error(transparent)] + MissingTextureUsage(#[from] MissingTextureUsageError), + #[error(transparent)] + PushConstants(#[from] PushConstantUploadError), + #[error("Invalid Viewport parameters")] + InvalidViewport, + #[error("Invalid ScissorRect parameters")] + InvalidScissorRect, +} + +#[derive(Clone, Copy, Debug, Default)] +#[cfg_attr( + any(feature = "serial-pass", feature = "trace"), + derive(serde::Serialize) +)] +#[cfg_attr( + any(feature = "serial-pass", feature = "replay"), + derive(serde::Deserialize) +)] +pub struct Rect<T> { + pub x: T, + pub y: T, + pub w: T, + pub h: T, +} + +#[doc(hidden)] +#[derive(Clone, Copy, Debug)] +#[cfg_attr( + any(feature = "serial-pass", feature = "trace"), + derive(serde::Serialize) +)] +#[cfg_attr( + any(feature = "serial-pass", feature = "replay"), + derive(serde::Deserialize) +)] +pub enum RenderCommand { + SetBindGroup { + index: u8, + num_dynamic_offsets: u8, + bind_group_id: id::BindGroupId, + }, + SetPipeline(id::RenderPipelineId), + SetIndexBuffer { + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option<BufferSize>, + }, + SetVertexBuffer { + slot: u32, + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option<BufferSize>, + }, + SetBlendColor(Color), + SetStencilReference(u32), + SetViewport { + rect: Rect<f32>, + //TODO: use half-float to reduce the size? + depth_min: f32, + depth_max: f32, + }, + SetScissor(Rect<u32>), + SetPushConstant { + stages: wgt::ShaderStage, + offset: u32, + size_bytes: u32, + /// None means there is no data and the data should be an array of zeros. + /// + /// Facilitates clears in renderbundles which explicitly do their clears. + values_offset: Option<u32>, + }, + Draw { + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, + }, + DrawIndexed { + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, + }, + MultiDrawIndirect { + buffer_id: id::BufferId, + offset: BufferAddress, + /// Count of `None` represents a non-multi call. + count: Option<NonZeroU32>, + indexed: bool, + }, + MultiDrawIndirectCount { + buffer_id: id::BufferId, + offset: BufferAddress, + count_buffer_id: id::BufferId, + count_buffer_offset: BufferAddress, + max_count: u32, + indexed: bool, + }, + PushDebugGroup { + color: u32, + len: usize, + }, + PopDebugGroup, + InsertDebugMarker { + color: u32, + len: usize, + }, + ExecuteBundle(id::RenderBundleId), +} diff --git a/gfx/wgpu/wgpu-core/src/command/mod.rs b/gfx/wgpu/wgpu-core/src/command/mod.rs new file mode 100644 index 0000000000..1093bde155 --- /dev/null +++ b/gfx/wgpu/wgpu-core/src/command/mod.rs @@ -0,0 +1,362 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +mod allocator; +mod bind; +mod bundle; +mod compute; +mod draw; +mod render; +mod transfer; + +pub(crate) use self::allocator::CommandAllocator; +pub use self::allocator::CommandAllocatorError; +pub use self::bundle::*; +pub use self::compute::*; +pub use self::draw::*; +pub use self::render::*; +pub use self::transfer::*; + +use crate::{ + device::{all_buffer_stages, all_image_stages}, + hub::{GfxBackend, Global, GlobalIdentityHandlerFactory, Storage, Token}, + id, + resource::{Buffer, Texture}, + span, + track::TrackerSet, + Label, PrivateFeatures, Stored, +}; + +use hal::command::CommandBuffer as _; +use thiserror::Error; + +use std::thread::ThreadId; + +const PUSH_CONSTANT_CLEAR_ARRAY: &[u32] = &[0_u32; 64]; + +#[derive(Debug)] +pub struct CommandBuffer<B: hal::Backend> { + pub(crate) raw: Vec<B::CommandBuffer>, + is_recording: bool, + recorded_thread_id: ThreadId, + pub(crate) device_id: Stored<id::DeviceId>, + pub(crate) trackers: TrackerSet, + pub(crate) used_swap_chain: Option<(Stored<id::SwapChainId>, B::Framebuffer)>, + limits: wgt::Limits, + private_features: PrivateFeatures, + #[cfg(feature = "trace")] + pub(crate) commands: Option<Vec<crate::device::trace::Command>>, + #[cfg(debug_assertions)] + pub(crate) label: String, +} + +impl<B: GfxBackend> CommandBuffer<B> { + fn get_encoder( + storage: &mut Storage<Self, id::CommandEncoderId>, + id: id::CommandEncoderId, + ) -> Result<&mut Self, CommandEncoderError> { + match storage.get_mut(id) { + Ok(cmd_buf) if cmd_buf.is_recording => Ok(cmd_buf), + Ok(_) => Err(CommandEncoderError::NotRecording), + Err(_) => Err(CommandEncoderError::Invalid), + } + } + + pub(crate) fn insert_barriers( + raw: &mut B::CommandBuffer, + base: &mut TrackerSet, + head: &TrackerSet, + buffer_guard: &Storage<Buffer<B>, id::BufferId>, + texture_guard: &Storage<Texture<B>, id::TextureId>, + ) { + use hal::command::CommandBuffer as _; + + debug_assert_eq!(B::VARIANT, base.backend()); + debug_assert_eq!(B::VARIANT, head.backend()); + + let buffer_barriers = base.buffers.merge_replace(&head.buffers).map(|pending| { + let buf = &buffer_guard[pending.id]; + pending.into_hal(buf) + }); + let texture_barriers = base.textures.merge_replace(&head.textures).map(|pending| { + let tex = &texture_guard[pending.id]; + pending.into_hal(tex) + }); + base.views.merge_extend(&head.views).unwrap(); + base.bind_groups.merge_extend(&head.bind_groups).unwrap(); + base.samplers.merge_extend(&head.samplers).unwrap(); + base.compute_pipes + .merge_extend(&head.compute_pipes) + .unwrap(); + base.render_pipes.merge_extend(&head.render_pipes).unwrap(); + base.bundles.merge_extend(&head.bundles).unwrap(); + + let stages = all_buffer_stages() | all_image_stages(); + unsafe { + raw.pipeline_barrier( + stages..stages, + hal::memory::Dependencies::empty(), + buffer_barriers.chain(texture_barriers), + ); + } + } +} + +impl<B: hal::Backend> crate::hub::Resource for CommandBuffer<B> { + const TYPE: &'static str = "CommandBuffer"; + + fn life_guard(&self) -> &crate::LifeGuard { + unreachable!() + } + + fn label(&self) -> &str { + #[cfg(debug_assertions)] + return &self.label; + #[cfg(not(debug_assertions))] + return ""; + } +} + +#[derive(Copy, Clone, Debug)] +pub struct BasePassRef<'a, C> { + pub commands: &'a [C], + pub dynamic_offsets: &'a [wgt::DynamicOffset], + pub string_data: &'a [u8], + pub push_constant_data: &'a [u32], +} + +#[doc(hidden)] +#[derive(Debug)] +#[cfg_attr( + any(feature = "serial-pass", feature = "trace"), + derive(serde::Serialize) +)] +#[cfg_attr( + any(feature = "serial-pass", feature = "replay"), + derive(serde::Deserialize) +)] +pub struct BasePass<C> { + pub commands: Vec<C>, + pub dynamic_offsets: Vec<wgt::DynamicOffset>, + pub string_data: Vec<u8>, + pub push_constant_data: Vec<u32>, +} + +impl<C: Clone> BasePass<C> { + fn new() -> Self { + Self { + commands: Vec::new(), + dynamic_offsets: Vec::new(), + string_data: Vec::new(), + push_constant_data: Vec::new(), + } + } + + #[cfg(feature = "trace")] + fn from_ref(base: BasePassRef<C>) -> Self { + Self { + commands: base.commands.to_vec(), + dynamic_offsets: base.dynamic_offsets.to_vec(), + string_data: base.string_data.to_vec(), + push_constant_data: base.push_constant_data.to_vec(), + } + } + + pub fn as_ref(&self) -> BasePassRef<C> { + BasePassRef { + commands: &self.commands, + dynamic_offsets: &self.dynamic_offsets, + string_data: &self.string_data, + push_constant_data: &self.push_constant_data, + } + } +} + +#[derive(Clone, Debug, Error)] +pub enum CommandEncoderError { + #[error("command encoder is invalid")] + Invalid, + #[error("command encoder must be active")] + NotRecording, +} + +impl<G: GlobalIdentityHandlerFactory> Global<G> { + pub fn command_encoder_finish<B: GfxBackend>( + &self, + encoder_id: id::CommandEncoderId, + _desc: &wgt::CommandBufferDescriptor<Label>, + ) -> (id::CommandBufferId, Option<CommandEncoderError>) { + span!(_guard, INFO, "CommandEncoder::finish"); + + let hub = B::hub(self); + let mut token = Token::root(); + let (swap_chain_guard, mut token) = hub.swap_chains.read(&mut token); + //TODO: actually close the last recorded command buffer + let (mut cmd_buf_guard, _) = hub.command_buffers.write(&mut token); + + let error = match CommandBuffer::get_encoder(&mut *cmd_buf_guard, encoder_id) { + Ok(cmd_buf) => { + cmd_buf.is_recording = false; + // stop tracking the swapchain image, if used + if let Some((ref sc_id, _)) = cmd_buf.used_swap_chain { + let view_id = swap_chain_guard[sc_id.value] + .acquired_view_id + .as_ref() + .expect("Used swap chain frame has already presented"); + cmd_buf.trackers.views.remove(view_id.value); + } + tracing::trace!("Command buffer {:?} {:#?}", encoder_id, cmd_buf.trackers); + None + } + Err(e) => Some(e), + }; + + (encoder_id, error) + } + + pub fn command_encoder_push_debug_group<B: GfxBackend>( + &self, + encoder_id: id::CommandEncoderId, + label: &str, + ) -> Result<(), CommandEncoderError> { + span!(_guard, DEBUG, "CommandEncoder::push_debug_group"); + + let hub = B::hub(self); + let mut token = Token::root(); + + let (mut cmd_buf_guard, _) = hub.command_buffers.write(&mut token); + let cmd_buf = CommandBuffer::get_encoder(&mut *cmd_buf_guard, encoder_id)?; + let cmb_raw = cmd_buf.raw.last_mut().unwrap(); + + unsafe { + cmb_raw.begin_debug_marker(label, 0); + } + Ok(()) + } + + pub fn command_encoder_insert_debug_marker<B: GfxBackend>( + &self, + encoder_id: id::CommandEncoderId, + label: &str, + ) -> Result<(), CommandEncoderError> { + span!(_guard, DEBUG, "CommandEncoder::insert_debug_marker"); + + let hub = B::hub(self); + let mut token = Token::root(); + + let (mut cmd_buf_guard, _) = hub.command_buffers.write(&mut token); + let cmd_buf = CommandBuffer::get_encoder(&mut *cmd_buf_guard, encoder_id)?; + let cmb_raw = cmd_buf.raw.last_mut().unwrap(); + + unsafe { + cmb_raw.insert_debug_marker(label, 0); + } + Ok(()) + } + + pub fn command_encoder_pop_debug_group<B: GfxBackend>( + &self, + encoder_id: id::CommandEncoderId, + ) -> Result<(), CommandEncoderError> { + span!(_guard, DEBUG, "CommandEncoder::pop_debug_marker"); + + let hub = B::hub(self); + let mut token = Token::root(); + + let (mut cmd_buf_guard, _) = hub.command_buffers.write(&mut token); + let cmd_buf = CommandBuffer::get_encoder(&mut *cmd_buf_guard, encoder_id)?; + let cmb_raw = cmd_buf.raw.last_mut().unwrap(); + + unsafe { + cmb_raw.end_debug_marker(); + } + Ok(()) + } +} + +fn push_constant_clear<PushFn>(offset: u32, size_bytes: u32, mut push_fn: PushFn) +where + PushFn: FnMut(u32, &[u32]), +{ + let mut count_words = 0_u32; + let size_words = size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT; + while count_words < size_words { + let count_bytes = count_words * wgt::PUSH_CONSTANT_ALIGNMENT; + let size_to_write_words = + (size_words - count_words).min(PUSH_CONSTANT_CLEAR_ARRAY.len() as u32); + + push_fn( + offset + count_bytes, + &PUSH_CONSTANT_CLEAR_ARRAY[0..size_to_write_words as usize], + ); + + count_words += size_to_write_words; + } +} + +#[derive(Debug)] +struct StateChange<T> { + last_state: Option<T>, +} + +impl<T: Copy + PartialEq> StateChange<T> { + fn new() -> Self { + Self { last_state: None } + } + fn set_and_check_redundant(&mut self, new_state: T) -> bool { + let already_set = self.last_state == Some(new_state); + self.last_state = Some(new_state); + already_set + } + fn is_unset(&self) -> bool { + self.last_state.is_none() + } + fn reset(&mut self) { + self.last_state = None; + } +} + +trait MapPassErr<T, O> { + fn map_pass_err(self, scope: PassErrorScope) -> Result<T, O>; +} + +#[derive(Clone, Copy, Debug, Error)] +pub enum PassErrorScope { + #[error("In a bundle parameter")] + Bundle, + #[error("In a pass parameter")] + Pass(id::CommandEncoderId), + #[error("In a set_bind_group command")] + SetBindGroup(id::BindGroupId), + #[error("In a set_pipeline command")] + SetPipelineRender(id::RenderPipelineId), + #[error("In a set_pipeline command")] + SetPipelineCompute(id::ComputePipelineId), + #[error("In a set_push_constant command")] + SetPushConstant, + #[error("In a set_vertex_buffer command")] + SetVertexBuffer(id::BufferId), + #[error("In a set_index_buffer command")] + SetIndexBuffer(id::BufferId), + #[error("In a set_viewport command")] + SetViewport, + #[error("In a set_scissor_rect command")] + SetScissorRect, + #[error("In a draw command")] + Draw, + #[error("In a draw_indexed command")] + DrawIndexed, + #[error("In a draw_indirect command")] + DrawIndirect, + #[error("In a draw_indexed_indirect command")] + DrawIndexedIndirect, + #[error("In a execute_bundle command")] + ExecuteBundle, + #[error("In a dispatch command")] + Dispatch, + #[error("In a dispatch_indirect command")] + DispatchIndirect, + #[error("In a pop_debug_group command")] + PopDebugGroup, +} diff --git a/gfx/wgpu/wgpu-core/src/command/render.rs b/gfx/wgpu/wgpu-core/src/command/render.rs new file mode 100644 index 0000000000..531107956a --- /dev/null +++ b/gfx/wgpu/wgpu-core/src/command/render.rs @@ -0,0 +1,2078 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::{ + binding_model::BindError, + command::{ + bind::{Binder, LayoutChange}, + BasePass, BasePassRef, CommandBuffer, CommandEncoderError, DrawError, ExecutionError, + MapPassErr, PassErrorScope, RenderCommand, RenderCommandError, StateChange, + }, + conv, + device::{ + AttachmentData, AttachmentDataVec, FramebufferKey, RenderPassCompatibilityError, + RenderPassContext, RenderPassKey, MAX_COLOR_TARGETS, MAX_VERTEX_BUFFERS, + }, + hub::{GfxBackend, Global, GlobalIdentityHandlerFactory, Token}, + id, + pipeline::PipelineFlags, + resource::{BufferUse, TextureUse, TextureView, TextureViewInner}, + span, + track::{TextureSelector, TrackerSet, UsageConflict}, + validation::{ + check_buffer_usage, check_texture_usage, MissingBufferUsageError, MissingTextureUsageError, + }, + Stored, MAX_BIND_GROUPS, +}; + +use arrayvec::ArrayVec; +use hal::command::CommandBuffer as _; +use thiserror::Error; +use wgt::{BufferAddress, BufferUsage, Color, IndexFormat, InputStepMode, TextureUsage}; + +#[cfg(any(feature = "serial-pass", feature = "replay"))] +use serde::Deserialize; +#[cfg(any(feature = "serial-pass", feature = "trace"))] +use serde::Serialize; + +use std::{ + borrow::{Borrow, Cow}, + collections::hash_map::Entry, + fmt, iter, + num::NonZeroU32, + ops::Range, + str, +}; + +/// Operation to perform to the output attachment at the start of a renderpass. +#[repr(C)] +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] +#[cfg_attr(any(feature = "serial-pass", feature = "trace"), derive(Serialize))] +#[cfg_attr(any(feature = "serial-pass", feature = "replay"), derive(Deserialize))] +pub enum LoadOp { + /// Clear the output attachment with the clear color. Clearing is faster than loading. + Clear = 0, + /// Do not clear output attachment. + Load = 1, +} + +/// Operation to perform to the output attachment at the end of a renderpass. +#[repr(C)] +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] +#[cfg_attr(any(feature = "serial-pass", feature = "trace"), derive(Serialize))] +#[cfg_attr(any(feature = "serial-pass", feature = "replay"), derive(Deserialize))] +pub enum StoreOp { + /// Clear the render target. If you don't care about the contents of the target, this can be faster. + Clear = 0, + /// Store the result of the renderpass. + Store = 1, +} + +/// Describes an individual channel within a render pass, such as color, depth, or stencil. +#[repr(C)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(any(feature = "serial-pass", feature = "trace"), derive(Serialize))] +#[cfg_attr(any(feature = "serial-pass", feature = "replay"), derive(Deserialize))] +pub struct PassChannel<V> { + /// Operation to perform to the output attachment at the start of a renderpass. This must be clear if it + /// is the first renderpass rendering to a swap chain image. + pub load_op: LoadOp, + /// Operation to perform to the output attachment at the end of a renderpass. + pub store_op: StoreOp, + /// If load_op is [`LoadOp::Clear`], the attachement will be cleared to this color. + pub clear_value: V, + /// If true, the relevant channel is not changed by a renderpass, and the corresponding attachment + /// can be used inside the pass by other read-only usages. + pub read_only: bool, +} + +/// Describes a color attachment to a render pass. +#[repr(C)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(any(feature = "serial-pass", feature = "trace"), derive(Serialize))] +#[cfg_attr(any(feature = "serial-pass", feature = "replay"), derive(Deserialize))] +pub struct ColorAttachmentDescriptor { + /// The view to use as an attachment. + pub attachment: id::TextureViewId, + /// The view that will receive the resolved output if multisampling is used. + pub resolve_target: Option<id::TextureViewId>, + /// What operations will be performed on this color attachment. + pub channel: PassChannel<Color>, +} + +/// Describes a depth/stencil attachment to a render pass. +#[repr(C)] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(any(feature = "serial-pass", feature = "trace"), derive(Serialize))] +#[cfg_attr(any(feature = "serial-pass", feature = "replay"), derive(Deserialize))] +pub struct DepthStencilAttachmentDescriptor { + /// The view to use as an attachment. + pub attachment: id::TextureViewId, + /// What operations will be performed on the depth part of the attachment. + pub depth: PassChannel<f32>, + /// What operations will be performed on the stencil part of the attachment. + pub stencil: PassChannel<u32>, +} + +impl DepthStencilAttachmentDescriptor { + fn is_read_only(&self, aspects: hal::format::Aspects) -> Result<bool, RenderPassErrorInner> { + if aspects.contains(hal::format::Aspects::DEPTH) && !self.depth.read_only { + return Ok(false); + } + if (self.depth.load_op, self.depth.store_op) != (LoadOp::Load, StoreOp::Store) { + return Err(RenderPassErrorInner::InvalidDepthOps); + } + if aspects.contains(hal::format::Aspects::STENCIL) && !self.stencil.read_only { + return Ok(false); + } + if (self.stencil.load_op, self.stencil.store_op) != (LoadOp::Load, StoreOp::Store) { + return Err(RenderPassErrorInner::InvalidStencilOps); + } + Ok(true) + } +} + +/// Describes the attachments of a render pass. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct RenderPassDescriptor<'a> { + /// The color attachments of the render pass. + pub color_attachments: Cow<'a, [ColorAttachmentDescriptor]>, + /// The depth and stencil attachment of the render pass, if any. + pub depth_stencil_attachment: Option<&'a DepthStencilAttachmentDescriptor>, +} + +#[cfg_attr(feature = "serial-pass", derive(Deserialize, Serialize))] +pub struct RenderPass { + base: BasePass<RenderCommand>, + parent_id: id::CommandEncoderId, + color_targets: ArrayVec<[ColorAttachmentDescriptor; MAX_COLOR_TARGETS]>, + depth_stencil_target: Option<DepthStencilAttachmentDescriptor>, +} + +impl RenderPass { + pub fn new(parent_id: id::CommandEncoderId, desc: RenderPassDescriptor) -> Self { + Self { + base: BasePass::new(), + parent_id, + color_targets: desc.color_attachments.iter().cloned().collect(), + depth_stencil_target: desc.depth_stencil_attachment.cloned(), + } + } + + pub fn parent_id(&self) -> id::CommandEncoderId { + self.parent_id + } + + #[cfg(feature = "trace")] + pub fn into_command(self) -> crate::device::trace::Command { + crate::device::trace::Command::RunRenderPass { + base: self.base, + target_colors: self.color_targets.into_iter().collect(), + target_depth_stencil: self.depth_stencil_target, + } + } +} + +impl fmt::Debug for RenderPass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "RenderPass {{ encoder_id: {:?}, color_targets: {:?}, depth_stencil_target: {:?}, data: {:?} commands, {:?} dynamic offsets, and {:?} push constant u32s }}", + self.parent_id, + self.color_targets, + self.depth_stencil_target, + self.base.commands.len(), + self.base.dynamic_offsets.len(), + self.base.push_constant_data.len(), + ) + } +} + +#[derive(Debug, PartialEq)] +enum OptionalState { + Unused, + Required, + Set, +} + +impl OptionalState { + fn require(&mut self, require: bool) { + if require && *self == OptionalState::Unused { + *self = OptionalState::Required; + } + } +} + +#[derive(Debug, Default)] +struct IndexState { + bound_buffer_view: Option<(id::Valid<id::BufferId>, Range<BufferAddress>)>, + format: IndexFormat, + limit: u32, +} + +impl IndexState { + fn update_limit(&mut self) { + self.limit = match self.bound_buffer_view { + Some((_, ref range)) => { + let shift = match self.format { + IndexFormat::Uint16 => 1, + IndexFormat::Uint32 => 2, + }; + ((range.end - range.start) >> shift) as u32 + } + None => 0, + } + } + + fn reset(&mut self) { + self.bound_buffer_view = None; + self.limit = 0; + } +} + +#[derive(Clone, Copy, Debug)] +struct VertexBufferState { + total_size: BufferAddress, + stride: BufferAddress, + rate: InputStepMode, +} + +impl VertexBufferState { + const EMPTY: Self = VertexBufferState { + total_size: 0, + stride: 0, + rate: InputStepMode::Vertex, + }; +} + +#[derive(Debug, Default)] +struct VertexState { + inputs: ArrayVec<[VertexBufferState; MAX_VERTEX_BUFFERS]>, + vertex_limit: u32, + instance_limit: u32, +} + +impl VertexState { + fn update_limits(&mut self) { + self.vertex_limit = !0; + self.instance_limit = !0; + for vbs in &self.inputs { + if vbs.stride == 0 { + continue; + } + let limit = (vbs.total_size / vbs.stride) as u32; + match vbs.rate { + InputStepMode::Vertex => self.vertex_limit = self.vertex_limit.min(limit), + InputStepMode::Instance => self.instance_limit = self.instance_limit.min(limit), + } + } + } + + fn reset(&mut self) { + self.inputs.clear(); + self.vertex_limit = 0; + self.instance_limit = 0; + } +} + +#[derive(Debug)] +struct State { + pipeline_flags: PipelineFlags, + binder: Binder, + blend_color: OptionalState, + stencil_reference: u32, + pipeline: StateChange<id::RenderPipelineId>, + index: IndexState, + vertex: VertexState, + debug_scope_depth: u32, +} + +impl State { + fn is_ready(&self) -> Result<(), DrawError> { + //TODO: vertex buffers + let bind_mask = self.binder.invalid_mask(); + if bind_mask != 0 { + //let (expected, provided) = self.binder.entries[index as usize].info(); + return Err(DrawError::IncompatibleBindGroup { + index: bind_mask.trailing_zeros(), + }); + } + if self.pipeline.is_unset() { + return Err(DrawError::MissingPipeline); + } + if self.blend_color == OptionalState::Required { + return Err(DrawError::MissingBlendColor); + } + Ok(()) + } + + /// Reset the `RenderBundle`-related states. + fn reset_bundle(&mut self) { + self.binder.reset(); + self.pipeline.reset(); + self.index.reset(); + self.vertex.reset(); + } +} + +/// Error encountered when performing a render pass. +#[derive(Clone, Debug, Error)] +pub enum RenderPassErrorInner { + #[error(transparent)] + Encoder(#[from] CommandEncoderError), + #[error("attachment texture view {0:?} is invalid")] + InvalidAttachment(id::TextureViewId), + #[error("attachments have different sizes")] + MismatchAttachments, + #[error("attachment's sample count {0} is invalid")] + InvalidSampleCount(u8), + #[error("attachment with resolve target must be multi-sampled")] + InvalidResolveSourceSampleCount, + #[error("resolve target must have a sample count of 1")] + InvalidResolveTargetSampleCount, + #[error("not enough memory left")] + OutOfMemory, + #[error("extent state {state_extent:?} must match extent from view {view_extent:?}")] + ExtentStateMismatch { + state_extent: hal::image::Extent, + view_extent: hal::image::Extent, + }, + #[error("attempted to use a swap chain image as a depth/stencil attachment")] + SwapChainImageAsDepthStencil, + #[error("unable to clear non-present/read-only depth")] + InvalidDepthOps, + #[error("unable to clear non-present/read-only stencil")] + InvalidStencilOps, + #[error("all attachments must have the same sample count, found {actual} != {expected}")] + SampleCountMismatch { actual: u8, expected: u8 }, + #[error("texture view's swap chain must match swap chain in use")] + SwapChainMismatch, + #[error("setting `values_offset` to be `None` is only for internal use in render bundles")] + InvalidValuesOffset, + #[error("required device features not enabled: {0:?}")] + MissingDeviceFeatures(wgt::Features), + #[error("indirect draw with offset {offset}{} uses bytes {begin_offset}..{end_offset} which overruns indirect buffer of size {buffer_size}", count.map_or_else(String::new, |v| format!(" and count {}", v)))] + IndirectBufferOverrun { + offset: u64, + count: Option<NonZeroU32>, + begin_offset: u64, + end_offset: u64, + buffer_size: u64, + }, + #[error("indirect draw uses bytes {begin_count_offset}..{end_count_offset} which overruns indirect buffer of size {count_buffer_size}")] + IndirectCountBufferOverrun { + begin_count_offset: u64, + end_count_offset: u64, + count_buffer_size: u64, + }, + #[error("cannot pop debug group, because number of pushed debug groups is zero")] + InvalidPopDebugGroup, + #[error(transparent)] + ResourceUsageConflict(#[from] UsageConflict), + #[error("render bundle is incompatible, {0}")] + IncompatibleRenderBundle(#[from] RenderPassCompatibilityError), + #[error(transparent)] + RenderCommand(#[from] RenderCommandError), + #[error(transparent)] + Draw(#[from] DrawError), + #[error(transparent)] + Bind(#[from] BindError), +} + +impl From<MissingBufferUsageError> for RenderPassErrorInner { + fn from(error: MissingBufferUsageError) -> Self { + Self::RenderCommand(error.into()) + } +} + +impl From<MissingTextureUsageError> for RenderPassErrorInner { + fn from(error: MissingTextureUsageError) -> Self { + Self::RenderCommand(error.into()) + } +} + +/// Error encountered when performing a render pass. +#[derive(Clone, Debug, Error)] +#[error("Render pass error {scope}: {inner}")] +pub struct RenderPassError { + pub scope: PassErrorScope, + #[source] + inner: RenderPassErrorInner, +} + +impl<T, E> MapPassErr<T, RenderPassError> for Result<T, E> +where + E: Into<RenderPassErrorInner>, +{ + fn map_pass_err(self, scope: PassErrorScope) -> Result<T, RenderPassError> { + self.map_err(|inner| RenderPassError { + scope, + inner: inner.into(), + }) + } +} + +fn check_device_features( + actual: wgt::Features, + expected: wgt::Features, +) -> Result<(), RenderPassErrorInner> { + if !actual.contains(expected) { + Err(RenderPassErrorInner::MissingDeviceFeatures(expected)) + } else { + Ok(()) + } +} + +// Common routines between render/compute + +impl<G: GlobalIdentityHandlerFactory> Global<G> { + pub fn command_encoder_run_render_pass<B: GfxBackend>( + &self, + encoder_id: id::CommandEncoderId, + pass: &RenderPass, + ) -> Result<(), RenderPassError> { + self.command_encoder_run_render_pass_impl::<B>( + encoder_id, + pass.base.as_ref(), + &pass.color_targets, + pass.depth_stencil_target.as_ref(), + ) + } + + #[doc(hidden)] + pub fn command_encoder_run_render_pass_impl<B: GfxBackend>( + &self, + encoder_id: id::CommandEncoderId, + mut base: BasePassRef<RenderCommand>, + color_attachments: &[ColorAttachmentDescriptor], + depth_stencil_attachment: Option<&DepthStencilAttachmentDescriptor>, + ) -> Result<(), RenderPassError> { + span!(_guard, INFO, "CommandEncoder::run_render_pass"); + let scope = PassErrorScope::Pass(encoder_id); + + let hub = B::hub(self); + let mut token = Token::root(); + + let (device_guard, mut token) = hub.devices.read(&mut token); + let (mut cmb_guard, mut token) = hub.command_buffers.write(&mut token); + + let mut trackers = TrackerSet::new(B::VARIANT); + let cmd_buf = + CommandBuffer::get_encoder(&mut *cmb_guard, encoder_id).map_pass_err(scope)?; + let device = &device_guard[cmd_buf.device_id.value]; + let mut raw = device.cmd_allocator.extend(cmd_buf); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf.commands { + list.push(crate::device::trace::Command::RunRenderPass { + base: BasePass::from_ref(base), + target_colors: color_attachments.iter().cloned().collect(), + target_depth_stencil: depth_stencil_attachment.cloned(), + }); + } + + unsafe { + raw.begin_primary(hal::command::CommandBufferFlags::ONE_TIME_SUBMIT); + } + + let (bundle_guard, mut token) = hub.render_bundles.read(&mut token); + let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(&mut token); + let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); + let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); + let (buffer_guard, mut token) = hub.buffers.read(&mut token); + let (texture_guard, mut token) = hub.textures.read(&mut token); + let (view_guard, _) = hub.texture_views.read(&mut token); + + // We default to false intentionally, even if depth-stencil isn't used at all. + // This allows us to use the primary raw pipeline in `RenderPipeline`, + // instead of the special read-only one, which would be `None`. + let mut is_ds_read_only = false; + + struct RenderAttachment<'a> { + texture_id: &'a Stored<id::TextureId>, + selector: &'a TextureSelector, + previous_use: Option<TextureUse>, + new_use: TextureUse, + } + let mut render_attachments = AttachmentDataVec::<RenderAttachment>::new(); + + let mut attachment_width = None; + let mut attachment_height = None; + let mut valid_attachment = true; + + let context = { + use hal::device::Device as _; + + let sample_count_limit = device.hal_limits.framebuffer_color_sample_counts; + let base_trackers = &cmd_buf.trackers; + + let mut extent = None; + let mut sample_count = 0; + let mut depth_stencil_aspects = hal::format::Aspects::empty(); + let mut used_swap_chain = None::<Stored<id::SwapChainId>>; + + let mut add_view = |view: &TextureView<B>| { + if let Some(ex) = extent { + if ex != view.extent { + return Err(RenderPassErrorInner::ExtentStateMismatch { + state_extent: ex, + view_extent: view.extent, + }); + } + } else { + extent = Some(view.extent); + } + if sample_count == 0 { + sample_count = view.samples; + } else if sample_count != view.samples { + return Err(RenderPassErrorInner::SampleCountMismatch { + actual: view.samples, + expected: sample_count, + }); + } + Ok(()) + }; + + tracing::trace!( + "Encoding render pass begin in command buffer {:?}", + encoder_id + ); + let rp_key = { + let depth_stencil = match depth_stencil_attachment { + Some(at) => { + let view = trackers + .views + .use_extend(&*view_guard, at.attachment, (), ()) + .map_err(|_| RenderPassErrorInner::InvalidAttachment(at.attachment)) + .map_pass_err(scope)?; + add_view(view).map_pass_err(scope)?; + depth_stencil_aspects = view.aspects; + + let source_id = match view.inner { + TextureViewInner::Native { ref source_id, .. } => source_id, + TextureViewInner::SwapChain { .. } => { + return Err(RenderPassErrorInner::SwapChainImageAsDepthStencil) + .map_pass_err(scope) + } + }; + + // Using render pass for transition. + let previous_use = base_trackers + .textures + .query(source_id.value, view.selector.clone()); + let new_use = if at.is_read_only(view.aspects).map_pass_err(scope)? { + is_ds_read_only = true; + TextureUse::ATTACHMENT_READ + } else { + TextureUse::ATTACHMENT_WRITE + }; + render_attachments.push(RenderAttachment { + texture_id: source_id, + selector: &view.selector, + previous_use, + new_use, + }); + + let new_layout = conv::map_texture_state(new_use, view.aspects).1; + let old_layout = match previous_use { + Some(usage) => conv::map_texture_state(usage, view.aspects).1, + None => new_layout, + }; + + let ds_at = hal::pass::Attachment { + format: Some(conv::map_texture_format( + view.format, + device.private_features, + )), + samples: view.samples, + ops: conv::map_load_store_ops(&at.depth), + stencil_ops: conv::map_load_store_ops(&at.stencil), + layouts: old_layout..new_layout, + }; + Some((ds_at, new_layout)) + } + None => None, + }; + + let mut colors = ArrayVec::new(); + let mut resolves = ArrayVec::new(); + + for at in color_attachments { + let view = trackers + .views + .use_extend(&*view_guard, at.attachment, (), ()) + .map_err(|_| RenderPassErrorInner::InvalidAttachment(at.attachment)) + .map_pass_err(scope)?; + add_view(view).map_pass_err(scope)?; + + valid_attachment &= *attachment_width.get_or_insert(view.extent.width) + == view.extent.width + && *attachment_height.get_or_insert(view.extent.height) + == view.extent.height; + + let layouts = match view.inner { + TextureViewInner::Native { ref source_id, .. } => { + let previous_use = base_trackers + .textures + .query(source_id.value, view.selector.clone()); + let new_use = TextureUse::ATTACHMENT_WRITE; + render_attachments.push(RenderAttachment { + texture_id: source_id, + selector: &view.selector, + previous_use, + new_use, + }); + + let new_layout = + conv::map_texture_state(new_use, hal::format::Aspects::COLOR).1; + let old_layout = match previous_use { + Some(usage) => { + conv::map_texture_state(usage, hal::format::Aspects::COLOR).1 + } + None => new_layout, + }; + old_layout..new_layout + } + TextureViewInner::SwapChain { ref source_id, .. } => { + if let Some((ref sc_id, _)) = cmd_buf.used_swap_chain { + if source_id.value != sc_id.value { + return Err(RenderPassErrorInner::SwapChainMismatch) + .map_pass_err(scope); + } + } else { + assert!(used_swap_chain.is_none()); + used_swap_chain = Some(source_id.clone()); + } + + let end = hal::image::Layout::Present; + let start = match at.channel.load_op { + LoadOp::Clear => hal::image::Layout::Undefined, + LoadOp::Load => end, + }; + start..end + } + }; + + let color_at = hal::pass::Attachment { + format: Some(conv::map_texture_format( + view.format, + device.private_features, + )), + samples: view.samples, + ops: conv::map_load_store_ops(&at.channel), + stencil_ops: hal::pass::AttachmentOps::DONT_CARE, + layouts, + }; + colors.push((color_at, hal::image::Layout::ColorAttachmentOptimal)); + } + + if !valid_attachment { + return Err(RenderPassErrorInner::MismatchAttachments).map_pass_err(scope); + } + + for resolve_target in color_attachments.iter().flat_map(|at| at.resolve_target) { + let view = trackers + .views + .use_extend(&*view_guard, resolve_target, (), ()) + .map_err(|_| RenderPassErrorInner::InvalidAttachment(resolve_target)) + .map_pass_err(scope)?; + if extent != Some(view.extent) { + return Err(RenderPassErrorInner::ExtentStateMismatch { + state_extent: extent.unwrap_or_default(), + view_extent: view.extent, + }) + .map_pass_err(scope); + } + if view.samples != 1 { + return Err(RenderPassErrorInner::InvalidResolveTargetSampleCount) + .map_pass_err(scope); + } + if sample_count == 1 { + return Err(RenderPassErrorInner::InvalidResolveSourceSampleCount) + .map_pass_err(scope); + } + + let layouts = match view.inner { + TextureViewInner::Native { ref source_id, .. } => { + let previous_use = base_trackers + .textures + .query(source_id.value, view.selector.clone()); + let new_use = TextureUse::ATTACHMENT_WRITE; + render_attachments.push(RenderAttachment { + texture_id: source_id, + selector: &view.selector, + previous_use, + new_use, + }); + + let new_layout = + conv::map_texture_state(new_use, hal::format::Aspects::COLOR).1; + let old_layout = match previous_use { + Some(usage) => { + conv::map_texture_state(usage, hal::format::Aspects::COLOR).1 + } + None => new_layout, + }; + old_layout..new_layout + } + TextureViewInner::SwapChain { ref source_id, .. } => { + if let Some((ref sc_id, _)) = cmd_buf.used_swap_chain { + if source_id.value != sc_id.value { + return Err(RenderPassErrorInner::SwapChainMismatch) + .map_pass_err(scope); + } + } else { + assert!(used_swap_chain.is_none()); + used_swap_chain = Some(source_id.clone()); + } + hal::image::Layout::Undefined..hal::image::Layout::Present + } + }; + + let resolve_at = hal::pass::Attachment { + format: Some(conv::map_texture_format( + view.format, + device.private_features, + )), + samples: view.samples, + ops: hal::pass::AttachmentOps::new( + hal::pass::AttachmentLoadOp::DontCare, + hal::pass::AttachmentStoreOp::Store, + ), + stencil_ops: hal::pass::AttachmentOps::DONT_CARE, + layouts, + }; + resolves.push((resolve_at, hal::image::Layout::ColorAttachmentOptimal)); + } + + RenderPassKey { + colors, + resolves, + depth_stencil, + } + }; + + if sample_count & sample_count_limit == 0 { + return Err(RenderPassErrorInner::InvalidSampleCount(sample_count)) + .map_pass_err(scope); + } + + let mut render_pass_cache = device.render_passes.lock(); + let render_pass = match render_pass_cache.entry(rp_key.clone()) { + Entry::Occupied(e) => e.into_mut(), + Entry::Vacant(entry) => { + let color_ids: [hal::pass::AttachmentRef; MAX_COLOR_TARGETS] = [ + (0, hal::image::Layout::ColorAttachmentOptimal), + (1, hal::image::Layout::ColorAttachmentOptimal), + (2, hal::image::Layout::ColorAttachmentOptimal), + (3, hal::image::Layout::ColorAttachmentOptimal), + ]; + + let mut resolve_ids = ArrayVec::<[_; MAX_COLOR_TARGETS]>::new(); + let mut attachment_index = color_attachments.len(); + if color_attachments + .iter() + .any(|at| at.resolve_target.is_some()) + { + for ((i, at), &(_, layout)) in color_attachments + .iter() + .enumerate() + .zip(entry.key().resolves.iter()) + { + let real_attachment_index = match at.resolve_target { + Some(_) => attachment_index + i, + None => hal::pass::ATTACHMENT_UNUSED, + }; + resolve_ids.push((real_attachment_index, layout)); + } + attachment_index += color_attachments.len(); + } + + let depth_id = depth_stencil_attachment.map(|_| { + let usage = if is_ds_read_only { + TextureUse::ATTACHMENT_READ + } else { + TextureUse::ATTACHMENT_WRITE + }; + ( + attachment_index, + conv::map_texture_state(usage, depth_stencil_aspects).1, + ) + }); + + let subpass = hal::pass::SubpassDesc { + colors: &color_ids[..color_attachments.len()], + resolves: &resolve_ids, + depth_stencil: depth_id.as_ref(), + inputs: &[], + preserves: &[], + }; + let all = entry + .key() + .all() + .map(|(at, _)| at) + .collect::<AttachmentDataVec<_>>(); + + let pass = + unsafe { device.raw.create_render_pass(all, iter::once(subpass), &[]) } + .unwrap(); + entry.insert(pass) + } + }; + + let mut framebuffer_cache; + let fb_key = FramebufferKey { + colors: color_attachments + .iter() + .map(|at| id::Valid(at.attachment)) + .collect(), + resolves: color_attachments + .iter() + .filter_map(|at| at.resolve_target) + .map(id::Valid) + .collect(), + depth_stencil: depth_stencil_attachment.map(|at| id::Valid(at.attachment)), + }; + let context = RenderPassContext { + attachments: AttachmentData { + colors: fb_key + .colors + .iter() + .map(|&at| view_guard[at].format) + .collect(), + resolves: fb_key + .resolves + .iter() + .map(|&at| view_guard[at].format) + .collect(), + depth_stencil: fb_key.depth_stencil.map(|at| view_guard[at].format), + }, + sample_count, + }; + + let framebuffer = match used_swap_chain.take() { + Some(sc_id) => { + assert!(cmd_buf.used_swap_chain.is_none()); + // Always create a new framebuffer and delete it after presentation. + let attachments = fb_key + .all() + .map(|&id| match view_guard[id].inner { + TextureViewInner::Native { ref raw, .. } => raw, + TextureViewInner::SwapChain { ref image, .. } => Borrow::borrow(image), + }) + .collect::<AttachmentDataVec<_>>(); + let framebuffer = unsafe { + device + .raw + .create_framebuffer(&render_pass, attachments, extent.unwrap()) + .or(Err(RenderPassErrorInner::OutOfMemory)) + .map_pass_err(scope)? + }; + cmd_buf.used_swap_chain = Some((sc_id, framebuffer)); + &mut cmd_buf.used_swap_chain.as_mut().unwrap().1 + } + None => { + // Cache framebuffers by the device. + framebuffer_cache = device.framebuffers.lock(); + match framebuffer_cache.entry(fb_key) { + Entry::Occupied(e) => e.into_mut(), + Entry::Vacant(e) => { + let fb = { + let attachments = e + .key() + .all() + .map(|&id| match view_guard[id].inner { + TextureViewInner::Native { ref raw, .. } => raw, + TextureViewInner::SwapChain { ref image, .. } => { + Borrow::borrow(image) + } + }) + .collect::<AttachmentDataVec<_>>(); + unsafe { + device + .raw + .create_framebuffer( + &render_pass, + attachments, + extent.unwrap(), + ) + .or(Err(RenderPassErrorInner::OutOfMemory)) + .map_pass_err(scope)? + } + }; + e.insert(fb) + } + } + } + }; + + let rect = { + let ex = extent.unwrap(); + hal::pso::Rect { + x: 0, + y: 0, + w: ex.width as _, + h: ex.height as _, + } + }; + + let clear_values = color_attachments + .iter() + .zip(&rp_key.colors) + .flat_map(|(at, (rat, _layout))| { + match at.channel.load_op { + LoadOp::Load => None, + LoadOp::Clear => { + use hal::format::ChannelType; + //TODO: validate sign/unsign and normalized ranges of the color values + let value = match rat.format.unwrap().base_format().1 { + ChannelType::Unorm + | ChannelType::Snorm + | ChannelType::Ufloat + | ChannelType::Sfloat + | ChannelType::Uscaled + | ChannelType::Sscaled + | ChannelType::Srgb => hal::command::ClearColor { + float32: conv::map_color_f32(&at.channel.clear_value), + }, + ChannelType::Sint => hal::command::ClearColor { + sint32: conv::map_color_i32(&at.channel.clear_value), + }, + ChannelType::Uint => hal::command::ClearColor { + uint32: conv::map_color_u32(&at.channel.clear_value), + }, + }; + Some(hal::command::ClearValue { color: value }) + } + } + }) + .chain(depth_stencil_attachment.and_then(|at| { + match (at.depth.load_op, at.stencil.load_op) { + (LoadOp::Load, LoadOp::Load) => None, + (LoadOp::Clear, _) | (_, LoadOp::Clear) => { + let value = hal::command::ClearDepthStencil { + depth: at.depth.clear_value, + stencil: at.stencil.clear_value, + }; + Some(hal::command::ClearValue { + depth_stencil: value, + }) + } + } + })) + .collect::<ArrayVec<[_; MAX_COLOR_TARGETS + 1]>>(); + + unsafe { + raw.begin_render_pass( + render_pass, + framebuffer, + rect, + clear_values, + hal::command::SubpassContents::Inline, + ); + raw.set_scissors(0, iter::once(&rect)); + raw.set_viewports( + 0, + iter::once(hal::pso::Viewport { + rect, + depth: 0.0..1.0, + }), + ); + } + + context + }; + + let mut state = State { + pipeline_flags: PipelineFlags::empty(), + binder: Binder::new(cmd_buf.limits.max_bind_groups), + blend_color: OptionalState::Unused, + stencil_reference: 0, + pipeline: StateChange::new(), + index: IndexState::default(), + vertex: VertexState::default(), + debug_scope_depth: 0, + }; + let mut temp_offsets = Vec::new(); + + for command in base.commands { + match *command { + RenderCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group_id, + } => { + let scope = PassErrorScope::SetBindGroup(bind_group_id); + let max_bind_groups = device.limits.max_bind_groups; + if (index as u32) >= max_bind_groups { + return Err(RenderCommandError::BindGroupIndexOutOfRange { + index, + max: max_bind_groups, + }) + .map_pass_err(scope); + } + + temp_offsets.clear(); + temp_offsets + .extend_from_slice(&base.dynamic_offsets[..num_dynamic_offsets as usize]); + base.dynamic_offsets = &base.dynamic_offsets[num_dynamic_offsets as usize..]; + + let bind_group = trackers + .bind_groups + .use_extend(&*bind_group_guard, bind_group_id, (), ()) + .unwrap(); + bind_group + .validate_dynamic_bindings(&temp_offsets) + .map_pass_err(scope)?; + + trackers + .merge_extend(&bind_group.used) + .map_pass_err(scope)?; + + if let Some((pipeline_layout_id, follow_ups)) = state.binder.provide_entry( + index as usize, + id::Valid(bind_group_id), + bind_group, + &temp_offsets, + ) { + let bind_groups = iter::once(bind_group.raw.raw()) + .chain( + follow_ups + .clone() + .map(|(bg_id, _)| bind_group_guard[bg_id].raw.raw()), + ) + .collect::<ArrayVec<[_; MAX_BIND_GROUPS]>>(); + temp_offsets.extend(follow_ups.flat_map(|(_, offsets)| offsets)); + unsafe { + raw.bind_graphics_descriptor_sets( + &pipeline_layout_guard[pipeline_layout_id].raw, + index as usize, + bind_groups, + &temp_offsets, + ); + } + }; + } + RenderCommand::SetPipeline(pipeline_id) => { + let scope = PassErrorScope::SetPipelineRender(pipeline_id); + if state.pipeline.set_and_check_redundant(pipeline_id) { + continue; + } + + let pipeline = trackers + .render_pipes + .use_extend(&*pipeline_guard, pipeline_id, (), ()) + .map_err(|_| RenderCommandError::InvalidPipeline(pipeline_id)) + .map_pass_err(scope)?; + + context + .check_compatible(&pipeline.pass_context) + .map_err(RenderCommandError::IncompatiblePipeline) + .map_pass_err(scope)?; + + state.pipeline_flags = pipeline.flags; + + if pipeline.flags.contains(PipelineFlags::WRITES_DEPTH_STENCIL) + && is_ds_read_only + { + return Err(RenderCommandError::IncompatibleReadOnlyDepthStencil) + .map_pass_err(scope); + } + + state + .blend_color + .require(pipeline.flags.contains(PipelineFlags::BLEND_COLOR)); + + unsafe { + raw.bind_graphics_pipeline(&pipeline.raw); + } + + if pipeline.flags.contains(PipelineFlags::STENCIL_REFERENCE) { + unsafe { + raw.set_stencil_reference( + hal::pso::Face::all(), + state.stencil_reference, + ); + } + } + + // Rebind resource + if state.binder.pipeline_layout_id != Some(pipeline.layout_id.value) { + let pipeline_layout = &pipeline_layout_guard[pipeline.layout_id.value]; + + state.binder.change_pipeline_layout( + &*pipeline_layout_guard, + pipeline.layout_id.value, + ); + + let mut is_compatible = true; + + for (index, (entry, &bgl_id)) in state + .binder + .entries + .iter_mut() + .zip(&pipeline_layout.bind_group_layout_ids) + .enumerate() + { + match entry.expect_layout(bgl_id) { + LayoutChange::Match(bg_id, offsets) if is_compatible => { + let desc_set = bind_group_guard[bg_id].raw.raw(); + unsafe { + raw.bind_graphics_descriptor_sets( + &pipeline_layout.raw, + index, + iter::once(desc_set), + offsets.iter().cloned(), + ); + } + } + LayoutChange::Match(..) | LayoutChange::Unchanged => {} + LayoutChange::Mismatch => { + is_compatible = false; + } + } + } + + // Clear push constant ranges + let non_overlapping = super::bind::compute_nonoverlapping_ranges( + &pipeline_layout.push_constant_ranges, + ); + for range in non_overlapping { + let offset = range.range.start; + let size_bytes = range.range.end - offset; + super::push_constant_clear( + offset, + size_bytes, + |clear_offset, clear_data| unsafe { + raw.push_graphics_constants( + &pipeline_layout.raw, + conv::map_shader_stage_flags(range.stages), + clear_offset, + clear_data, + ); + }, + ); + } + } + + // Rebind index buffer if the index format has changed with the pipeline switch + if state.index.format != pipeline.index_format { + state.index.format = pipeline.index_format; + state.index.update_limit(); + + if let Some((buffer_id, ref range)) = state.index.bound_buffer_view { + let &(ref buffer, _) = buffer_guard[buffer_id].raw.as_ref().unwrap(); + + let range = hal::buffer::SubRange { + offset: range.start, + size: Some(range.end - range.start), + }; + let index_type = conv::map_index_format(state.index.format); + unsafe { + raw.bind_index_buffer(buffer, range, index_type); + } + } + } + // Update vertex buffer limits + for (vbs, &(stride, rate)) in + state.vertex.inputs.iter_mut().zip(&pipeline.vertex_strides) + { + vbs.stride = stride; + vbs.rate = rate; + } + let vertex_strides_len = pipeline.vertex_strides.len(); + for vbs in state.vertex.inputs.iter_mut().skip(vertex_strides_len) { + vbs.stride = 0; + vbs.rate = InputStepMode::Vertex; + } + state.vertex.update_limits(); + } + RenderCommand::SetIndexBuffer { + buffer_id, + offset, + size, + } => { + let scope = PassErrorScope::SetIndexBuffer(buffer_id); + let buffer = trackers + .buffers + .use_extend(&*buffer_guard, buffer_id, (), BufferUse::INDEX) + .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) + .map_pass_err(scope)?; + check_buffer_usage(buffer.usage, BufferUsage::INDEX).map_pass_err(scope)?; + let &(ref buf_raw, _) = buffer + .raw + .as_ref() + .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) + .map_pass_err(scope)?; + + let end = match size { + Some(s) => offset + s.get(), + None => buffer.size, + }; + state.index.bound_buffer_view = Some((id::Valid(buffer_id), offset..end)); + state.index.update_limit(); + + let range = hal::buffer::SubRange { + offset, + size: Some(end - offset), + }; + let index_type = conv::map_index_format(state.index.format); + unsafe { + raw.bind_index_buffer(buf_raw, range, index_type); + } + } + RenderCommand::SetVertexBuffer { + slot, + buffer_id, + offset, + size, + } => { + let scope = PassErrorScope::SetVertexBuffer(buffer_id); + let buffer = trackers + .buffers + .use_extend(&*buffer_guard, buffer_id, (), BufferUse::VERTEX) + .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) + .map_pass_err(scope)?; + check_buffer_usage(buffer.usage, BufferUsage::VERTEX).map_pass_err(scope)?; + let &(ref buf_raw, _) = buffer + .raw + .as_ref() + .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) + .map_pass_err(scope)?; + + let empty_slots = (1 + slot as usize).saturating_sub(state.vertex.inputs.len()); + state + .vertex + .inputs + .extend(iter::repeat(VertexBufferState::EMPTY).take(empty_slots)); + state.vertex.inputs[slot as usize].total_size = match size { + Some(s) => s.get(), + None => buffer.size - offset, + }; + + let range = hal::buffer::SubRange { + offset, + size: size.map(|s| s.get()), + }; + unsafe { + raw.bind_vertex_buffers(slot, iter::once((buf_raw, range))); + } + state.vertex.update_limits(); + } + RenderCommand::SetBlendColor(ref color) => { + state.blend_color = OptionalState::Set; + unsafe { + raw.set_blend_constants(conv::map_color_f32(color)); + } + } + RenderCommand::SetStencilReference(value) => { + state.stencil_reference = value; + if state + .pipeline_flags + .contains(PipelineFlags::STENCIL_REFERENCE) + { + unsafe { + raw.set_stencil_reference(hal::pso::Face::all(), value); + } + } + } + RenderCommand::SetViewport { + ref rect, + depth_min, + depth_max, + } => { + let scope = PassErrorScope::SetViewport; + use std::{convert::TryFrom, i16}; + if rect.w <= 0.0 + || rect.h <= 0.0 + || depth_min < 0.0 + || depth_min > 1.0 + || depth_max < 0.0 + || depth_max > 1.0 + { + return Err(RenderCommandError::InvalidViewport).map_pass_err(scope); + } + let r = hal::pso::Rect { + x: i16::try_from(rect.x.round() as i64).unwrap_or(0), + y: i16::try_from(rect.y.round() as i64).unwrap_or(0), + w: i16::try_from(rect.w.round() as i64).unwrap_or(i16::MAX), + h: i16::try_from(rect.h.round() as i64).unwrap_or(i16::MAX), + }; + unsafe { + raw.set_viewports( + 0, + iter::once(hal::pso::Viewport { + rect: r, + depth: depth_min..depth_max, + }), + ); + } + } + RenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset, + } => { + let scope = PassErrorScope::SetPushConstant; + let values_offset = values_offset + .ok_or(RenderPassErrorInner::InvalidValuesOffset) + .map_pass_err(scope)?; + + let end_offset_bytes = offset + size_bytes; + let values_end_offset = + (values_offset + size_bytes / wgt::PUSH_CONSTANT_ALIGNMENT) as usize; + let data_slice = + &base.push_constant_data[(values_offset as usize)..values_end_offset]; + + let pipeline_layout_id = state + .binder + .pipeline_layout_id + .ok_or(DrawError::MissingPipeline) + .map_pass_err(scope)?; + let pipeline_layout = &pipeline_layout_guard[pipeline_layout_id]; + + pipeline_layout + .validate_push_constant_ranges(stages, offset, end_offset_bytes) + .map_err(RenderCommandError::from) + .map_pass_err(scope)?; + + unsafe { + raw.push_graphics_constants( + &pipeline_layout.raw, + conv::map_shader_stage_flags(stages), + offset, + data_slice, + ) + } + } + RenderCommand::SetScissor(ref rect) => { + let scope = PassErrorScope::SetScissorRect; + use std::{convert::TryFrom, i16}; + if rect.w == 0 + || rect.h == 0 + || rect.x + rect.w > attachment_width.unwrap() + || rect.y + rect.h > attachment_height.unwrap() + { + return Err(RenderCommandError::InvalidScissorRect).map_pass_err(scope); + } + let r = hal::pso::Rect { + x: i16::try_from(rect.x).unwrap_or(0), + y: i16::try_from(rect.y).unwrap_or(0), + w: i16::try_from(rect.w).unwrap_or(i16::MAX), + h: i16::try_from(rect.h).unwrap_or(i16::MAX), + }; + unsafe { + raw.set_scissors(0, iter::once(r)); + } + } + RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + } => { + let scope = PassErrorScope::Draw; + state.is_ready().map_pass_err(scope)?; + let last_vertex = first_vertex + vertex_count; + let vertex_limit = state.vertex.vertex_limit; + if last_vertex > vertex_limit { + return Err(DrawError::VertexBeyondLimit { + last_vertex, + vertex_limit, + }) + .map_pass_err(scope); + } + let last_instance = first_instance + instance_count; + let instance_limit = state.vertex.instance_limit; + if last_instance > instance_limit { + return Err(DrawError::InstanceBeyondLimit { + last_instance, + instance_limit, + }) + .map_pass_err(scope); + } + + unsafe { + raw.draw( + first_vertex..first_vertex + vertex_count, + first_instance..first_instance + instance_count, + ); + } + } + RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + } => { + let scope = PassErrorScope::DrawIndexed; + state.is_ready().map_pass_err(scope)?; + + //TODO: validate that base_vertex + max_index() is within the provided range + let last_index = first_index + index_count; + let index_limit = state.index.limit; + if last_index > index_limit { + return Err(DrawError::IndexBeyondLimit { + last_index, + index_limit, + }) + .map_pass_err(scope); + } + let last_instance = first_instance + instance_count; + let instance_limit = state.vertex.instance_limit; + if last_instance > instance_limit { + return Err(DrawError::InstanceBeyondLimit { + last_instance, + instance_limit, + }) + .map_pass_err(scope); + } + + unsafe { + raw.draw_indexed( + first_index..first_index + index_count, + base_vertex, + first_instance..first_instance + instance_count, + ); + } + } + RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count, + indexed, + } => { + let scope = if indexed { + PassErrorScope::DrawIndexedIndirect + } else { + PassErrorScope::DrawIndirect + }; + state.is_ready().map_pass_err(scope)?; + + let stride = match indexed { + false => 16, + true => 20, + }; + + if count.is_some() { + check_device_features(device.features, wgt::Features::MULTI_DRAW_INDIRECT) + .map_pass_err(scope)?; + } + + let indirect_buffer = trackers + .buffers + .use_extend(&*buffer_guard, buffer_id, (), BufferUse::INDIRECT) + .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) + .map_pass_err(scope)?; + check_buffer_usage(indirect_buffer.usage, BufferUsage::INDIRECT) + .map_pass_err(scope)?; + let &(ref indirect_raw, _) = indirect_buffer + .raw + .as_ref() + .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) + .map_pass_err(scope)?; + + let actual_count = count.map_or(1, |c| c.get()); + + let begin_offset = offset; + let end_offset = offset + stride * actual_count as u64; + if end_offset > indirect_buffer.size { + return Err(RenderPassErrorInner::IndirectBufferOverrun { + offset, + count, + begin_offset, + end_offset, + buffer_size: indirect_buffer.size, + }) + .map_pass_err(scope); + } + + match indexed { + false => unsafe { + raw.draw_indirect(indirect_raw, offset, actual_count, stride as u32); + }, + true => unsafe { + raw.draw_indexed_indirect( + indirect_raw, + offset, + actual_count, + stride as u32, + ); + }, + } + } + RenderCommand::MultiDrawIndirectCount { + buffer_id, + offset, + count_buffer_id, + count_buffer_offset, + max_count, + indexed, + } => { + let scope = if indexed { + PassErrorScope::DrawIndexedIndirect + } else { + PassErrorScope::DrawIndirect + }; + state.is_ready().map_pass_err(scope)?; + + let stride = match indexed { + false => 16, + true => 20, + }; + + check_device_features( + device.features, + wgt::Features::MULTI_DRAW_INDIRECT_COUNT, + ) + .map_pass_err(scope)?; + + let indirect_buffer = trackers + .buffers + .use_extend(&*buffer_guard, buffer_id, (), BufferUse::INDIRECT) + .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) + .map_pass_err(scope)?; + check_buffer_usage(indirect_buffer.usage, BufferUsage::INDIRECT) + .map_pass_err(scope)?; + let &(ref indirect_raw, _) = indirect_buffer + .raw + .as_ref() + .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) + .map_pass_err(scope)?; + + let count_buffer = trackers + .buffers + .use_extend(&*buffer_guard, count_buffer_id, (), BufferUse::INDIRECT) + .map_err(|e| RenderCommandError::Buffer(count_buffer_id, e)) + .map_pass_err(scope)?; + check_buffer_usage(count_buffer.usage, BufferUsage::INDIRECT) + .map_pass_err(scope)?; + let &(ref count_raw, _) = count_buffer + .raw + .as_ref() + .ok_or(RenderCommandError::DestroyedBuffer(count_buffer_id)) + .map_pass_err(scope)?; + + let begin_offset = offset; + let end_offset = offset + stride * max_count as u64; + if end_offset > indirect_buffer.size { + return Err(RenderPassErrorInner::IndirectBufferOverrun { + offset, + count: None, + begin_offset, + end_offset, + buffer_size: indirect_buffer.size, + }) + .map_pass_err(scope); + } + + let begin_count_offset = count_buffer_offset; + let end_count_offset = count_buffer_offset + 4; + if end_count_offset > count_buffer.size { + return Err(RenderPassErrorInner::IndirectCountBufferOverrun { + begin_count_offset, + end_count_offset, + count_buffer_size: count_buffer.size, + }) + .map_pass_err(scope); + } + + match indexed { + false => unsafe { + raw.draw_indirect_count( + indirect_raw, + offset, + count_raw, + count_buffer_offset, + max_count, + stride as u32, + ); + }, + true => unsafe { + raw.draw_indexed_indirect_count( + indirect_raw, + offset, + count_raw, + count_buffer_offset, + max_count, + stride as u32, + ); + }, + } + } + RenderCommand::PushDebugGroup { color, len } => { + state.debug_scope_depth += 1; + let label = str::from_utf8(&base.string_data[..len]).unwrap(); + unsafe { + raw.begin_debug_marker(label, color); + } + base.string_data = &base.string_data[len..]; + } + RenderCommand::PopDebugGroup => { + let scope = PassErrorScope::PopDebugGroup; + if state.debug_scope_depth == 0 { + return Err(RenderPassErrorInner::InvalidPopDebugGroup).map_pass_err(scope); + } + state.debug_scope_depth -= 1; + unsafe { + raw.end_debug_marker(); + } + } + RenderCommand::InsertDebugMarker { color, len } => { + let label = str::from_utf8(&base.string_data[..len]).unwrap(); + unsafe { + raw.insert_debug_marker(label, color); + } + base.string_data = &base.string_data[len..]; + } + RenderCommand::ExecuteBundle(bundle_id) => { + let scope = PassErrorScope::ExecuteBundle; + let bundle = trackers + .bundles + .use_extend(&*bundle_guard, bundle_id, (), ()) + .unwrap(); + + context + .check_compatible(&bundle.context) + .map_err(RenderPassErrorInner::IncompatibleRenderBundle) + .map_pass_err(scope)?; + + unsafe { + bundle.execute( + &mut raw, + &*pipeline_layout_guard, + &*bind_group_guard, + &*pipeline_guard, + &*buffer_guard, + ) + } + .map_err(|e| match e { + ExecutionError::DestroyedBuffer(id) => { + RenderCommandError::DestroyedBuffer(id) + } + }) + .map_pass_err(scope)?; + + trackers.merge_extend(&bundle.used).map_pass_err(scope)?; + state.reset_bundle(); + } + } + } + + tracing::trace!("Merging {:?} with the render pass", encoder_id); + unsafe { + raw.end_render_pass(); + } + + for ra in render_attachments { + let texture = &texture_guard[ra.texture_id.value]; + check_texture_usage(texture.usage, TextureUsage::RENDER_ATTACHMENT) + .map_pass_err(scope)?; + + // the tracker set of the pass is always in "extend" mode + trackers + .textures + .change_extend( + ra.texture_id.value, + &ra.texture_id.ref_count, + ra.selector.clone(), + ra.new_use, + ) + .unwrap(); + + if let Some(usage) = ra.previous_use { + // Make the attachment tracks to be aware of the internal + // transition done by the render pass, by registering the + // previous usage as the initial state. + trackers + .textures + .prepend( + ra.texture_id.value, + &ra.texture_id.ref_count, + ra.selector.clone(), + usage, + ) + .unwrap(); + } + } + + super::CommandBuffer::insert_barriers( + cmd_buf.raw.last_mut().unwrap(), + &mut cmd_buf.trackers, + &trackers, + &*buffer_guard, + &*texture_guard, + ); + unsafe { + cmd_buf.raw.last_mut().unwrap().finish(); + } + cmd_buf.raw.push(raw); + + Ok(()) + } +} + +pub mod render_ffi { + use super::{ + super::{Rect, RenderCommand}, + RenderPass, + }; + use crate::{id, span, RawString}; + use std::{convert::TryInto, ffi, num::NonZeroU32, slice}; + use wgt::{BufferAddress, BufferSize, Color, DynamicOffset}; + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `offset_length` elements. + // TODO: There might be other safety issues, such as using the unsafe + // `RawPass::encode` and `RawPass::encode_slice`. + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_pass_set_bind_group( + pass: &mut RenderPass, + index: u32, + bind_group_id: id::BindGroupId, + offsets: *const DynamicOffset, + offset_length: usize, + ) { + span!(_guard, DEBUG, "RenderPass::set_bind_group"); + pass.base.commands.push(RenderCommand::SetBindGroup { + index: index.try_into().unwrap(), + num_dynamic_offsets: offset_length.try_into().unwrap(), + bind_group_id, + }); + pass.base + .dynamic_offsets + .extend_from_slice(slice::from_raw_parts(offsets, offset_length)); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_pipeline( + pass: &mut RenderPass, + pipeline_id: id::RenderPipelineId, + ) { + span!(_guard, DEBUG, "RenderPass::set_pipeline"); + pass.base + .commands + .push(RenderCommand::SetPipeline(pipeline_id)); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_index_buffer( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option<BufferSize>, + ) { + span!(_guard, DEBUG, "RenderPass::set_index_buffer"); + pass.base.commands.push(RenderCommand::SetIndexBuffer { + buffer_id, + offset, + size, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_vertex_buffer( + pass: &mut RenderPass, + slot: u32, + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option<BufferSize>, + ) { + span!(_guard, DEBUG, "RenderPass::set_vertex_buffer"); + pass.base.commands.push(RenderCommand::SetVertexBuffer { + slot, + buffer_id, + offset, + size, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_blend_color(pass: &mut RenderPass, color: &Color) { + span!(_guard, DEBUG, "RenderPass::set_blend_color"); + pass.base + .commands + .push(RenderCommand::SetBlendColor(*color)); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_stencil_reference(pass: &mut RenderPass, value: u32) { + span!(_guard, DEBUG, "RenderPass::set_stencil_buffer"); + pass.base + .commands + .push(RenderCommand::SetStencilReference(value)); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_viewport( + pass: &mut RenderPass, + x: f32, + y: f32, + w: f32, + h: f32, + depth_min: f32, + depth_max: f32, + ) { + span!(_guard, DEBUG, "RenderPass::set_viewport"); + pass.base.commands.push(RenderCommand::SetViewport { + rect: Rect { x, y, w, h }, + depth_min, + depth_max, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_set_scissor_rect( + pass: &mut RenderPass, + x: u32, + y: u32, + w: u32, + h: u32, + ) { + span!(_guard, DEBUG, "RenderPass::set_scissor_rect"); + pass.base + .commands + .push(RenderCommand::SetScissor(Rect { x, y, w, h })); + } + + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_pass_set_push_constants( + pass: &mut RenderPass, + stages: wgt::ShaderStage, + offset: u32, + size_bytes: u32, + data: *const u8, + ) { + span!(_guard, DEBUG, "RenderPass::set_push_constants"); + assert_eq!( + offset & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant offset must be aligned to 4 bytes." + ); + assert_eq!( + size_bytes & (wgt::PUSH_CONSTANT_ALIGNMENT - 1), + 0, + "Push constant size must be aligned to 4 bytes." + ); + let data_slice = slice::from_raw_parts(data, size_bytes as usize); + let value_offset = pass.base.push_constant_data.len().try_into().expect( + "Ran out of push constant space. Don't set 4gb of push constants per RenderPass.", + ); + + pass.base.push_constant_data.extend( + data_slice + .chunks_exact(wgt::PUSH_CONSTANT_ALIGNMENT as usize) + .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])), + ); + + pass.base.commands.push(RenderCommand::SetPushConstant { + stages, + offset, + size_bytes, + values_offset: Some(value_offset), + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_draw( + pass: &mut RenderPass, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, + ) { + span!(_guard, DEBUG, "RenderPass::draw"); + pass.base.commands.push(RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_draw_indexed( + pass: &mut RenderPass, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, + ) { + span!(_guard, DEBUG, "RenderPass::draw_indexed"); + pass.base.commands.push(RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_draw_indirect( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + span!(_guard, DEBUG, "RenderPass::draw_indirect"); + pass.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: false, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_draw_indexed_indirect( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + span!(_guard, DEBUG, "RenderPass::draw_indexed_indirect"); + pass.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: None, + indexed: true, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_multi_draw_indirect( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + count: u32, + ) { + span!(_guard, DEBUG, "RenderPass::multi_draw_indirect"); + pass.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: NonZeroU32::new(count), + indexed: false, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_multi_draw_indexed_indirect( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + count: u32, + ) { + span!(_guard, DEBUG, "RenderPass::multi_draw_indexed_indirect"); + pass.base.commands.push(RenderCommand::MultiDrawIndirect { + buffer_id, + offset, + count: NonZeroU32::new(count), + indexed: true, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_multi_draw_indirect_count( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + count_buffer_id: id::BufferId, + count_buffer_offset: BufferAddress, + max_count: u32, + ) { + span!(_guard, DEBUG, "RenderPass::multi_draw_indirect_count"); + pass.base + .commands + .push(RenderCommand::MultiDrawIndirectCount { + buffer_id, + offset, + count_buffer_id, + count_buffer_offset, + max_count, + indexed: false, + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_multi_draw_indexed_indirect_count( + pass: &mut RenderPass, + buffer_id: id::BufferId, + offset: BufferAddress, + count_buffer_id: id::BufferId, + count_buffer_offset: BufferAddress, + max_count: u32, + ) { + span!( + _guard, + DEBUG, + "RenderPass::multi_draw_indexed_indirect_count" + ); + pass.base + .commands + .push(RenderCommand::MultiDrawIndirectCount { + buffer_id, + offset, + count_buffer_id, + count_buffer_offset, + max_count, + indexed: true, + }); + } + + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_pass_push_debug_group( + pass: &mut RenderPass, + label: RawString, + color: u32, + ) { + span!(_guard, DEBUG, "RenderPass::push_debug_group"); + let bytes = ffi::CStr::from_ptr(label).to_bytes(); + pass.base.string_data.extend_from_slice(bytes); + + pass.base.commands.push(RenderCommand::PushDebugGroup { + color, + len: bytes.len(), + }); + } + + #[no_mangle] + pub extern "C" fn wgpu_render_pass_pop_debug_group(pass: &mut RenderPass) { + span!(_guard, DEBUG, "RenderPass::pop_debug_group"); + pass.base.commands.push(RenderCommand::PopDebugGroup); + } + + #[no_mangle] + pub unsafe extern "C" fn wgpu_render_pass_insert_debug_marker( + pass: &mut RenderPass, + label: RawString, + color: u32, + ) { + span!(_guard, DEBUG, "RenderPass::insert_debug_marker"); + let bytes = ffi::CStr::from_ptr(label).to_bytes(); + pass.base.string_data.extend_from_slice(bytes); + + pass.base.commands.push(RenderCommand::InsertDebugMarker { + color, + len: bytes.len(), + }); + } + + #[no_mangle] + pub unsafe fn wgpu_render_pass_execute_bundles( + pass: &mut RenderPass, + render_bundle_ids: *const id::RenderBundleId, + render_bundle_ids_length: usize, + ) { + span!(_guard, DEBUG, "RenderPass::execute_bundles"); + for &bundle_id in slice::from_raw_parts(render_bundle_ids, render_bundle_ids_length) { + pass.base + .commands + .push(RenderCommand::ExecuteBundle(bundle_id)); + } + } +} diff --git a/gfx/wgpu/wgpu-core/src/command/transfer.rs b/gfx/wgpu/wgpu-core/src/command/transfer.rs new file mode 100644 index 0000000000..98ab294cfc --- /dev/null +++ b/gfx/wgpu/wgpu-core/src/command/transfer.rs @@ -0,0 +1,789 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#[cfg(feature = "trace")] +use crate::device::trace::Command as TraceCommand; +use crate::{ + command::{CommandBuffer, CommandEncoderError}, + conv, + device::{all_buffer_stages, all_image_stages}, + hub::{GfxBackend, Global, GlobalIdentityHandlerFactory, Storage, Token}, + id::{BufferId, CommandEncoderId, TextureId}, + resource::{BufferUse, Texture, TextureErrorDimension, TextureUse}, + span, + track::TextureSelector, +}; + +use hal::command::CommandBuffer as _; +use thiserror::Error; +use wgt::{BufferAddress, BufferUsage, Extent3d, TextureUsage}; + +use std::iter; + +pub(crate) const BITS_PER_BYTE: u32 = 8; + +pub type BufferCopyView = wgt::BufferCopyView<BufferId>; +pub type TextureCopyView = wgt::TextureCopyView<TextureId>; + +#[derive(Clone, Debug)] +pub enum CopySide { + Source, + Destination, +} + +/// Error encountered while attempting a data transfer. +#[derive(Clone, Debug, Error)] +pub enum TransferError { + #[error("buffer {0:?} is invalid or destroyed")] + InvalidBuffer(BufferId), + #[error("texture {0:?} is invalid or destroyed")] + InvalidTexture(TextureId), + #[error("Source and destination cannot be the same buffer")] + SameSourceDestinationBuffer, + #[error("source buffer/texture is missing the `COPY_SRC` usage flag")] + MissingCopySrcUsageFlag, + #[error("destination buffer/texture is missing the `COPY_DST` usage flag")] + MissingCopyDstUsageFlag(Option<BufferId>, Option<TextureId>), + #[error("copy of {start_offset}..{end_offset} would end up overruning the bounds of the {side:?} buffer of size {buffer_size}")] + BufferOverrun { + start_offset: BufferAddress, + end_offset: BufferAddress, + buffer_size: BufferAddress, + side: CopySide, + }, + #[error("copy of {dimension:?} {start_offset}..{end_offset} would end up overruning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")] + TextureOverrun { + start_offset: u32, + end_offset: u32, + texture_size: u32, + dimension: TextureErrorDimension, + side: CopySide, + }, + #[error("buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")] + UnalignedBufferOffset(BufferAddress), + #[error("copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")] + UnalignedCopySize(BufferAddress), + #[error("copy width is not a multiple of block width")] + UnalignedCopyWidth, + #[error("copy height is not a multiple of block height")] + UnalignedCopyHeight, + #[error("copy origin's x component is not a multiple of block width")] + UnalignedCopyOriginX, + #[error("copy origin's y component is not a multiple of block height")] + UnalignedCopyOriginY, + #[error("bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")] + UnalignedBytesPerRow, + #[error("number of rows per image is not a multiple of block height")] + UnalignedRowsPerImage, + #[error("number of bytes per row is less than the number of bytes in a complete row")] + InvalidBytesPerRow, + #[error("image is 1D and the copy height and depth are not both set to 1")] + InvalidCopySize, + #[error("number of rows per image is invalid")] + InvalidRowsPerImage, + #[error("source and destination layers have different aspects")] + MismatchedAspects, + #[error("copying from textures with format {0:?} is forbidden")] + CopyFromForbiddenTextureFormat(wgt::TextureFormat), + #[error("copying to textures with format {0:?} is forbidden")] + CopyToForbiddenTextureFormat(wgt::TextureFormat), +} + +/// Error encountered while attempting to do a copy on a command encoder. +#[derive(Clone, Debug, Error)] +pub enum CopyError { + #[error(transparent)] + Encoder(#[from] CommandEncoderError), + #[error("Copy error")] + Transfer(#[from] TransferError), +} + +//TODO: we currently access each texture twice for a transfer, +// once only to get the aspect flags, which is unfortunate. +pub(crate) fn texture_copy_view_to_hal<B: hal::Backend>( + view: &TextureCopyView, + size: &Extent3d, + texture_guard: &Storage<Texture<B>, TextureId>, +) -> Result< + ( + hal::image::SubresourceLayers, + TextureSelector, + hal::image::Offset, + ), + TransferError, +> { + let texture = texture_guard + .get(view.texture) + .map_err(|_| TransferError::InvalidTexture(view.texture))?; + + let level = view.mip_level as hal::image::Level; + let (layer, layer_count, z) = match texture.dimension { + wgt::TextureDimension::D1 | wgt::TextureDimension::D2 => ( + view.origin.z as hal::image::Layer, + size.depth as hal::image::Layer, + 0, + ), + wgt::TextureDimension::D3 => (0, 1, view.origin.z as i32), + }; + + // TODO: Can't satisfy clippy here unless we modify + // `TextureSelector` to use `std::ops::RangeBounds`. + #[allow(clippy::range_plus_one)] + Ok(( + hal::image::SubresourceLayers { + aspects: texture.aspects, + level, + layers: layer..layer + layer_count, + }, + TextureSelector { + levels: level..level + 1, + layers: layer..layer + layer_count, + }, + hal::image::Offset { + x: view.origin.x as i32, + y: view.origin.y as i32, + z, + }, + )) +} + +/// Function copied with minor modifications from webgpu standard https://gpuweb.github.io/gpuweb/#valid-texture-copy-range +pub(crate) fn validate_linear_texture_data( + layout: &wgt::TextureDataLayout, + format: wgt::TextureFormat, + buffer_size: BufferAddress, + buffer_side: CopySide, + bytes_per_block: BufferAddress, + copy_size: &Extent3d, +) -> Result<(), TransferError> { + // Convert all inputs to BufferAddress (u64) to prevent overflow issues + let copy_width = copy_size.width as BufferAddress; + let copy_height = copy_size.height as BufferAddress; + let copy_depth = copy_size.depth as BufferAddress; + + let offset = layout.offset; + let rows_per_image = layout.rows_per_image as BufferAddress; + let bytes_per_row = layout.bytes_per_row as BufferAddress; + + let (block_width, block_height) = conv::texture_block_size(format); + let block_width = block_width as BufferAddress; + let block_height = block_height as BufferAddress; + let block_size = bytes_per_block; + + if copy_width % block_width != 0 { + return Err(TransferError::UnalignedCopyWidth); + } + if copy_height % block_height != 0 { + return Err(TransferError::UnalignedCopyHeight); + } + if rows_per_image % block_height != 0 { + return Err(TransferError::UnalignedRowsPerImage); + } + + let bytes_in_a_complete_row = block_size * copy_width / block_width; + let required_bytes_in_copy = if copy_width == 0 || copy_height == 0 || copy_depth == 0 { + 0 + } else { + let actual_rows_per_image = if rows_per_image == 0 { + copy_height + } else { + rows_per_image + }; + let texel_block_rows_per_image = actual_rows_per_image / block_height; + let bytes_per_image = bytes_per_row * texel_block_rows_per_image; + let bytes_in_last_slice = + bytes_per_row * (copy_height / block_height - 1) + bytes_in_a_complete_row; + bytes_per_image * (copy_depth - 1) + bytes_in_last_slice + }; + + if rows_per_image != 0 && rows_per_image < copy_height { + return Err(TransferError::InvalidRowsPerImage); + } + if offset + required_bytes_in_copy > buffer_size { + return Err(TransferError::BufferOverrun { + start_offset: offset, + end_offset: offset + required_bytes_in_copy, + buffer_size, + side: buffer_side, + }); + } + if offset % block_size != 0 { + return Err(TransferError::UnalignedBufferOffset(offset)); + } + if copy_height > 1 && bytes_per_row < bytes_in_a_complete_row { + return Err(TransferError::InvalidBytesPerRow); + } + if copy_depth > 1 && rows_per_image == 0 { + return Err(TransferError::InvalidRowsPerImage); + } + Ok(()) +} + +/// Function copied with minor modifications from webgpu standard https://gpuweb.github.io/gpuweb/#valid-texture-copy-range +pub(crate) fn validate_texture_copy_range( + texture_copy_view: &TextureCopyView, + texture_format: wgt::TextureFormat, + texture_dimension: hal::image::Kind, + texture_side: CopySide, + copy_size: &Extent3d, +) -> Result<(), TransferError> { + let (block_width, block_height) = conv::texture_block_size(texture_format); + + let mut extent = texture_dimension.level_extent(texture_copy_view.mip_level as u8); + match texture_dimension { + hal::image::Kind::D1(..) => { + if (copy_size.height, copy_size.depth) != (1, 1) { + return Err(TransferError::InvalidCopySize); + } + } + hal::image::Kind::D2(_, _, array_layers, _) => { + extent.depth = array_layers as u32; + } + hal::image::Kind::D3(..) => {} + }; + + let x_copy_max = texture_copy_view.origin.x + copy_size.width; + if x_copy_max > extent.width { + return Err(TransferError::TextureOverrun { + start_offset: texture_copy_view.origin.x, + end_offset: x_copy_max, + texture_size: extent.width, + dimension: TextureErrorDimension::X, + side: texture_side, + }); + } + let y_copy_max = texture_copy_view.origin.y + copy_size.height; + if y_copy_max > extent.height { + return Err(TransferError::TextureOverrun { + start_offset: texture_copy_view.origin.y, + end_offset: y_copy_max, + texture_size: extent.height, + dimension: TextureErrorDimension::Y, + side: texture_side, + }); + } + let z_copy_max = texture_copy_view.origin.z + copy_size.depth; + if z_copy_max > extent.depth { + return Err(TransferError::TextureOverrun { + start_offset: texture_copy_view.origin.z, + end_offset: z_copy_max, + texture_size: extent.depth, + dimension: TextureErrorDimension::Z, + side: texture_side, + }); + } + + if texture_copy_view.origin.x % block_width != 0 { + return Err(TransferError::UnalignedCopyOriginX); + } + if texture_copy_view.origin.y % block_height != 0 { + return Err(TransferError::UnalignedCopyOriginY); + } + if copy_size.width % block_width != 0 { + return Err(TransferError::UnalignedCopyWidth); + } + if copy_size.height % block_height != 0 { + return Err(TransferError::UnalignedCopyHeight); + } + Ok(()) +} + +impl<G: GlobalIdentityHandlerFactory> Global<G> { + pub fn command_encoder_copy_buffer_to_buffer<B: GfxBackend>( + &self, + command_encoder_id: CommandEncoderId, + source: BufferId, + source_offset: BufferAddress, + destination: BufferId, + destination_offset: BufferAddress, + size: BufferAddress, + ) -> Result<(), CopyError> { + span!(_guard, INFO, "CommandEncoder::copy_buffer_to_buffer"); + + if source == destination { + Err(TransferError::SameSourceDestinationBuffer)? + } + let hub = B::hub(self); + let mut token = Token::root(); + + let (mut cmd_buf_guard, mut token) = hub.command_buffers.write(&mut token); + let cmd_buf = CommandBuffer::get_encoder(&mut *cmd_buf_guard, command_encoder_id)?; + let (buffer_guard, _) = hub.buffers.read(&mut token); + // we can't hold both src_pending and dst_pending in scope because they + // borrow the buffer tracker mutably... + let mut barriers = Vec::new(); + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf.commands { + list.push(TraceCommand::CopyBufferToBuffer { + src: source, + src_offset: source_offset, + dst: destination, + dst_offset: destination_offset, + size, + }); + } + + let (src_buffer, src_pending) = cmd_buf + .trackers + .buffers + .use_replace(&*buffer_guard, source, (), BufferUse::COPY_SRC) + .map_err(TransferError::InvalidBuffer)?; + let &(ref src_raw, _) = src_buffer + .raw + .as_ref() + .ok_or(TransferError::InvalidBuffer(source))?; + if !src_buffer.usage.contains(BufferUsage::COPY_SRC) { + Err(TransferError::MissingCopySrcUsageFlag)? + } + barriers.extend(src_pending.map(|pending| pending.into_hal(src_buffer))); + + let (dst_buffer, dst_pending) = cmd_buf + .trackers + .buffers + .use_replace(&*buffer_guard, destination, (), BufferUse::COPY_DST) + .map_err(TransferError::InvalidBuffer)?; + let &(ref dst_raw, _) = dst_buffer + .raw + .as_ref() + .ok_or(TransferError::InvalidBuffer(destination))?; + if !dst_buffer.usage.contains(BufferUsage::COPY_DST) { + Err(TransferError::MissingCopyDstUsageFlag( + Some(destination), + None, + ))? + } + barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_buffer))); + + if size % wgt::COPY_BUFFER_ALIGNMENT != 0 { + Err(TransferError::UnalignedCopySize(size))? + } + if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 { + Err(TransferError::UnalignedBufferOffset(source_offset))? + } + if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 { + Err(TransferError::UnalignedBufferOffset(destination_offset))? + } + + let source_end_offset = source_offset + size; + let destination_end_offset = destination_offset + size; + if source_end_offset > src_buffer.size { + Err(TransferError::BufferOverrun { + start_offset: source_offset, + end_offset: source_end_offset, + buffer_size: src_buffer.size, + side: CopySide::Source, + })? + } + if destination_end_offset > dst_buffer.size { + Err(TransferError::BufferOverrun { + start_offset: destination_offset, + end_offset: destination_end_offset, + buffer_size: dst_buffer.size, + side: CopySide::Destination, + })? + } + + if size == 0 { + tracing::trace!("Ignoring copy_buffer_to_buffer of size 0"); + return Ok(()); + } + + let region = hal::command::BufferCopy { + src: source_offset, + dst: destination_offset, + size, + }; + let cmb_raw = cmd_buf.raw.last_mut().unwrap(); + unsafe { + cmb_raw.pipeline_barrier( + all_buffer_stages()..hal::pso::PipelineStage::TRANSFER, + hal::memory::Dependencies::empty(), + barriers, + ); + cmb_raw.copy_buffer(src_raw, dst_raw, iter::once(region)); + } + Ok(()) + } + + pub fn command_encoder_copy_buffer_to_texture<B: GfxBackend>( + &self, + command_encoder_id: CommandEncoderId, + source: &BufferCopyView, + destination: &TextureCopyView, + copy_size: &Extent3d, + ) -> Result<(), CopyError> { + span!(_guard, INFO, "CommandEncoder::copy_buffer_to_texture"); + + let hub = B::hub(self); + let mut token = Token::root(); + let (mut cmd_buf_guard, mut token) = hub.command_buffers.write(&mut token); + let cmd_buf = CommandBuffer::get_encoder(&mut *cmd_buf_guard, command_encoder_id)?; + let (buffer_guard, mut token) = hub.buffers.read(&mut token); + let (texture_guard, _) = hub.textures.read(&mut token); + let (dst_layers, dst_selector, dst_offset) = + texture_copy_view_to_hal(destination, copy_size, &*texture_guard)?; + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf.commands { + list.push(TraceCommand::CopyBufferToTexture { + src: source.clone(), + dst: destination.clone(), + size: *copy_size, + }); + } + + if copy_size.width == 0 || copy_size.height == 0 || copy_size.width == 0 { + tracing::trace!("Ignoring copy_buffer_to_texture of size 0"); + return Ok(()); + } + + let (src_buffer, src_pending) = cmd_buf + .trackers + .buffers + .use_replace(&*buffer_guard, source.buffer, (), BufferUse::COPY_SRC) + .map_err(TransferError::InvalidBuffer)?; + let &(ref src_raw, _) = src_buffer + .raw + .as_ref() + .ok_or(TransferError::InvalidBuffer(source.buffer))?; + if !src_buffer.usage.contains(BufferUsage::COPY_SRC) { + Err(TransferError::MissingCopySrcUsageFlag)? + } + let src_barriers = src_pending.map(|pending| pending.into_hal(src_buffer)); + + let (dst_texture, dst_pending) = cmd_buf + .trackers + .textures + .use_replace( + &*texture_guard, + destination.texture, + dst_selector, + TextureUse::COPY_DST, + ) + .unwrap(); + let &(ref dst_raw, _) = dst_texture + .raw + .as_ref() + .ok_or(TransferError::InvalidTexture(destination.texture))?; + if !dst_texture.usage.contains(TextureUsage::COPY_DST) { + Err(TransferError::MissingCopyDstUsageFlag( + None, + Some(destination.texture), + ))? + } + let dst_barriers = dst_pending.map(|pending| pending.into_hal(dst_texture)); + + let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT; + let bytes_per_block = conv::map_texture_format(dst_texture.format, cmd_buf.private_features) + .surface_desc() + .bits as u32 + / BITS_PER_BYTE; + let src_bytes_per_row = source.layout.bytes_per_row; + if bytes_per_row_alignment % bytes_per_block != 0 { + Err(TransferError::UnalignedBytesPerRow)? + } + if src_bytes_per_row % bytes_per_row_alignment != 0 { + Err(TransferError::UnalignedBytesPerRow)? + } + validate_texture_copy_range( + destination, + dst_texture.format, + dst_texture.kind, + CopySide::Destination, + copy_size, + )?; + validate_linear_texture_data( + &source.layout, + dst_texture.format, + src_buffer.size, + CopySide::Source, + bytes_per_block as BufferAddress, + copy_size, + )?; + + let (block_width, _) = conv::texture_block_size(dst_texture.format); + if !conv::is_valid_copy_dst_texture_format(dst_texture.format) { + Err(TransferError::CopyToForbiddenTextureFormat( + dst_texture.format, + ))? + } + + let buffer_width = (source.layout.bytes_per_row / bytes_per_block) * block_width; + let region = hal::command::BufferImageCopy { + buffer_offset: source.layout.offset, + buffer_width, + buffer_height: source.layout.rows_per_image, + image_layers: dst_layers, + image_offset: dst_offset, + image_extent: conv::map_extent(copy_size, dst_texture.dimension), + }; + let cmb_raw = cmd_buf.raw.last_mut().unwrap(); + unsafe { + cmb_raw.pipeline_barrier( + all_buffer_stages() | all_image_stages()..hal::pso::PipelineStage::TRANSFER, + hal::memory::Dependencies::empty(), + src_barriers.chain(dst_barriers), + ); + cmb_raw.copy_buffer_to_image( + src_raw, + dst_raw, + hal::image::Layout::TransferDstOptimal, + iter::once(region), + ); + } + Ok(()) + } + + pub fn command_encoder_copy_texture_to_buffer<B: GfxBackend>( + &self, + command_encoder_id: CommandEncoderId, + source: &TextureCopyView, + destination: &BufferCopyView, + copy_size: &Extent3d, + ) -> Result<(), CopyError> { + span!(_guard, INFO, "CommandEncoder::copy_texture_to_buffer"); + + let hub = B::hub(self); + let mut token = Token::root(); + let (mut cmd_buf_guard, mut token) = hub.command_buffers.write(&mut token); + let cmd_buf = CommandBuffer::get_encoder(&mut *cmd_buf_guard, command_encoder_id)?; + let (buffer_guard, mut token) = hub.buffers.read(&mut token); + let (texture_guard, _) = hub.textures.read(&mut token); + let (src_layers, src_selector, src_offset) = + texture_copy_view_to_hal(source, copy_size, &*texture_guard)?; + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf.commands { + list.push(TraceCommand::CopyTextureToBuffer { + src: source.clone(), + dst: destination.clone(), + size: *copy_size, + }); + } + + if copy_size.width == 0 || copy_size.height == 0 || copy_size.width == 0 { + tracing::trace!("Ignoring copy_texture_to_buffer of size 0"); + return Ok(()); + } + + let (src_texture, src_pending) = cmd_buf + .trackers + .textures + .use_replace( + &*texture_guard, + source.texture, + src_selector, + TextureUse::COPY_SRC, + ) + .unwrap(); + let &(ref src_raw, _) = src_texture + .raw + .as_ref() + .ok_or(TransferError::InvalidTexture(source.texture))?; + if !src_texture.usage.contains(TextureUsage::COPY_SRC) { + Err(TransferError::MissingCopySrcUsageFlag)? + } + let src_barriers = src_pending.map(|pending| pending.into_hal(src_texture)); + + let (dst_buffer, dst_barriers) = cmd_buf + .trackers + .buffers + .use_replace(&*buffer_guard, destination.buffer, (), BufferUse::COPY_DST) + .map_err(TransferError::InvalidBuffer)?; + let &(ref dst_raw, _) = dst_buffer + .raw + .as_ref() + .ok_or(TransferError::InvalidBuffer(destination.buffer))?; + if !dst_buffer.usage.contains(BufferUsage::COPY_DST) { + Err(TransferError::MissingCopyDstUsageFlag( + Some(destination.buffer), + None, + ))? + } + let dst_barrier = dst_barriers.map(|pending| pending.into_hal(dst_buffer)); + + let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT; + let bytes_per_block = conv::map_texture_format(src_texture.format, cmd_buf.private_features) + .surface_desc() + .bits as u32 + / BITS_PER_BYTE; + let dst_bytes_per_row = destination.layout.bytes_per_row; + if bytes_per_row_alignment % bytes_per_block != 0 { + Err(TransferError::UnalignedBytesPerRow)? + } + if dst_bytes_per_row % bytes_per_row_alignment != 0 { + Err(TransferError::UnalignedBytesPerRow)? + } + validate_texture_copy_range( + source, + src_texture.format, + src_texture.kind, + CopySide::Source, + copy_size, + )?; + validate_linear_texture_data( + &destination.layout, + src_texture.format, + dst_buffer.size, + CopySide::Destination, + bytes_per_block as BufferAddress, + copy_size, + )?; + + let (block_width, _) = conv::texture_block_size(src_texture.format); + if !conv::is_valid_copy_src_texture_format(src_texture.format) { + Err(TransferError::CopyFromForbiddenTextureFormat( + src_texture.format, + ))? + } + + let buffer_width = (destination.layout.bytes_per_row / bytes_per_block) * block_width; + let region = hal::command::BufferImageCopy { + buffer_offset: destination.layout.offset, + buffer_width, + buffer_height: destination.layout.rows_per_image, + image_layers: src_layers, + image_offset: src_offset, + image_extent: conv::map_extent(copy_size, src_texture.dimension), + }; + let cmb_raw = cmd_buf.raw.last_mut().unwrap(); + unsafe { + cmb_raw.pipeline_barrier( + all_buffer_stages() | all_image_stages()..hal::pso::PipelineStage::TRANSFER, + hal::memory::Dependencies::empty(), + src_barriers.chain(dst_barrier), + ); + cmb_raw.copy_image_to_buffer( + src_raw, + hal::image::Layout::TransferSrcOptimal, + dst_raw, + iter::once(region), + ); + } + Ok(()) + } + + pub fn command_encoder_copy_texture_to_texture<B: GfxBackend>( + &self, + command_encoder_id: CommandEncoderId, + source: &TextureCopyView, + destination: &TextureCopyView, + copy_size: &Extent3d, + ) -> Result<(), CopyError> { + span!(_guard, INFO, "CommandEncoder::copy_texture_to_texture"); + + let hub = B::hub(self); + let mut token = Token::root(); + + let (mut cmd_buf_guard, mut token) = hub.command_buffers.write(&mut token); + let cmd_buf = CommandBuffer::get_encoder(&mut *cmd_buf_guard, command_encoder_id)?; + let (_, mut token) = hub.buffers.read(&mut token); // skip token + let (texture_guard, _) = hub.textures.read(&mut token); + // we can't hold both src_pending and dst_pending in scope because they + // borrow the buffer tracker mutably... + let mut barriers = Vec::new(); + let (src_layers, src_selector, src_offset) = + texture_copy_view_to_hal(source, copy_size, &*texture_guard)?; + let (dst_layers, dst_selector, dst_offset) = + texture_copy_view_to_hal(destination, copy_size, &*texture_guard)?; + if src_layers.aspects != dst_layers.aspects { + Err(TransferError::MismatchedAspects)? + } + + #[cfg(feature = "trace")] + if let Some(ref mut list) = cmd_buf.commands { + list.push(TraceCommand::CopyTextureToTexture { + src: source.clone(), + dst: destination.clone(), + size: *copy_size, + }); + } + + if copy_size.width == 0 || copy_size.height == 0 || copy_size.width == 0 { + tracing::trace!("Ignoring copy_texture_to_texture of size 0"); + return Ok(()); + } + + let (src_texture, src_pending) = cmd_buf + .trackers + .textures + .use_replace( + &*texture_guard, + source.texture, + src_selector, + TextureUse::COPY_SRC, + ) + .unwrap(); + let &(ref src_raw, _) = src_texture + .raw + .as_ref() + .ok_or(TransferError::InvalidTexture(source.texture))?; + if !src_texture.usage.contains(TextureUsage::COPY_SRC) { + Err(TransferError::MissingCopySrcUsageFlag)? + } + barriers.extend(src_pending.map(|pending| pending.into_hal(src_texture))); + + let (dst_texture, dst_pending) = cmd_buf + .trackers + .textures + .use_replace( + &*texture_guard, + destination.texture, + dst_selector, + TextureUse::COPY_DST, + ) + .unwrap(); + let &(ref dst_raw, _) = dst_texture + .raw + .as_ref() + .ok_or(TransferError::InvalidTexture(destination.texture))?; + if !dst_texture.usage.contains(TextureUsage::COPY_DST) { + Err(TransferError::MissingCopyDstUsageFlag( + None, + Some(destination.texture), + ))? + } + barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_texture))); + + validate_texture_copy_range( + source, + src_texture.format, + src_texture.kind, + CopySide::Source, + copy_size, + )?; + validate_texture_copy_range( + destination, + dst_texture.format, + dst_texture.kind, + CopySide::Destination, + copy_size, + )?; + + let region = hal::command::ImageCopy { + src_subresource: src_layers, + src_offset, + dst_subresource: dst_layers, + dst_offset, + extent: conv::map_extent(copy_size, src_texture.dimension), + }; + let cmb_raw = cmd_buf.raw.last_mut().unwrap(); + unsafe { + cmb_raw.pipeline_barrier( + all_image_stages()..hal::pso::PipelineStage::TRANSFER, + hal::memory::Dependencies::empty(), + barriers, + ); + cmb_raw.copy_image( + src_raw, + hal::image::Layout::TransferSrcOptimal, + dst_raw, + hal::image::Layout::TransferDstOptimal, + iter::once(region), + ); + } + Ok(()) + } +} |