/*! Texture Trackers * * Texture trackers are significantly more complicated than * the buffer trackers because textures can be in a "complex" * state where each individual subresource can potentially be * in a different state from every other subtresource. These * complex states are stored separately from the simple states * because they are signifignatly more difficult to track and * most resources spend the vast majority of their lives in * simple states. * * There are two special texture usages: `UNKNOWN` and `UNINITIALIZED`. * - `UNKNOWN` is only used in complex states and is used to signify * that the complex state does not know anything about those subresources. * It cannot leak into transitions, it is invalid to transition into UNKNOWN * state. * - `UNINITIALIZED` is used in both simple and complex states to mean the texture * is known to be in some undefined state. Any transition away from UNINITIALIZED * will treat the contents as junk. !*/ use super::{range::RangedStates, PendingTransition, PendingTransitionList, ResourceTracker}; use crate::{ hal_api::HalApi, id::TextureId, resource::{Resource, Texture, TextureInner}, snatch::SnatchGuard, track::{ invalid_resource_state, skip_barrier, ResourceMetadata, ResourceMetadataProvider, ResourceUses, UsageConflict, }, }; use hal::TextureUses; use arrayvec::ArrayVec; use naga::FastHashMap; use parking_lot::Mutex; use wgt::{strict_assert, strict_assert_eq}; use std::{borrow::Cow, iter, marker::PhantomData, ops::Range, sync::Arc, vec::Drain}; /// Specifies a particular set of subresources in a texture. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TextureSelector { pub mips: Range, pub layers: Range, } impl ResourceUses for TextureUses { const EXCLUSIVE: Self = Self::EXCLUSIVE; type Selector = TextureSelector; fn bits(self) -> u16 { Self::bits(&self) } fn all_ordered(self) -> bool { Self::ORDERED.contains(self) } fn any_exclusive(self) -> bool { self.intersects(Self::EXCLUSIVE) } } /// Represents the complex state of textures where every subresource is potentially /// in a different state. #[derive(Clone, Debug, Default, PartialEq)] struct ComplexTextureState { mips: ArrayVec, { hal::MAX_MIP_LEVELS as usize }>, } impl ComplexTextureState { /// Creates complex texture state for the given sizes. /// /// This state will be initialized with the UNKNOWN state, a special state /// which means the trakcer knows nothing about the state. fn new(mip_level_count: u32, array_layer_count: u32) -> Self { Self { mips: iter::repeat_with(|| { RangedStates::from_range(0..array_layer_count, TextureUses::UNKNOWN) }) .take(mip_level_count as usize) .collect(), } } /// Initialize a complex state from a selector representing the full size of the texture /// and an iterator of a selector and a texture use, specifying a usage for a specific /// set of subresources. /// /// [`Self::to_selector_state_iter`] can be used to create such an iterator. /// /// # Safety /// /// All selectors in the iterator must be inside of the full_range selector. /// /// The full range selector must have mips and layers start at 0. unsafe fn from_selector_state_iter( full_range: TextureSelector, state_iter: impl Iterator, ) -> Self { strict_assert_eq!(full_range.layers.start, 0); strict_assert_eq!(full_range.mips.start, 0); let mut complex = ComplexTextureState::new(full_range.mips.len() as u32, full_range.layers.len() as u32); for (selector, desired_state) in state_iter { strict_assert!(selector.layers.end <= full_range.layers.end); strict_assert!(selector.mips.end <= full_range.mips.end); // This should only ever happen with a wgpu bug, but let's just double // check that resource states don't have any conflicts. strict_assert_eq!(invalid_resource_state(desired_state), false); let mips = selector.mips.start as usize..selector.mips.end as usize; for mip in unsafe { complex.mips.get_unchecked_mut(mips) } { for &mut (_, ref mut state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { *state = desired_state; } } } complex } /// Convert a complex state into an iterator over all states stored. /// /// [`Self::from_selector_state_iter`] can be used to consume such an iterator. fn to_selector_state_iter( &self, ) -> impl Iterator + Clone + '_ { self.mips.iter().enumerate().flat_map(|(mip, inner)| { let mip = mip as u32; { inner.iter().map(move |&(ref layers, inner)| { ( TextureSelector { mips: mip..mip + 1, layers: layers.clone(), }, inner, ) }) } }) } } #[derive(Debug)] struct TextureBindGroupStateData { selector: Option, texture: Arc>, usage: TextureUses, } /// Stores all the textures that a bind group stores. #[derive(Debug)] pub(crate) struct TextureBindGroupState { textures: Mutex>>, } impl TextureBindGroupState { pub fn new() -> Self { Self { textures: Mutex::new(Vec::new()), } } /// Optimize the texture bind group state by sorting it by ID. /// /// When this list of states is merged into a tracker, the memory /// accesses will be in a constant ascending order. pub(crate) fn optimize(&self) { let mut textures = self.textures.lock(); textures.sort_unstable_by_key(|v| v.texture.as_info().id().unzip().0); } /// Returns a list of all textures tracked. May contain duplicates. pub fn drain_resources(&self) -> impl Iterator>> + '_ { let mut textures = self.textures.lock(); textures .drain(..) .map(|v| v.texture) .collect::>() .into_iter() } /// Adds the given resource with the given state. pub fn add_single<'a>( &self, texture: &'a Arc>, selector: Option, state: TextureUses, ) -> Option<&'a Arc>> { let mut textures = self.textures.lock(); textures.push(TextureBindGroupStateData { selector, texture: texture.clone(), usage: state, }); Some(texture) } } /// Container for corresponding simple and complex texture states. #[derive(Debug)] pub(crate) struct TextureStateSet { simple: Vec, complex: FastHashMap, } impl TextureStateSet { fn new() -> Self { Self { simple: Vec::new(), complex: FastHashMap::default(), } } fn clear(&mut self) { self.simple.clear(); self.complex.clear(); } fn set_size(&mut self, size: usize) { self.simple.resize(size, TextureUses::UNINITIALIZED); } } /// Stores all texture state within a single usage scope. #[derive(Debug)] pub(crate) struct TextureUsageScope { set: TextureStateSet, metadata: ResourceMetadata>, } impl TextureUsageScope { pub fn new() -> Self { Self { set: TextureStateSet::new(), metadata: ResourceMetadata::new(), } } fn tracker_assert_in_bounds(&self, index: usize) { self.metadata.tracker_assert_in_bounds(index); strict_assert!(index < self.set.simple.len()); strict_assert!(if self.metadata.contains(index) && self.set.simple[index] == TextureUses::COMPLEX { self.set.complex.contains_key(&index) } else { true }); } /// Sets the size of all the vectors inside the tracker. /// /// Must be called with the highest possible Texture ID before /// all unsafe functions are called. pub fn set_size(&mut self, size: usize) { self.set.set_size(size); self.metadata.set_size(size); } /// Drains all textures tracked. pub(crate) fn drain_resources(&mut self) -> impl Iterator>> + '_ { let resources = self.metadata.drain_resources(); self.set.clear(); resources.into_iter() } /// Returns true if the tracker owns no resources. /// /// This is a O(n) operation. pub(crate) fn is_empty(&self) -> bool { self.metadata.is_empty() } /// Merge the list of texture states in the given usage scope into this UsageScope. /// /// If any of the resulting states is invalid, stops the merge and returns a usage /// conflict with the details of the invalid state. /// /// If the given tracker uses IDs higher than the length of internal vectors, /// the vectors will be extended. A call to set_size is not needed. pub fn merge_usage_scope(&mut self, scope: &Self) -> Result<(), UsageConflict> { let incoming_size = scope.set.simple.len(); if incoming_size > self.set.simple.len() { self.set_size(incoming_size); } for index in scope.metadata.owned_indices() { self.tracker_assert_in_bounds(index); scope.tracker_assert_in_bounds(index); let texture_selector = unsafe { &scope.metadata.get_resource_unchecked(index).full_range }; unsafe { insert_or_merge( texture_selector, &mut self.set, &mut self.metadata, index, TextureStateProvider::TextureSet { set: &scope.set }, ResourceMetadataProvider::Indirect { metadata: &scope.metadata, }, )? }; } Ok(()) } /// Merge the list of texture states in the given bind group into this usage scope. /// /// If any of the resulting states is invalid, stops the merge and returns a usage /// conflict with the details of the invalid state. /// /// Because bind groups do not check if the union of all their states is valid, /// this method is allowed to return Err on the first bind group bound. /// /// # Safety /// /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this /// method is called. pub unsafe fn merge_bind_group( &mut self, bind_group: &TextureBindGroupState, ) -> Result<(), UsageConflict> { let textures = bind_group.textures.lock(); for t in &*textures { unsafe { self.merge_single(&t.texture, t.selector.clone(), t.usage)? }; } Ok(()) } /// Merge a single state into the UsageScope. /// /// If the resulting state is invalid, returns a usage /// conflict with the details of the invalid state. /// /// # Safety /// /// Unlike other trackers whose merge_single is safe, this method is only /// called where there is already other unsafe tracking functions active, /// so we can prove this unsafe "for free". /// /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this /// method is called. pub unsafe fn merge_single( &mut self, texture: &Arc>, selector: Option, new_state: TextureUses, ) -> Result<(), UsageConflict> { let index = texture.as_info().id().unzip().0 as usize; self.tracker_assert_in_bounds(index); let texture_selector = &texture.full_range; unsafe { insert_or_merge( texture_selector, &mut self.set, &mut self.metadata, index, TextureStateProvider::from_option(selector, new_state), ResourceMetadataProvider::Direct { resource: Cow::Borrowed(texture), }, )? }; Ok(()) } } /// Stores all texture state within a command buffer or device. pub(crate) struct TextureTracker { start_set: TextureStateSet, end_set: TextureStateSet, metadata: ResourceMetadata>, temp: Vec>, _phantom: PhantomData, } impl ResourceTracker> for TextureTracker { /// Try to remove the given resource from the tracker iff we have the last reference to the /// resource and the epoch matches. /// /// Returns true if the resource was removed or if not existing in metadata. /// /// If the ID is higher than the length of internal vectors, /// false will be returned. fn remove_abandoned(&mut self, id: TextureId) -> bool { let index = id.unzip().0 as usize; if index > self.metadata.size() { return false; } self.tracker_assert_in_bounds(index); unsafe { if self.metadata.contains_unchecked(index) { let existing_ref_count = self.metadata.get_ref_count_unchecked(index); //RefCount 2 means that resource is hold just by DeviceTracker and this suspected resource itself //so it's already been released from user and so it's not inside Registry\Storage if existing_ref_count <= 2 { self.start_set.complex.remove(&index); self.end_set.complex.remove(&index); self.metadata.remove(index); log::trace!("Texture {:?} is not tracked anymore", id,); return true; } else { log::trace!( "Texture {:?} is still referenced from {}", id, existing_ref_count ); return false; } } } true } } impl TextureTracker { pub fn new() -> Self { Self { start_set: TextureStateSet::new(), end_set: TextureStateSet::new(), metadata: ResourceMetadata::new(), temp: Vec::new(), _phantom: PhantomData, } } fn tracker_assert_in_bounds(&self, index: usize) { self.metadata.tracker_assert_in_bounds(index); strict_assert!(index < self.start_set.simple.len()); strict_assert!(index < self.end_set.simple.len()); strict_assert!(if self.metadata.contains(index) && self.start_set.simple[index] == TextureUses::COMPLEX { self.start_set.complex.contains_key(&index) } else { true }); strict_assert!(if self.metadata.contains(index) && self.end_set.simple[index] == TextureUses::COMPLEX { self.end_set.complex.contains_key(&index) } else { true }); } /// Sets the size of all the vectors inside the tracker. /// /// Must be called with the highest possible Texture ID before /// all unsafe functions are called. pub fn set_size(&mut self, size: usize) { self.start_set.set_size(size); self.end_set.set_size(size); self.metadata.set_size(size); } /// Extend the vectors to let the given index be valid. fn allow_index(&mut self, index: usize) { if index >= self.start_set.simple.len() { self.set_size(index + 1); } } /// Returns a list of all textures tracked. pub fn used_resources(&self) -> impl Iterator>> + '_ { self.metadata.owned_resources() } /// Drain all currently pending transitions. pub fn drain_transitions<'a>( &'a mut self, snatch_guard: &'a SnatchGuard<'a>, ) -> (PendingTransitionList, Vec>>) { let mut textures = Vec::new(); let transitions = self .temp .drain(..) .map(|pending| { let tex = unsafe { self.metadata.get_resource_unchecked(pending.id as _) }; textures.push(tex.inner.get(snatch_guard)); pending }) .collect(); (transitions, textures) } /// Inserts a single texture and a state into the resource tracker. /// /// If the resource already exists in the tracker, this will panic. /// /// If the ID is higher than the length of internal vectors, /// the vectors will be extended. A call to set_size is not needed. pub fn insert_single(&mut self, id: TextureId, resource: Arc>, usage: TextureUses) { let index = id.unzip().0 as usize; self.allow_index(index); self.tracker_assert_in_bounds(index); unsafe { let currently_owned = self.metadata.contains_unchecked(index); if currently_owned { panic!("Tried to insert texture already tracked"); } insert( None, Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, index, TextureStateProvider::KnownSingle { state: usage }, None, ResourceMetadataProvider::Direct { resource: Cow::Owned(resource), }, ) }; } /// Sets the state of a single texture. /// /// If a transition is needed to get the texture into the given state, that transition /// is returned. /// /// If the ID is higher than the length of internal vectors, /// the vectors will be extended. A call to set_size is not needed. pub fn set_single( &mut self, texture: &Arc>, selector: TextureSelector, new_state: TextureUses, ) -> Option>> { let index = texture.as_info().id().unzip().0 as usize; self.allow_index(index); self.tracker_assert_in_bounds(index); unsafe { insert_or_barrier_update( &texture.full_range, Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, index, TextureStateProvider::Selector { selector, state: new_state, }, None, ResourceMetadataProvider::Direct { resource: Cow::Owned(texture.clone()), }, &mut self.temp, ) } Some(self.temp.drain(..)) } /// Sets the given state for all texture in the given tracker. /// /// If a transition is needed to get the texture into the needed state, /// those transitions are stored within the tracker. A subsequent /// call to [`Self::drain_transitions`] is needed to get those transitions. /// /// If the ID is higher than the length of internal vectors, /// the vectors will be extended. A call to set_size is not needed. pub fn set_from_tracker(&mut self, tracker: &Self) { let incoming_size = tracker.start_set.simple.len(); if incoming_size > self.start_set.simple.len() { self.set_size(incoming_size); } for index in tracker.metadata.owned_indices() { self.tracker_assert_in_bounds(index); tracker.tracker_assert_in_bounds(index); unsafe { let texture_selector = &tracker.metadata.get_resource_unchecked(index).full_range; insert_or_barrier_update( texture_selector, Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, index, TextureStateProvider::TextureSet { set: &tracker.start_set, }, Some(TextureStateProvider::TextureSet { set: &tracker.end_set, }), ResourceMetadataProvider::Indirect { metadata: &tracker.metadata, }, &mut self.temp, ); } } } /// Sets the given state for all textures in the given UsageScope. /// /// If a transition is needed to get the textures into the needed state, /// those transitions are stored within the tracker. A subsequent /// call to [`Self::drain_transitions`] is needed to get those transitions. /// /// If the ID is higher than the length of internal vectors, /// the vectors will be extended. A call to set_size is not needed. pub fn set_from_usage_scope(&mut self, scope: &TextureUsageScope) { let incoming_size = scope.set.simple.len(); if incoming_size > self.start_set.simple.len() { self.set_size(incoming_size); } for index in scope.metadata.owned_indices() { self.tracker_assert_in_bounds(index); scope.tracker_assert_in_bounds(index); unsafe { let texture_selector = &scope.metadata.get_resource_unchecked(index).full_range; insert_or_barrier_update( texture_selector, Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, index, TextureStateProvider::TextureSet { set: &scope.set }, None, ResourceMetadataProvider::Indirect { metadata: &scope.metadata, }, &mut self.temp, ); } } } /// Iterates through all textures in the given bind group and adopts /// the state given for those textures in the UsageScope. It also /// removes all touched textures from the usage scope. /// /// If a transition is needed to get the textures into the needed state, /// those transitions are stored within the tracker. A subsequent /// call to [`Self::drain_transitions`] is needed to get those transitions. /// /// This is a really funky method used by Compute Passes to generate /// barriers after a call to dispatch without needing to iterate /// over all elements in the usage scope. We use each the /// bind group as a source of which IDs to look at. The bind groups /// must have first been added to the usage scope. /// /// # Safety /// /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this /// method is called. pub unsafe fn set_and_remove_from_usage_scope_sparse( &mut self, scope: &mut TextureUsageScope, bind_group_state: &TextureBindGroupState, ) { let incoming_size = scope.set.simple.len(); if incoming_size > self.start_set.simple.len() { self.set_size(incoming_size); } let textures = bind_group_state.textures.lock(); for t in textures.iter() { let index = t.texture.as_info().id().unzip().0 as usize; scope.tracker_assert_in_bounds(index); if unsafe { !scope.metadata.contains_unchecked(index) } { continue; } let texture_selector = &t.texture.full_range; unsafe { insert_or_barrier_update( texture_selector, Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, index, TextureStateProvider::TextureSet { set: &scope.set }, None, ResourceMetadataProvider::Indirect { metadata: &scope.metadata, }, &mut self.temp, ) }; unsafe { scope.metadata.remove(index) }; } } /// Unconditionally removes the given resource from the tracker. /// /// Returns true if the resource was removed. /// /// If the ID is higher than the length of internal vectors, /// false will be returned. pub fn remove(&mut self, id: TextureId) -> bool { let index = id.unzip().0 as usize; if index > self.metadata.size() { return false; } self.tracker_assert_in_bounds(index); unsafe { if self.metadata.contains_unchecked(index) { self.start_set.complex.remove(&index); self.end_set.complex.remove(&index); self.metadata.remove(index); return true; } } false } } /// An iterator adapter that can store two different iterator types. #[derive(Clone)] enum EitherIter { Left(L), Right(R), } impl Iterator for EitherIter where L: Iterator, R: Iterator, { type Item = D; fn next(&mut self) -> Option { match *self { EitherIter::Left(ref mut inner) => inner.next(), EitherIter::Right(ref mut inner) => inner.next(), } } } /// Container that signifies storing both different things /// if there is a single state or many different states /// involved in the operation. #[derive(Debug, Clone)] enum SingleOrManyStates { Single(S), Many(M), } /// A source of texture state. #[derive(Clone)] enum TextureStateProvider<'a> { /// Comes directly from a single state. KnownSingle { state: TextureUses }, /// Comes from a selector and a single state. Selector { selector: TextureSelector, state: TextureUses, }, /// Comes from another texture set. TextureSet { set: &'a TextureStateSet }, } impl<'a> TextureStateProvider<'a> { /// Convenience function turning `Option` into this enum. fn from_option(selector: Option, state: TextureUses) -> Self { match selector { Some(selector) => Self::Selector { selector, state }, None => Self::KnownSingle { state }, } } /// Get the state provided by this. /// /// # Panics /// /// Panics if texture_selector is None and this uses a Selector source. /// /// # Safety /// /// - The index must be in bounds of the state set if this uses an TextureSet source. #[inline(always)] unsafe fn get_state( self, texture_selector: Option<&TextureSelector>, index: usize, ) -> SingleOrManyStates< TextureUses, impl Iterator + Clone + 'a, > { match self { TextureStateProvider::KnownSingle { state } => SingleOrManyStates::Single(state), TextureStateProvider::Selector { selector, state } => { // We check if the selector given is actually for the full resource, // and if it is we promote to a simple state. This allows upstream // code to specify selectors willy nilly, and all that are really // single states are promoted here. if *texture_selector.unwrap() == selector { SingleOrManyStates::Single(state) } else { SingleOrManyStates::Many(EitherIter::Left(iter::once((selector, state)))) } } TextureStateProvider::TextureSet { set } => { let new_state = *unsafe { set.simple.get_unchecked(index) }; if new_state == TextureUses::COMPLEX { let new_complex = unsafe { set.complex.get(&index).unwrap_unchecked() }; SingleOrManyStates::Many(EitherIter::Right( new_complex.to_selector_state_iter(), )) } else { SingleOrManyStates::Single(new_state) } } } } } /// Does an insertion operation if the index isn't tracked /// in the current metadata, otherwise merges the given state /// with the current state. If the merging would cause /// a conflict, returns that usage conflict. /// /// # Safety /// /// Indexes must be valid indexes into all arrays passed in /// to this function, either directly or via metadata or provider structs. #[inline(always)] unsafe fn insert_or_merge( texture_selector: &TextureSelector, current_state_set: &mut TextureStateSet, resource_metadata: &mut ResourceMetadata>, index: usize, state_provider: TextureStateProvider<'_>, metadata_provider: ResourceMetadataProvider<'_, Texture>, ) -> Result<(), UsageConflict> { let currently_owned = unsafe { resource_metadata.contains_unchecked(index) }; if !currently_owned { unsafe { insert( Some(texture_selector), None, current_state_set, resource_metadata, index, state_provider, None, metadata_provider, ) }; return Ok(()); } unsafe { merge( texture_selector, current_state_set, index, state_provider, metadata_provider, ) } } /// If the resource isn't tracked /// - Inserts the given resource. /// - Uses the `start_state_provider` to populate `start_states` /// - Uses either `end_state_provider` or `start_state_provider` /// to populate `current_states`. /// If the resource is tracked /// - Inserts barriers from the state in `current_states` /// to the state provided by `start_state_provider`. /// - Updates the `current_states` with either the state from /// `end_state_provider` or `start_state_provider`. /// /// Any barriers are added to the barrier vector. /// /// # Safety /// /// Indexes must be valid indexes into all arrays passed in /// to this function, either directly or via metadata or provider structs. #[inline(always)] unsafe fn insert_or_barrier_update( texture_selector: &TextureSelector, start_state: Option<&mut TextureStateSet>, current_state_set: &mut TextureStateSet, resource_metadata: &mut ResourceMetadata>, index: usize, start_state_provider: TextureStateProvider<'_>, end_state_provider: Option>, metadata_provider: ResourceMetadataProvider<'_, Texture>, barriers: &mut Vec>, ) { let currently_owned = unsafe { resource_metadata.contains_unchecked(index) }; if !currently_owned { unsafe { insert( Some(texture_selector), start_state, current_state_set, resource_metadata, index, start_state_provider, end_state_provider, metadata_provider, ) }; return; } let update_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider.clone()); unsafe { barrier( texture_selector, current_state_set, index, start_state_provider, barriers, ) }; let start_state_set = start_state.unwrap(); unsafe { update( texture_selector, start_state_set, current_state_set, index, update_state_provider, ) }; } #[inline(always)] unsafe fn insert( texture_selector: Option<&TextureSelector>, start_state: Option<&mut TextureStateSet>, end_state: &mut TextureStateSet, resource_metadata: &mut ResourceMetadata>, index: usize, start_state_provider: TextureStateProvider<'_>, end_state_provider: Option>, metadata_provider: ResourceMetadataProvider<'_, Texture>, ) { let start_layers = unsafe { start_state_provider.get_state(texture_selector, index) }; match start_layers { SingleOrManyStates::Single(state) => { // This should only ever happen with a wgpu bug, but let's just double // check that resource states don't have any conflicts. strict_assert_eq!(invalid_resource_state(state), false); log::trace!("\ttex {index}: insert start {state:?}"); if let Some(start_state) = start_state { unsafe { *start_state.simple.get_unchecked_mut(index) = state }; } // We only need to insert ourselves the end state if there is no end state provider. if end_state_provider.is_none() { unsafe { *end_state.simple.get_unchecked_mut(index) = state }; } } SingleOrManyStates::Many(state_iter) => { let full_range = texture_selector.unwrap().clone(); let complex = unsafe { ComplexTextureState::from_selector_state_iter(full_range, state_iter) }; log::trace!("\ttex {index}: insert start {complex:?}"); if let Some(start_state) = start_state { unsafe { *start_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX }; start_state.complex.insert(index, complex.clone()); } // We only need to insert ourselves the end state if there is no end state provider. if end_state_provider.is_none() { unsafe { *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX }; end_state.complex.insert(index, complex); } } } if let Some(end_state_provider) = end_state_provider { match unsafe { end_state_provider.get_state(texture_selector, index) } { SingleOrManyStates::Single(state) => { // This should only ever happen with a wgpu bug, but let's just double // check that resource states don't have any conflicts. strict_assert_eq!(invalid_resource_state(state), false); log::trace!("\ttex {index}: insert end {state:?}"); // We only need to insert into the end, as there is guaranteed to be // a start state provider. unsafe { *end_state.simple.get_unchecked_mut(index) = state }; } SingleOrManyStates::Many(state_iter) => { let full_range = texture_selector.unwrap().clone(); let complex = unsafe { ComplexTextureState::from_selector_state_iter(full_range, state_iter) }; log::trace!("\ttex {index}: insert end {complex:?}"); // We only need to insert into the end, as there is guaranteed to be // a start state provider. unsafe { *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX }; end_state.complex.insert(index, complex); } } } unsafe { let resource = metadata_provider.get_own(index); resource_metadata.insert(index, resource); } } #[inline(always)] unsafe fn merge( texture_selector: &TextureSelector, current_state_set: &mut TextureStateSet, index: usize, state_provider: TextureStateProvider<'_>, metadata_provider: ResourceMetadataProvider<'_, Texture>, ) -> Result<(), UsageConflict> { let current_simple = unsafe { current_state_set.simple.get_unchecked_mut(index) }; let current_state = if *current_simple == TextureUses::COMPLEX { SingleOrManyStates::Many(unsafe { current_state_set.complex.get_mut(&index).unwrap_unchecked() }) } else { SingleOrManyStates::Single(current_simple) }; let new_state = unsafe { state_provider.get_state(Some(texture_selector), index) }; match (current_state, new_state) { (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { let merged_state = *current_simple | new_simple; log::trace!("\ttex {index}: merge simple {current_simple:?} + {new_simple:?}"); if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( TextureId::zip( index as _, unsafe { metadata_provider.get_epoch(index) }, A::VARIANT, ), texture_selector.clone(), *current_simple, new_simple, )); } *current_simple = merged_state; } (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { // Because we are now demoting this simple state to a complex state, // we actually need to make a whole new complex state for us to use // as there wasn't one before. let mut new_complex = unsafe { ComplexTextureState::from_selector_state_iter( texture_selector.clone(), iter::once((texture_selector.clone(), *current_simple)), ) }; for (selector, new_state) in new_many { let merged_state = *current_simple | new_state; log::trace!("\ttex {index}: merge {selector:?} {current_simple:?} + {new_state:?}"); if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( TextureId::zip( index as _, unsafe { metadata_provider.get_epoch(index) }, A::VARIANT, ), selector, *current_simple, new_state, )); } for mip in &mut new_complex.mips[selector.mips.start as usize..selector.mips.end as usize] { for &mut (_, ref mut current_layer_state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { *current_layer_state = merged_state; } mip.coalesce(); } } *current_simple = TextureUses::COMPLEX; current_state_set.complex.insert(index, new_complex); } (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_simple)) => { for (mip_id, mip) in current_complex.mips.iter_mut().enumerate() { let mip_id = mip_id as u32; for &mut (ref layers, ref mut current_layer_state) in mip.iter_mut() { let merged_state = *current_layer_state | new_simple; // Once we remove unknown, this will never be empty, as // simple states are never unknown. let merged_state = merged_state - TextureUses::UNKNOWN; log::trace!( "\ttex {index}: merge mip {mip_id} layers {layers:?} \ {current_layer_state:?} + {new_simple:?}" ); if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( TextureId::zip( index as _, unsafe { metadata_provider.get_epoch(index) }, A::VARIANT, ), TextureSelector { mips: mip_id..mip_id + 1, layers: layers.clone(), }, *current_layer_state, new_simple, )); } *current_layer_state = merged_state; } mip.coalesce(); } } (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { for (selector, new_state) in new_many { for mip_id in selector.mips { strict_assert!((mip_id as usize) < current_complex.mips.len()); let mip = unsafe { current_complex.mips.get_unchecked_mut(mip_id as usize) }; for &mut (ref layers, ref mut current_layer_state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { let merged_state = *current_layer_state | new_state; let merged_state = merged_state - TextureUses::UNKNOWN; if merged_state.is_empty() { // We know nothing about this state, lets just move on. continue; } log::trace!( "\ttex {index}: merge mip {mip_id} layers {layers:?} \ {current_layer_state:?} + {new_state:?}" ); if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( TextureId::zip( index as _, unsafe { metadata_provider.get_epoch(index) }, A::VARIANT, ), TextureSelector { mips: mip_id..mip_id + 1, layers: layers.clone(), }, *current_layer_state, new_state, )); } *current_layer_state = merged_state; } mip.coalesce(); } } } } Ok(()) } #[inline(always)] unsafe fn barrier( texture_selector: &TextureSelector, current_state_set: &TextureStateSet, index: usize, state_provider: TextureStateProvider<'_>, barriers: &mut Vec>, ) { let current_simple = unsafe { *current_state_set.simple.get_unchecked(index) }; let current_state = if current_simple == TextureUses::COMPLEX { SingleOrManyStates::Many(unsafe { current_state_set.complex.get(&index).unwrap_unchecked() }) } else { SingleOrManyStates::Single(current_simple) }; let new_state = unsafe { state_provider.get_state(Some(texture_selector), index) }; match (current_state, new_state) { (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { if skip_barrier(current_simple, new_simple) { return; } log::trace!("\ttex {index}: transition simple {current_simple:?} -> {new_simple:?}"); barriers.push(PendingTransition { id: index as _, selector: texture_selector.clone(), usage: current_simple..new_simple, }); } (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { for (selector, new_state) in new_many { if new_state == TextureUses::UNKNOWN { continue; } if skip_barrier(current_simple, new_state) { continue; } log::trace!( "\ttex {index}: transition {selector:?} {current_simple:?} -> {new_state:?}" ); barriers.push(PendingTransition { id: index as _, selector, usage: current_simple..new_state, }); } } (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_simple)) => { for (mip_id, mip) in current_complex.mips.iter().enumerate() { let mip_id = mip_id as u32; for &(ref layers, current_layer_state) in mip.iter() { if current_layer_state == TextureUses::UNKNOWN { continue; } if skip_barrier(current_layer_state, new_simple) { continue; } log::trace!( "\ttex {index}: transition mip {mip_id} layers {layers:?} \ {current_layer_state:?} -> {new_simple:?}" ); barriers.push(PendingTransition { id: index as _, selector: TextureSelector { mips: mip_id..mip_id + 1, layers: layers.clone(), }, usage: current_layer_state..new_simple, }); } } } (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { for (selector, new_state) in new_many { for mip_id in selector.mips { strict_assert!((mip_id as usize) < current_complex.mips.len()); let mip = unsafe { current_complex.mips.get_unchecked(mip_id as usize) }; for (layers, current_layer_state) in mip.iter_filter(&selector.layers) { if *current_layer_state == TextureUses::UNKNOWN || new_state == TextureUses::UNKNOWN { continue; } if skip_barrier(*current_layer_state, new_state) { continue; } log::trace!( "\ttex {index}: transition mip {mip_id} layers {layers:?} \ {current_layer_state:?} -> {new_state:?}" ); barriers.push(PendingTransition { id: index as _, selector: TextureSelector { mips: mip_id..mip_id + 1, layers, }, usage: *current_layer_state..new_state, }); } } } } } } #[allow(clippy::needless_option_as_deref)] // we use this for reborrowing Option<&mut T> #[inline(always)] unsafe fn update( texture_selector: &TextureSelector, start_state_set: &mut TextureStateSet, current_state_set: &mut TextureStateSet, index: usize, state_provider: TextureStateProvider<'_>, ) { let start_simple = unsafe { *start_state_set.simple.get_unchecked(index) }; // We only ever need to update the start state here if the state is complex. // // If the state is simple, the first insert to the tracker would cover it. let mut start_complex = None; if start_simple == TextureUses::COMPLEX { start_complex = Some(unsafe { start_state_set.complex.get_mut(&index).unwrap_unchecked() }); } let current_simple = unsafe { current_state_set.simple.get_unchecked_mut(index) }; let current_state = if *current_simple == TextureUses::COMPLEX { SingleOrManyStates::Many(unsafe { current_state_set.complex.get_mut(&index).unwrap_unchecked() }) } else { SingleOrManyStates::Single(current_simple) }; let new_state = unsafe { state_provider.get_state(Some(texture_selector), index) }; match (current_state, new_state) { (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { *current_simple = new_simple; } (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { // Because we are now demoting this simple state to a complex state, // we actually need to make a whole new complex state for us to use // as there wasn't one before. let mut new_complex = unsafe { ComplexTextureState::from_selector_state_iter( texture_selector.clone(), iter::once((texture_selector.clone(), *current_simple)), ) }; for (selector, mut new_state) in new_many { if new_state == TextureUses::UNKNOWN { new_state = *current_simple; } for mip in &mut new_complex.mips[selector.mips.start as usize..selector.mips.end as usize] { for &mut (_, ref mut current_layer_state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { *current_layer_state = new_state; } mip.coalesce(); } } *current_simple = TextureUses::COMPLEX; current_state_set.complex.insert(index, new_complex); } (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_single)) => { for (mip_id, mip) in current_complex.mips.iter().enumerate() { for &(ref layers, current_layer_state) in mip.iter() { // If this state is unknown, that means that the start is _also_ unknown. if current_layer_state == TextureUses::UNKNOWN { if let Some(&mut ref mut start_complex) = start_complex { strict_assert!(mip_id < start_complex.mips.len()); let start_mip = unsafe { start_complex.mips.get_unchecked_mut(mip_id) }; for &mut (_, ref mut current_start_state) in start_mip.isolate(layers, TextureUses::UNKNOWN) { strict_assert_eq!(*current_start_state, TextureUses::UNKNOWN); *current_start_state = new_single; } start_mip.coalesce(); } } } } unsafe { *current_state_set.simple.get_unchecked_mut(index) = new_single }; unsafe { current_state_set.complex.remove(&index).unwrap_unchecked() }; } (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { for (selector, new_state) in new_many { if new_state == TextureUses::UNKNOWN { // We know nothing new continue; } for mip_id in selector.mips { let mip_id = mip_id as usize; strict_assert!(mip_id < current_complex.mips.len()); let mip = unsafe { current_complex.mips.get_unchecked_mut(mip_id) }; for &mut (ref layers, ref mut current_layer_state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { if *current_layer_state == TextureUses::UNKNOWN && new_state != TextureUses::UNKNOWN { // We now know something about this subresource that // we didn't before so we should go back and update // the start state. // // We know we must have starter state be complex, // otherwise we would know about this state. strict_assert!(start_complex.is_some()); let start_complex = unsafe { start_complex.as_deref_mut().unwrap_unchecked() }; strict_assert!(mip_id < start_complex.mips.len()); let start_mip = unsafe { start_complex.mips.get_unchecked_mut(mip_id) }; for &mut (_, ref mut current_start_state) in start_mip.isolate(layers, TextureUses::UNKNOWN) { strict_assert_eq!(*current_start_state, TextureUses::UNKNOWN); *current_start_state = new_state; } start_mip.coalesce(); } *current_layer_state = new_state; } mip.coalesce(); } } } } }