//! Render-graph compilation derives command dependencies from conflicting resource //! accesses and computes their transitive happens-before closure. Transient //! resources may share memory only when every terminal use of one precedes every //! initial use of the other; device-dependent placement uses separate //! linear/optimal pools and inserts alias barriers at the new occupant's first //! use. //! //! Aliasing safety assumes all graph commands are recorded into one command //! buffer that is submitted to one queue. Cross-queue aliasing is unsupported: //! when command-to-queue assignment is introduced, alias transitions between //! queues must also be represented by semaphore edges. // TODO: count image creations by hash of desc to see if some particular image is being created very frequently. // maybe it can be pooled. use std::{ cmp::Reverse, collections::{BTreeSet, BinaryHeap, HashMap, HashSet, VecDeque}, sync::Arc, }; use ash::vk; use fixedbitset::FixedBitSet; use renderer_resources::{ Device, buffers::{Buffer as BufferResource, BufferDesc}, descriptors::DescriptorResourceBinding, device::{AllocationStrategy, MemoryLocation, QueueFlags}, images::{Image as ImageResource, ImageDesc, ImageSubresourceExt, ImageView, ImageViewDesc}, pipeline::DescriptorSet, }; use renderer_device::QueueFamilyIndex; use crate::graph_compiler::{Batch, CommandOrderings, build_command_orderings}; use crate::{ commands::{AttachmentView, CommandKind, RenderPassDesc}, recorder::{ CommandIndex, CommandList, CommandRecorder, RecordError, SideEffectMap, scope_range, }, resources::{ Buffer, BufferRange, DescriptorResourceAccess, DescriptorResourceIdentity, GraphBufferDesc, GraphDescriptorSet, GraphTextureDesc, ResourceDesc, ResourceId, ResourceState, Subresource, SubresourceScope, SubresourceScopeInner, Texture, access_flag_is_write, }, }; #[derive(Debug, Clone, PartialEq, Eq)] pub enum GraphError { UnknownResource(ResourceId), DependencyCycle, DescriptorImport(String), StaleDescriptorSet, ConflictingInitialState(ResourceId), ConflictingDrawLayouts(ResourceId), ConflictingDescriptorInitialStates, ConflictingDescriptorDrawLayouts, UnbalancedRenderPass, NestedRenderPass, DrawOutsideRenderPass(CommandIndex), CommandNotAllowedInRenderPass(CommandIndex), InvalidRenderPassAttachment(String), /// An image layout transition would need to occur inside dynamic rendering. UnsupportedIntraPassDependency, /// An interior access overlaps a render-pass attachment subresource. RenderPassFeedbackLoop(String), /// A created attachment view could not be constructed or did not match its image. ViewCreationFailed(String), PlacementFailed(ResourceId), BindFailed(ResourceId), InvalidPooledImageLayout(ResourceId), } impl From<(ResourceId, renderer_resources::memory::Error)> for GraphError { fn from((resource, error): (ResourceId, renderer_resources::memory::Error)) -> Self { match error { renderer_resources::memory::Error::Bind(_) | renderer_resources::memory::Error::MisalignedOffset { .. } | renderer_resources::memory::Error::IncompatibleMemoryType { .. } | renderer_resources::memory::Error::OutOfBounds { .. } | renderer_resources::memory::Error::AlreadyBound => Self::BindFailed(resource), _ => Self::PlacementFailed(resource), } } } impl From<(ResourceId, renderer_resources::Error)> for GraphError { fn from((resource, _error): (ResourceId, renderer_resources::Error)) -> Self { Self::PlacementFailed(resource) } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct QueueTransfer { pub src: QueueFamilyIndex, pub dst: QueueFamilyIndex, } /// Identifies what orders a barrier before its destination command. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BarrierOrigin { /// Transition from the resource's declared initial state (or first use of a created resource). Initial, /// Ordered after the command that previously accessed the resource. Command(CommandIndex), } #[derive(Debug, Clone, Copy)] pub struct BarrierResource { pub id: ResourceId, pub range: Subresource, pub origin: BarrierOrigin, pub src_stage: vk::PipelineStageFlags2, pub src_access: vk::AccessFlags2, pub dst_stage: vk::PipelineStageFlags2, pub dst_access: vk::AccessFlags2, pub old_layout: Option, pub new_layout: Option, pub queue_transfer: Option, } // TinyVec initializes its inline backing array with defaults; slots beyond its // logical length remain inaccessible to barrier consumers. impl Default for BarrierResource { fn default() -> Self { Self { id: ResourceId::from_u32(0), range: Subresource::Buffer(crate::resources::BufferRange { offset: 0, size: 0 }), origin: BarrierOrigin::Initial, src_stage: vk::PipelineStageFlags2::NONE, src_access: vk::AccessFlags2::NONE, dst_stage: vk::PipelineStageFlags2::NONE, dst_access: vk::AccessFlags2::NONE, old_layout: None, new_layout: None, queue_transfer: None, } } } #[derive(Debug, Clone, Copy)] struct LayoutRegion { range: vk::ImageSubresourceRange, layout: vk::ImageLayout, } fn image_range_intersection( a: vk::ImageSubresourceRange, b: vk::ImageSubresourceRange, ) -> Option { let aspect_mask = a.aspect_mask & b.aspect_mask; if aspect_mask.is_empty() { return None; } let mip_start = u64::from(a.base_mip_level.max(b.base_mip_level)); let mip_end = (u64::from(a.base_mip_level) + u64::from(a.level_count)) .min(u64::from(b.base_mip_level) + u64::from(b.level_count)); let layer_start = u64::from(a.base_array_layer.max(b.base_array_layer)); let layer_end = (u64::from(a.base_array_layer) + u64::from(a.layer_count)) .min(u64::from(b.base_array_layer) + u64::from(b.layer_count)); if mip_start >= mip_end || layer_start >= layer_end { return None; } Some( vk::ImageSubresourceRange::default() .aspect_mask(aspect_mask) .base_mip_level(mip_start as u32) .level_count((mip_end - mip_start) as u32) .base_array_layer(layer_start as u32) .layer_count((layer_end - layer_start) as u32), ) } fn image_range_contains( range: vk::ImageSubresourceRange, other: vk::ImageSubresourceRange, ) -> bool { range.aspect_mask.contains(other.aspect_mask) && range.base_mip_level <= other.base_mip_level && u64::from(range.base_mip_level) + u64::from(range.level_count) >= u64::from(other.base_mip_level) + u64::from(other.level_count) && range.base_array_layer <= other.base_array_layer && u64::from(range.base_array_layer) + u64::from(range.layer_count) >= u64::from(other.base_array_layer) + u64::from(other.layer_count) } struct ImageRangeRemainders { ranges: [Option; 5], len: usize, } impl ImageRangeRemainders { fn new() -> Self { Self { ranges: [None; 5], len: 0, } } fn push(&mut self, range: vk::ImageSubresourceRange) { self.ranges[self.len] = Some(range); self.len += 1; } fn iter(self) -> impl Iterator { self.ranges.into_iter().take(self.len).flatten() } } /// Subtracts the cut's aspect/mip/layer coverage, returning disjoint remainder ranges. fn subtract_image_range( range: vk::ImageSubresourceRange, cut: vk::ImageSubresourceRange, ) -> ImageRangeRemainders { let mut remainder = ImageRangeRemainders::new(); let Some(intersection) = image_range_intersection(range, cut) else { remainder.push(range); return remainder; }; let other_aspects = range.aspect_mask & !intersection.aspect_mask; if !other_aspects.is_empty() { remainder.push(vk::ImageSubresourceRange { aspect_mask: other_aspects, ..range }); } let aspects = intersection.aspect_mask; let range_mip_start = u64::from(range.base_mip_level); let range_mip_end = range_mip_start + u64::from(range.level_count); let range_layer_start = u64::from(range.base_array_layer); let range_layer_end = range_layer_start + u64::from(range.layer_count); let cut_mip_start = u64::from(intersection.base_mip_level); let cut_mip_end = cut_mip_start + u64::from(intersection.level_count); let cut_layer_start = u64::from(intersection.base_array_layer); let cut_layer_end = cut_layer_start + u64::from(intersection.layer_count); let rect = |mip_start: u64, mip_end: u64, layer_start: u64, layer_end: u64| { (mip_start < mip_end && layer_start < layer_end).then(|| { vk::ImageSubresourceRange::default() .aspect_mask(aspects) .base_mip_level(mip_start as u32) .level_count((mip_end - mip_start) as u32) .base_array_layer(layer_start as u32) .layer_count((layer_end - layer_start) as u32) }) }; // Four non-overlapping strips cover the rectangle around the intersection. for piece in [ rect( range_mip_start, cut_mip_start, range_layer_start, range_layer_end, ), rect( cut_mip_end, range_mip_end, range_layer_start, range_layer_end, ), rect( cut_mip_start, cut_mip_end, range_layer_start, cut_layer_start, ), rect(cut_mip_start, cut_mip_end, cut_layer_end, range_layer_end), ] .into_iter() .flatten() { remainder.push(piece); } remainder } fn update_layout_regions( regions: &mut Vec, range: vk::ImageSubresourceRange, layout: vk::ImageLayout, ) { let intersects = regions .iter() .filter(|region| image_range_intersection(region.range, range).is_some()) .count(); if intersects == 0 { regions.push(LayoutRegion { range, layout }); return; } if regions .iter() .any(|region| same_image_subresource_range(region.range, range) && region.layout == layout) { return; } if let Some(region) = regions .iter_mut() .find(|region| image_range_contains(region.range, range)) { if region.layout == layout { return; } if same_image_subresource_range(region.range, range) { region.layout = layout; return; } } if regions .iter() .any(|region| image_range_contains(region.range, range) && region.layout == layout) { return; } // Replacing fully covered regions in place avoids draining and rebuilding the // common case where a transition supersedes existing layout coverage. if regions .iter() .filter(|region| image_range_intersection(region.range, range).is_some()) .all(|region| image_range_contains(range, region.range)) { regions.retain(|region| image_range_intersection(region.range, range).is_none()); regions.push(LayoutRegion { range, layout }); return; } let mut updated = Vec::with_capacity(regions.len() + 4); for region in regions.drain(..) { for remainder in subtract_image_range(region.range, range).iter() { updated.push(LayoutRegion { range: remainder, layout: region.layout, }); } } updated.push(LayoutRegion { range, layout }); *regions = updated; } fn layout_pieces( range: vk::ImageSubresourceRange, fallback: vk::ImageLayout, regions: &[LayoutRegion], ) -> Vec<(vk::ImageSubresourceRange, vk::ImageLayout)> { if !regions .iter() .any(|region| image_range_intersection(range, region.range).is_some()) { return vec![(range, fallback)]; } if let Some(region) = regions .iter() .find(|region| image_range_contains(region.range, range)) { return vec![(range, region.layout)]; } let mut uncovered = Vec::with_capacity(5); uncovered.push(range); let mut next_uncovered = Vec::with_capacity(5); let mut pieces = Vec::with_capacity(regions.len() + 1); for region in regions { let Some(intersection) = image_range_intersection(range, region.range) else { continue; }; pieces.push((intersection, region.layout)); next_uncovered.clear(); for piece in uncovered.drain(..) { next_uncovered.extend(subtract_image_range(piece, intersection).iter()); } std::mem::swap(&mut uncovered, &mut next_uncovered); } if uncovered.is_empty() && let Some((_, uniform_layout)) = pieces.first().copied() && pieces.iter().all(|(_, layout)| *layout == uniform_layout) { return vec![(range, uniform_layout)]; } pieces.extend(uncovered.into_iter().map(|piece| (piece, fallback))); pieces } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct MemoryBarrier { pub src_stage: vk::PipelineStageFlags2, pub src_access: vk::AccessFlags2, pub dst_stage: vk::PipelineStageFlags2, pub dst_access: vk::AccessFlags2, } #[derive(Debug, Clone)] pub struct PipelineBarrier { pub before_command: CommandIndex, pub memory_barriers: Vec, /// Inline storage avoids heap allocation for the common single-resource barrier; /// capacity two still lets draw-input barriers spill only when they have 3+ resources. pub resources: tinyvec::TinyVec<[BarrierResource; 2]>, } /// Command ids are dense `0..command_count`, so barrier-index groups use direct vector indexing. struct BarriersByCommand { entries: Vec>, } impl BarriersByCommand { fn new(command_count: usize) -> Self { Self { entries: vec![Vec::new(); command_count], } } fn get(&self, command: CommandIndex) -> &[usize] { &self.entries[command.index()] } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ResourceLifetime { pub first_batch: usize, pub last_batch: usize, pub placement: Placement, /// Imported resources participate in analysis, but never in transient packing. pub imported: bool, } /// Placement metadata reserved for the device-dependent packing phase. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Placement { pub offset: u64, pub size: u64, pub alignment: u64, pub pool: Option, pub aliased: bool, } /// One command's access scope for a resource, retained for alias barriers. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ResourceUse { pub command: CommandIndex, pub stage: vk::PipelineStageFlags2, pub access: vk::AccessFlags2, } /// Resource scope collapsed to one entry per contracted DAG node. /// /// The representative commands are both real accesses: the first is used to key /// initial-use barriers, while the last keys terminal-use ordering checks. #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct CollapsedUse { node: usize, first_command: CommandIndex, last_command: CommandIndex, stage: vk::PipelineStageFlags2, access: vk::AccessFlags2, } /// Per-resource terminal and initial uses cached for placement and alias barriers. #[derive(Debug, Default, Clone)] pub struct ResourceFrontiers { uses: Vec, Vec)>>, collapsed: Vec, Vec)>>, command_level_alias_frontiers: bool, } #[derive(Clone, Copy)] enum AliasUse<'a> { Command(&'a ResourceUse), Collapsed(&'a CollapsedUse), } impl AliasUse<'_> { fn first_command(self) -> CommandIndex { match self { Self::Command(use_) => use_.command, Self::Collapsed(use_) => use_.first_command, } } fn last_command(self) -> CommandIndex { match self { Self::Command(use_) => use_.command, Self::Collapsed(use_) => use_.last_command, } } fn stage(self) -> vk::PipelineStageFlags2 { match self { Self::Command(use_) => use_.stage, Self::Collapsed(use_) => use_.stage, } } fn access(self) -> vk::AccessFlags2 { match self { Self::Command(use_) => use_.access, Self::Collapsed(use_) => use_.access, } } } enum AliasUses<'a> { Command(&'a [ResourceUse]), Collapsed(&'a [CollapsedUse]), } enum AliasUseIter<'a> { Command(std::slice::Iter<'a, ResourceUse>), Collapsed(std::slice::Iter<'a, CollapsedUse>), } impl<'a> Iterator for AliasUseIter<'a> { type Item = AliasUse<'a>; fn next(&mut self) -> Option { match self { Self::Command(iter) => iter.next().map(AliasUse::Command), Self::Collapsed(iter) => iter.next().map(AliasUse::Collapsed), } } } impl<'a> AliasUses<'a> { fn iter(&self) -> AliasUseIter<'a> { match self { Self::Command(uses) => AliasUseIter::Command(uses.iter()), Self::Collapsed(uses) => AliasUseIter::Collapsed(uses.iter()), } } fn is_empty(&self) -> bool { match self { Self::Command(uses) => uses.is_empty(), Self::Collapsed(uses) => uses.is_empty(), } } } impl ResourceFrontiers { /// Computes frontiers directly from command-level uses when there are no /// render-pass spans. With identity command-to-node mapping, each command /// use is one collapsed use, so this produces the same frontiers as the /// contracted-node path without constructing a copy of every use. fn from_command_uses( uses_by_resource: &HashMap>, happens_before: &HappensBefore, ) -> Self { let mut uses_frontiers: Vec, Vec)>> = vec![None; resource_frontier_len(uses_by_resource)]; for (&id, uses) in uses_by_resource { let terminal_uses = terminal_uses(uses, happens_before); let initial_uses = initial_uses(uses, happens_before); uses_frontiers[id.index()] = Some(( terminal_uses.iter().map(|use_| **use_).collect(), initial_uses.iter().map(|use_| **use_).collect(), )); } Self { uses: uses_frontiers, collapsed: Vec::new(), command_level_alias_frontiers: true, } } /// Computes owned terminal/initial use sets once for each resource. pub fn from_uses( uses_by_resource: &HashMap>, happens_before: &HappensBefore, ) -> Self { let collapsed_uses = collapse_uses(uses_by_resource, &happens_before.node_of_command); let collapsed = Self::frontier_map(&collapsed_uses, happens_before); let mut uses: Vec, Vec)>> = vec![None; resource_frontier_len(uses_by_resource)]; for (&id, resource_uses) in uses_by_resource { let terminal = terminal_uses(resource_uses, happens_before) .into_iter() .copied() .collect(); let initial = initial_uses(resource_uses, happens_before) .into_iter() .copied() .collect(); uses[id.index()] = Some((terminal, initial)); } Self { uses, collapsed, command_level_alias_frontiers: false, } } /// Computes frontiers from uses already collapsed by contracted DAG node. fn from_collapsed_uses( uses_by_resource: &HashMap>, happens_before: &HappensBefore, ) -> Self { let collapsed = Self::frontier_map(uses_by_resource, happens_before); let mut uses: Vec, Vec)>> = vec![None; resource_frontier_len(uses_by_resource)]; for (index, frontier) in collapsed.iter().enumerate() { if let Some((terminal, initial)) = frontier { let terminal = terminal.iter().map(collapsed_use_as_resource_use).collect(); let initial = initial.iter().map(collapsed_use_as_resource_use).collect(); uses[index] = Some((terminal, initial)); } } Self { uses, collapsed, command_level_alias_frontiers: false, } } fn frontier_map( uses_by_resource: &HashMap>, happens_before: &HappensBefore, ) -> Vec, Vec)>> { let mut frontiers: Vec, Vec)>> = vec![None; resource_frontier_len(uses_by_resource)]; for (&id, uses) in uses_by_resource { frontiers[id.index()] = Some(( terminal_collapsed_uses(uses, happens_before), initial_collapsed_uses(uses, happens_before), )); } frontiers } /// Terminal uses not ordered before another use of this resource. pub fn terminal(&self, id: ResourceId) -> &[ResourceUse] { self.uses .get(id.index()) .and_then(Option::as_ref) .map(|(terminal, _)| terminal.as_slice()) .unwrap_or_default() } /// Initial uses not ordered after another use of this resource. pub fn initial(&self, id: ResourceId) -> &[ResourceUse] { self.uses .get(id.index()) .and_then(Option::as_ref) .map(|(_, initial)| initial.as_slice()) .unwrap_or_default() } fn collapsed_terminal(&self, id: ResourceId) -> &[CollapsedUse] { self.collapsed .get(id.index()) .and_then(Option::as_ref) .map(|(terminal, _)| terminal.as_slice()) .unwrap_or_default() } fn collapsed_initial(&self, id: ResourceId) -> &[CollapsedUse] { self.collapsed .get(id.index()) .and_then(Option::as_ref) .map(|(_, initial)| initial.as_slice()) .unwrap_or_default() } fn alias_terminal(&self, id: ResourceId) -> AliasUses<'_> { if self.command_level_alias_frontiers { AliasUses::Command(self.terminal(id)) } else { AliasUses::Collapsed(self.collapsed_terminal(id)) } } fn alias_initial(&self, id: ResourceId) -> AliasUses<'_> { if self.command_level_alias_frontiers { AliasUses::Command(self.initial(id)) } else { AliasUses::Collapsed(self.collapsed_initial(id)) } } /// Whether every terminal use of `a` precedes every initial use of `b`. /// Empty frontier sets are rejected rather than treated as vacuously safe. pub fn alias_in_order( &self, a: ResourceId, b: ResourceId, happens_before: &HappensBefore, ) -> bool { if self.command_level_alias_frontiers { let terminal = self.terminal(a); let initial = self.initial(b); return !terminal.is_empty() && !initial.is_empty() && terminal.iter().all(|terminal| { initial.iter().all(|initial| { happens_before.happens_before(terminal.command, initial.command) }) }); } let terminal = self.alias_terminal(a); let initial = self.alias_initial(b); !terminal.is_empty() && !initial.is_empty() && terminal.iter().all(|terminal| { initial.iter().all(|initial| { happens_before.happens_before(terminal.last_command(), initial.first_command()) }) }) } /// Whether either resource order permits safe placement reuse. pub fn alias_in_order_either( &self, a: ResourceId, b: ResourceId, happens_before: &HappensBefore, ) -> bool { self.alias_in_order(a, b, happens_before) || self.alias_in_order(b, a, happens_before) } } /// Resource IDs are dense graph indices, so frontier slots can be queried /// directly without hashing on the placement/interference hot path. fn resource_frontier_len(uses_by_resource: &HashMap) -> usize { uses_by_resource .keys() .map(|id| id.index()) .max() .map_or(0, |max_index| max_index + 1) } fn collapsed_use_as_resource_use(use_: &CollapsedUse) -> ResourceUse { ResourceUse { command: use_.first_command, stage: use_.stage, access: use_.access, } } fn collapse_uses( uses_by_resource: &HashMap>, node_of_command: &[usize], ) -> HashMap> { uses_by_resource .iter() .map(|(&id, uses)| { let mut collapsed: Vec = Vec::new(); for use_ in uses { let node = node_of_command .get(use_.command.index()) .copied() .unwrap_or(use_.command.index()); if let Some(last) = collapsed.last_mut() && last.node == node { last.last_command = use_.command; last.stage |= use_.stage; last.access |= use_.access; } else { collapsed.push(CollapsedUse { node, first_command: use_.command, last_command: use_.command, stage: use_.stage, access: use_.access, }); } } (id, collapsed) }) .collect() } /* * Frontier analysis operates on the contracted DAG nodes. For uses in one node, * same-node ordering remains deliberately false and the single collapsed entry * therefore remains both an initial and terminal use. */ fn terminal_collapsed_uses( uses: &[CollapsedUse], happens_before: &HappensBefore, ) -> Vec { uses.iter() .filter(|candidate| { !uses.iter().any(|other| { candidate.node != other.node && happens_before.happens_before(candidate.last_command, other.first_command) }) }) .copied() .collect() } fn initial_collapsed_uses( uses: &[CollapsedUse], happens_before: &HappensBefore, ) -> Vec { uses.iter() .filter(|candidate| { !uses.iter().any(|other| { candidate.node != other.node && happens_before.happens_before(other.last_command, candidate.first_command) }) }) .copied() .collect() } /// A render-pass span in original command-recording order. #[derive(Debug, Clone)] pub struct RenderPassSpan { /// Begin-rendering command index. pub begin: CommandIndex, /// End-rendering command index. pub end: CommandIndex, } impl RenderPassSpan { fn interior_range(&self) -> std::ops::Range { debug_assert!( self.end >= self.begin, "RenderPassSpan end must not precede begin" ); self.begin.index() + 1..self.end.index() } fn commands(&self) -> impl Iterator + '_ { (self.begin.index()..=self.end.index()).map(CommandIndex::from) } } /// The resource-level interference graph used by transient-resource aliasing. /// An edge means no safe ordering between the resources was proven. #[derive(Debug, Clone, Default)] pub struct ResourceInterference { edges: Vec>, } impl ResourceInterference { /// Build a conservative interference graph from inclusive batch intervals. /// /// This interval-only constructor is useful for reporting/tests. The compiler /// uses command reachability instead; batch separation is not an ordering proof. pub fn from_lifetimes( lifetimes: impl IntoIterator, ) -> Self { let mut max_id = ResourceId::from_u32(0); let mut resources: Vec<_> = lifetimes .into_iter() .inspect(|(id, _)| { if *id > max_id { max_id = *id; } }) .collect(); let mut edges = vec![BTreeSet::new(); max_id.index() + 1]; resources.sort_unstable_by_key(|&(_, lifetime)| lifetime.first_batch); let mut resources = VecDeque::from(resources); let mut index = 0; while index < resources.len() { let (id, lifetime) = resources[index]; let mut j = 0; while j < index { if resources[j].1.last_batch < lifetime.first_batch { resources.swap_remove_front(j); index -= 1; } else { j += 1; } } index += 1; // Active entries are partitioned by their inclusive batch intervals. let active = resources.partition_point(|&(_, other)| other.first_batch <= lifetime.last_batch); for &(other_id, _) in resources.range(..active) { edges[id.index()].insert(other_id); edges[other_id.index()].insert(id); } } Self { edges } } /// Returns whether the resources interfere and therefore must not share memory. pub fn interferes(&self, a: ResourceId, b: ResourceId) -> bool { self.edges[a.index()].contains(&b) } } /// Compact transitive-closure rows over contracted nodes, queried by command index. /// Commands within one render-pass span map to the same node and are deliberately /// unordered with respect to one another for aliasing and interference analysis. #[derive(Debug, Clone, Default)] pub struct HappensBefore { rows: Vec, node_of_command: Vec, } impl HappensBefore { fn from_dag( edges: &[Vec], execution_order: &[CommandIndex], node_of_command: &[usize], node_representatives: &[CommandIndex], ) -> Self { let mut closure = Self { rows: (0..edges.len()) .map(|_| FixedBitSet::with_capacity(edges.len())) .collect(), node_of_command: node_of_command.to_vec(), }; // The execution order restricted to representatives is a valid DAG topological // order; process it backwards so each successor row is complete. for &representative in execution_order.iter().rev() { let node = closure.node_of_command[representative.index()]; if node_representatives[node] != representative { continue; } for &successor in &edges[node] { closure.rows[node].insert(successor); if node != successor { let (node_row, successor_row) = if node < successor { let (before_successor, from_successor) = closure.rows.split_at_mut(successor); (&mut before_successor[node], &from_successor[0]) } else { let (before_node, from_node) = closure.rows.split_at_mut(node); (&mut from_node[0], &before_node[successor]) }; node_row.union_with(successor_row); } } } closure } fn from_command_dag(edges: &[Vec], batches: &[Vec]) -> Self { let node_of_command = (0..edges.len()).collect::>(); let execution_order = batches.iter().flatten().copied().collect::>(); let node_representatives = node_of_command .iter() .map(|&command| CommandIndex(command)) .collect::>(); Self::from_dag( edges, &execution_order, &node_of_command, &node_representatives, ) } fn contains(&self, from: usize, to: usize) -> bool { let (Some(&from_node), Some(&to_node)) = (self.node_of_command.get(from), self.node_of_command.get(to)) else { return false; }; from_node != to_node && self .rows .get(from_node) .is_some_and(|row| row.contains(to_node)) } /// Whether command `from` transitively precedes command `to`. pub fn happens_before(&self, from: CommandIndex, to: CommandIndex) -> bool { self.contains(from.index(), to.index()) } } bitflags::bitflags! { #[derive(Debug, Default, Copy, Clone)] struct KahnVertex: u8 { const READY = 0b0000_0001; const OCCUPIED = 0b0000_0010; } } impl KahnVertex { fn is_ready(self) -> bool { self.contains(KahnVertex::READY) } fn is_occupied(self) -> bool { self.contains(KahnVertex::OCCUPIED) } fn make_unready(&mut self) -> bool { let was_ready = self.contains(KahnVertex::READY); self.remove(KahnVertex::READY); was_ready } fn make_ready(&mut self) -> bool { let was_ready = self.contains(KahnVertex::READY); self.insert(KahnVertex::READY); was_ready } fn set_occupied(&mut self) -> bool { let was_unoccupied = !self.contains(KahnVertex::OCCUPIED); self.insert(KahnVertex::OCCUPIED); was_unoccupied } } /// Deterministic Kahn topological-sort state for command dependencies. /// Ready nodes are added lazily when an occupied node has no remaining dependencies. #[derive(Debug)] pub(crate) struct Kahn { pub(crate) edges: Vec>, indegree: Vec, ready: BinaryHeap>, // TODO: bitvec nodes: Vec, emitted: usize, total: usize, } /// An iterator over the next batch of ready nodes in a Kahn topological sort. /// It drains the ready heap, skipping stale or unoccupied entries, and the caller /// must call `complete` on each node in the batch before querying for another batch. pub(crate) struct KahnBatchIter<'a> { kahn: &'a mut Kahn, } impl Iterator for KahnBatchIter<'_> { type Item = usize; fn next(&mut self) -> Option { while let Some(Reverse(node)) = self.kahn.ready.pop() { if let Some(vert) = self.kahn.nodes.get_mut(node) && vert.make_unready() && vert.is_occupied() { return Some(node); } } None } } impl Kahn { pub(crate) fn new(node_count: usize) -> Self { Self { edges: vec![Vec::new(); node_count], indegree: vec![0; node_count], ready: BinaryHeap::new(), nodes: vec![KahnVertex::default(); node_count], emitted: 0, total: 0, } } pub(crate) fn add_node(&mut self, node: usize) { if self.nodes[node].set_occupied() { self.total += 1; if self.indegree[node] == 0 { self.nodes[node].make_ready(); self.ready.push(Reverse(node)); } } } pub(crate) fn add_edge(&mut self, from: usize, to: usize) { let edges = &mut self.edges[from]; let mut newly_occupied = [false; 2]; for (index, node) in self .nodes .get_disjoint_mut([from, to]) .expect("Kahn nodes must be distinct") .into_iter() .enumerate() { if node.set_occupied() { self.total += 1; newly_occupied[index] = true; } } for (node, newly_occupied) in [from, to].into_iter().zip(newly_occupied) { if newly_occupied && self.indegree[node] == 0 { self.nodes[node].make_ready(); self.ready.push(Reverse(node)); } } if let Err(index) = edges.binary_search(&to) { edges.insert(index, to); self.indegree[to] += 1; self.nodes[to].remove(KahnVertex::READY); } } pub(crate) fn iter_batch(&mut self) -> KahnBatchIter<'_> { KahnBatchIter { kahn: self } } pub(crate) fn pop_batch(&mut self) -> Vec { Vec::from_iter(self.iter_batch()) } pub(crate) fn complete(&mut self, node: usize) { if !self.nodes[node].is_occupied() { return; } self.emitted += 1; for &dependent in &self.edges[node] { if self.nodes[dependent].is_occupied() { self.indegree[dependent] -= 1; if self.indegree[dependent] == 0 { self.nodes[dependent].make_ready(); self.ready.push(Reverse(dependent)); } } } } pub(crate) fn is_complete(&self) -> bool { self.emitted == self.total } } /// A graph of resource uses whose compiled ordering also governs transient /// resource aliasing. Aliasing is currently safe only under the single-queue /// submission contract documented at the module level. pub struct RenderGraph { next_id: u32, resources: Vec, imported_buffers_by_ptr: HashMap, imported_buffers_by_raw: HashMap, imported_textures_by_ptr: HashMap, imported_textures_by_raw: HashMap, pub commands: CommandList, } impl RenderGraph { pub fn new() -> Self { Self { next_id: 0, resources: Vec::new(), imported_buffers_by_ptr: HashMap::new(), imported_buffers_by_raw: HashMap::new(), imported_textures_by_ptr: HashMap::new(), imported_textures_by_raw: HashMap::new(), commands: CommandList::new(), } } fn add_resource(&mut self, desc: ResourceDesc) -> ResourceId { let id = ResourceId::from_u32(self.next_id); self.next_id = self.next_id.checked_add(1).expect("resource id overflow"); self.resources.push(desc); id } pub fn create_buffer(&mut self, desc: GraphBufferDesc) -> Buffer { Buffer(self.add_resource(ResourceDesc::CreatedBuffer(desc))) } pub fn create_texture(&mut self, desc: GraphTextureDesc) -> Texture { Texture(self.add_resource(ResourceDesc::CreatedTexture(desc))) } pub fn import_buffer( &mut self, resource: impl Into>, initial: crate::resources::ResourceState, ) -> Result { let resource = resource.into(); let ptr = Arc::as_ptr(&resource) as usize; let raw = resource.raw(); let pointer_id = self.imported_buffers_by_ptr.get(&ptr).copied(); let found_id = pointer_id.or_else(|| self.imported_buffers_by_raw.get(&raw).copied()); if let Some(id) = found_id { let prior = self.resources[id.index()].initial_state(); if !same_state(prior, initial) { return Err(GraphError::ConflictingInitialState(id)); } if pointer_id.is_some() { self.imported_buffers_by_raw.insert(raw, id); } return Ok(Buffer(id)); } let id = self.add_resource(ResourceDesc::ImportedBuffer { resource, initial }); self.imported_buffers_by_ptr.insert(ptr, id); self.imported_buffers_by_raw.insert(raw, id); Ok(Buffer(id)) } pub fn import_texture( &mut self, resource: impl Into>, initial: crate::resources::ResourceState, ) -> Result { let resource = resource.into(); let ptr = Arc::as_ptr(&resource) as usize; let raw = resource.raw(); let pointer_id = self.imported_textures_by_ptr.get(&ptr).copied(); let found_id = pointer_id.or_else(|| self.imported_textures_by_raw.get(&raw).copied()); if let Some(id) = found_id { let prior = self.resources[id.index()].initial_state(); if !same_state(prior, initial) { return Err(GraphError::ConflictingInitialState(id)); } if pointer_id.is_some() { self.imported_textures_by_raw.insert(raw, id); } return Ok(Texture(id)); } let id = self.add_resource(ResourceDesc::ImportedTexture { resource, initial }); self.imported_textures_by_ptr.insert(ptr, id); self.imported_textures_by_raw.insert(raw, id); Ok(Texture(id)) } /// Transitively import resources retained by a managed descriptor set. /// /// `initial_states` must provide one explicit initial state for every unique /// buffer/image referenced by the set. Descriptor access and image layout are /// draw-use states and are not used as initial state. pub fn import_descriptor_set( &mut self, set: Arc, initial_states: &[(DescriptorResourceIdentity, ResourceState)], dynamic_offsets: &[u32], ) -> Result { let fail = |message: String| GraphError::DescriptorImport(message); let raw = set.raw().map_err(map_descriptor_set_error)?; let layout = set.layout_bindings().map_err(map_descriptor_set_error)?; let tracked = set .bindings() .map_err(map_descriptor_set_error)? .cloned() .collect::>(); let dynamic_slots = dynamic_offset_slots(layout); if dynamic_slots.len() != dynamic_offsets.len() { return Err(fail(format!( "expected {} dynamic offsets, received {}", dynamic_slots.len(), dynamic_offsets.len() ))); } for (index, &(_, _, kind)) in dynamic_slots.iter().enumerate() { let alignment = set .dynamic_offset_alignment(kind) .map_err(map_descriptor_set_error)?; if !is_dynamic_offset_aligned(dynamic_offsets[index], alignment) { return Err(fail(format!( "dynamic offset at index {index} is misaligned" ))); } } for &(binding, element, _) in &dynamic_slots { if !tracked .iter() .any(|entry| entry.binding == binding && entry.array_element == element) { return Err(fail(format!( "dynamic descriptor binding {binding} element {element} has no retained buffer" ))); } } let mut staged_accesses = Vec::new(); let mut staged_resources = Vec::new(); let mut access_resource_indices: Vec = Vec::new(); for entry in &tracked { let stage = shader_stages_to_pipeline(entry.stages) .ok_or_else(|| fail(format!("unsupported shader stage mask {:?}", entry.stages)))?; match (&entry.resource, entry.descriptor_type) { ( DescriptorResourceBinding::Buffer { buffer, offset, range, dynamic, }, kind, ) if matches!( kind, vk::DescriptorType::UNIFORM_BUFFER | vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC | vk::DescriptorType::STORAGE_BUFFER | vk::DescriptorType::STORAGE_BUFFER_DYNAMIC ) => { let dynamic_offset = if *dynamic { let index = dynamic_slots .iter() .position(|&(binding, element, _)| { binding == entry.binding && element == entry.array_element }) .ok_or_else(|| fail("dynamic descriptor ordering mismatch".into()))?; let value = u64::from(dynamic_offsets[index]); value } else { 0 }; let range = effective_buffer_range(buffer.len(), *offset, *range, dynamic_offset) .map_err(|message| fail(message.into()))?; let (access, _) = descriptor_access(kind) .ok_or_else(|| fail(format!("unsupported descriptor type {kind:?}")))?; staged_accesses.push(DescriptorResourceAccess::Buffer { buffer: Buffer(ResourceId::from_u32(u32::MAX)), range, stage, access, }); staged_resources.push(StagedImportResource::Buffer(buffer.clone())); access_resource_indices.push(staged_resources.len() - 1); } ( DescriptorResourceBinding::Image { view, range, layout, .. }, kind, ) if matches!( kind, vk::DescriptorType::SAMPLED_IMAGE | vk::DescriptorType::COMBINED_IMAGE_SAMPLER | vk::DescriptorType::STORAGE_IMAGE ) => { let (access, forced_layout) = descriptor_access(kind) .ok_or_else(|| fail(format!("unsupported descriptor type {kind:?}")))?; let layout = forced_layout.unwrap_or(*layout); staged_accesses.push(DescriptorResourceAccess::Texture { texture: Texture(ResourceId::from_u32(u32::MAX)), range: *range, layout, stage, access, }); staged_resources.push(StagedImportResource::Texture(view.image().clone())); access_resource_indices.push(staged_resources.len() - 1); } (DescriptorResourceBinding::Sampler { .. }, vk::DescriptorType::SAMPLER) => {} (DescriptorResourceBinding::TexelBuffer { view, range }, kind) if matches!( kind, vk::DescriptorType::UNIFORM_TEXEL_BUFFER | vk::DescriptorType::STORAGE_TEXEL_BUFFER ) => { let buffer = view.buffer(); let start = range.offset; let size = range.size; if start.checked_add(size).is_none_or(|end| end > buffer.len()) { return Err(fail("texel buffer range exceeds buffer length".into())); } let (access, _) = descriptor_access(kind) .ok_or_else(|| fail(format!("unsupported descriptor type {kind:?}")))?; staged_accesses.push(DescriptorResourceAccess::Buffer { buffer: Buffer(ResourceId::from_u32(u32::MAX)), range: BufferRange { offset: start, size, }, stage, access, }); staged_resources.push(StagedImportResource::Buffer(buffer.clone())); access_resource_indices.push(staged_resources.len() - 1); } _ => { return Err(fail(format!( "descriptor type {:?} has incompatible or unsupported owner", entry.descriptor_type ))); } } } // Validate that every distinct referenced owner has exactly one usable state // before mutating graph resources, then route imports through the deduping APIs. let mut resolved_states = Vec::with_capacity(staged_resources.len()); for owner in &staged_resources { let state = state_for_identity(initial_states, owner).ok_or_else(|| { fail("missing or conflicting explicit initial state for descriptor resource".into()) })?; resolved_states.push(state); } for (identity, state) in initial_states { let supplied = match identity { DescriptorResourceIdentity::Buffer(buffer) => { StagedImportResource::Buffer(buffer.clone()) } DescriptorResourceIdentity::Texture(image) => { StagedImportResource::Texture(image.clone()) } }; if !staged_resources .iter() .any(|resource| same_imported_identity(resource, &supplied)) { return Err(fail( "initial state supplied for a resource not referenced by the descriptor set" .into(), )); } if let Some((id, existing)) = self.find_imported(&supplied) && !same_state(existing, *state) { return Err(GraphError::ConflictingInitialState(id)); } } for (owner, state) in staged_resources.iter().zip(&resolved_states) { if let Some((id, existing_state)) = self.find_imported(owner) && !same_state(existing_state, *state) { return Err(GraphError::ConflictingInitialState(id)); } } for index in 0..staged_resources.len() { for other in (index + 1)..staged_resources.len() { if same_imported_identity(&staged_resources[index], &staged_resources[other]) && !same_state(resolved_states[index], resolved_states[other]) { if let Some((id, _)) = self.find_imported(&staged_resources[index]) { return Err(GraphError::ConflictingInitialState(id)); } return Err(GraphError::ConflictingDescriptorInitialStates); } } } let mut texture_scopes = Vec::new(); for (access, &resource_index) in staged_accesses.iter().zip(&access_resource_indices) { if let DescriptorResourceAccess::Texture { range, layout, .. } = access { texture_scopes.push((resource_index, *range, *layout)); } } for (index, &(resource_a, range_a, layout_a)) in texture_scopes.iter().enumerate() { for &(resource_b, range_b, layout_b) in &texture_scopes[index + 1..] { if same_imported_identity( &staged_resources[resource_a], &staged_resources[resource_b], ) && texture_layouts_conflict(range_a, layout_a, range_b, layout_b) { return Err(GraphError::ConflictingDescriptorDrawLayouts); } } } for (index, first) in staged_accesses.iter().enumerate() { if !matches!(first, DescriptorResourceAccess::Texture { .. }) { continue; } let DescriptorResourceAccess::Texture { range: range_a, layout: layout_a, .. } = first else { continue; }; let first_resource = access_resource_indices[index]; for (second_index, second) in staged_accesses.iter().enumerate().skip(index + 1) { let DescriptorResourceAccess::Texture { range: range_b, layout: layout_b, .. } = second else { continue; }; let second_resource = access_resource_indices[second_index]; if same_imported_identity( &staged_resources[first_resource], &staged_resources[second_resource], ) && texture_layouts_conflict(*range_a, *layout_a, *range_b, *layout_b) { return Err(GraphError::ConflictingDescriptorDrawLayouts); } } } let resolved_ids = self.import_transaction(|graph| { let mut resolved_ids = Vec::with_capacity(staged_resources.len()); for (owner, state) in staged_resources.iter().zip(resolved_states) { let id = match owner { StagedImportResource::Buffer(resource) => graph .import_buffer(resource.clone(), state) .map(|id| id.id()), StagedImportResource::Texture(resource) => graph .import_texture(resource.clone(), state) .map(|id| id.id()), }?; resolved_ids.push(id); } Ok(resolved_ids) })?; for (access, &resource_index) in staged_accesses.iter_mut().zip(&access_resource_indices) { let id = resolved_ids[resource_index]; match access { DescriptorResourceAccess::Buffer { buffer, .. } => { *buffer = Buffer(id); } DescriptorResourceAccess::Texture { texture, .. } => { *texture = Texture(id); } } } Ok(GraphDescriptorSet { raw, accesses: staged_accesses, managed: Some(set), dynamic_offsets: dynamic_offsets.to_vec(), }) } /// Tracks a raw descriptor set without ownership or content verification. /// /// This is the untracked escape hatch. The caller must retain every bound /// resource and declare all accesses accurately; prefer `import_descriptor_set` /// for managed sets. The graph cannot verify raw descriptor contents. pub fn track_descriptor_set( &self, raw: vk::DescriptorSet, accesses: Vec, ) -> GraphDescriptorSet { GraphDescriptorSet { raw, accesses, managed: None, dynamic_offsets: Vec::new(), } } pub fn push(&mut self, command: C) -> CommandIndex { self.commands.push(command) } /// Compile dependencies, their transitive happens-before closure, and /// lifetimes for transient-alias analysis. Placement-specific alias barriers /// are added by `CompiledGraph::resolve`. The closure is derived from /// resource-hazard edges; batch membership alone is not proof that uses are /// ordered. pub fn compile(mut self) -> Result { // Keep dependency construction on the recorded accesses; the walk adds // lifetime-only attachment accesses to `side_effects` as it proceeds. let orderings = build_command_orderings( &self.resources, &self.commands.commands, &mut self.commands.side_effects, )?; let CommandOrderings { execution_order, batches, spans: render_pass_spans, node_of_command, node_representatives, dependency_edges, } = orderings; validate_feedback_loops( &self.commands.commands, &self.commands.side_effects, &render_pass_spans, )?; // Check for conflicting draw layouts before building the dependency graph. for accesses in self.commands.side_effects.command_accesses() { for (index, first) in accesses.iter().enumerate() { for second in &accesses[index + 1..] { if first.resource == second.resource && let ( SubresourceScopeInner::Texture { range: a, layout: la, }, SubresourceScopeInner::Texture { range: b, layout: lb, }, ) = (first.scope.subresource, second.scope.subresource) && texture_layouts_conflict(a, la, b, lb) { return Err(GraphError::ConflictingDrawLayouts(first.resource)); } } } } let command_count = self.commands.len(); let accesses = self.commands.side_effects.command_accesses(); let compiler = GraphCompiler::new_with_nodes( &self.resources, accesses, command_count, &self.commands.side_effects, node_of_command, node_representatives, &render_pass_spans, ); compiler.validate_resources()?; let mut barriers = compiler.build_barriers(&execution_order); hoist_span_barriers(&mut barriers, &render_pass_spans, command_count); validate_intra_pass_barriers(&barriers, &render_pass_spans)?; let happens_before = HappensBefore::from_dag( &dependency_edges, &execution_order, &compiler.node_of_command, &compiler.node_representatives, ); let uses_by_resource = compiler.resource_uses(); // Span graphs need contracted uses so accesses within one pass become // a single node. Without spans, command nodes are the identity mapping: // command-level frontiers are therefore exactly the contracted // frontiers, and avoid the otherwise redundant collapsed-use map. let frontiers = if render_pass_spans.is_empty() { let has_created_resource_uses = uses_by_resource .keys() .any(|id| self.resources[id.index()].is_created()); if has_created_resource_uses { ResourceFrontiers::from_command_uses(&uses_by_resource, &happens_before) } else { ResourceFrontiers::default() } } else { let collapsed_uses_by_resource = compiler.collapsed_resource_uses(&uses_by_resource); // Imported resources never enter transient packing; avoid frontier // construction when there is no created resource that could be aliased. let has_created_resource_uses = collapsed_uses_by_resource .keys() .any(|id| self.resources[id.index()].is_created()); if !has_created_resource_uses { ResourceFrontiers::default() } else { ResourceFrontiers::from_collapsed_uses(&collapsed_uses_by_resource, &happens_before) } }; let (lifetimes, interference) = compiler.bind_lifetimes( &execution_order, &batches, &frontiers, &happens_before, &uses_by_resource, &render_pass_spans, ); let ordinary_barrier_count = barriers.len(); let has_created_views = render_pass_spans.iter().any(|span| { self.commands.commands[span.begin.index()] .render_pass_desc() .is_some_and(|desc| { desc.color .iter() .any(|attachment| matches!(attachment.view, AttachmentView::Created(_))) || desc.depth.is_some_and(|attachment| { matches!(attachment.view, AttachmentView::Created(_)) }) }) }); compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); hoist_span_barriers(&mut barriers, &render_pass_spans, command_count); validate_intra_pass_barriers(&barriers, &render_pass_spans)?; let barrier_indices_by_command = grouped_barrier_indices(&barriers, command_count); let GraphCompiler { node_of_command: compiled_node_of_command, node_representatives: compiled_node_representatives, .. } = compiler; Ok(CompiledGraph { resources: self.resources, resolved: HashMap::new(), views: HashMap::new(), created_view_bindings: Vec::new(), has_created_views, placements_resolved: false, commands: self.commands, batches, execution_order, barriers, barrier_indices_by_command, ordinary_barrier_count, lifetimes, interference, dependency_edges, node_of_command: compiled_node_of_command, node_representatives: compiled_node_representatives, happens_before, uses_by_resource, frontiers, render_pass_spans, }) } } fn validate_feedback_loops( commands: &[Box], side_effects: &SideEffectMap, spans: &[RenderPassSpan], ) -> Result<(), GraphError> { for span in spans { let Some(desc) = commands[span.begin.index()].render_pass_desc() else { continue; }; let attachments = desc .color .iter() .map(|attachment| (attachment.texture.id(), attachment.range)) .chain( desc.depth .into_iter() .map(|attachment| (attachment.texture.id(), attachment.range)), ) .collect::>(); for command in span.interior_range().map(CommandIndex::from) { for access in side_effects.accesses(command) { let SubresourceScopeInner::Texture { range, .. } = access.scope.subresource else { continue; }; if attachments.iter().any(|(id, attachment_range)| { *id == access.resource && image_range_intersection(range, *attachment_range).is_some() }) { return Err(GraphError::RenderPassFeedbackLoop(format!( "texture {:?} is accessed by command {:?} inside its render pass", access.resource, command ))); } } } } Ok(()) } /// Move outside-origin interior transitions to their span begin, retaining the /// sequence of transitions so Vulkan observes chained layouts in order. fn hoist_span_barriers( barriers: &mut Vec, spans: &[RenderPassSpan], command_count: usize, ) { if spans.is_empty() { return; } let mut hoisted = (0..command_count) .map(|_| None::>) .collect::>(); let mut hoisted_memory = (0..command_count) .map(|_| None::>) .collect::>(); let mut interior_owner = vec![None; command_count]; for span in spans { hoisted[span.begin.index()] = Some(Vec::new()); hoisted_memory[span.begin.index()] = Some(Vec::new()); for command in span.interior_range().map(CommandIndex::from) { interior_owner[command.index()] = Some(span.begin); } } for barrier in barriers.iter_mut() { let Some(begin) = interior_owner[barrier.before_command.index()] else { continue; }; // Legacy global memory barriers carry no per-resource origin, so retain // their conservative pass-boundary hoisting behavior. hoisted_memory[begin.index()] .as_mut() .expect("span begin has a memory-barrier entry") .append(&mut barrier.memory_barriers); let mut retained = tinyvec::TinyVec::<[BarrierResource; 2]>::new(); for resource in barrier.resources.drain(..) { let outside_origin = match resource.origin { BarrierOrigin::Initial => true, BarrierOrigin::Command(origin) => interior_owner[origin.index()] != Some(begin), }; if outside_origin { let resources = hoisted[begin.index()] .as_mut() .expect("span has hoisted resources"); resources.push(resource); } else { retained.push(resource); } } barrier.resources = retained; } barriers.retain(|barrier| !barrier.resources.is_empty() || !barrier.memory_barriers.is_empty()); let mut barrier_index_by_command = vec![None; command_count]; for (index, barrier) in barriers.iter().enumerate() { barrier_index_by_command[barrier.before_command.index()].get_or_insert(index); } for span in spans { let begin = span.begin; let resources = hoisted[begin.index()].take().unwrap_or_default(); let memory_barriers = coalesce_hoisted_memory(hoisted_memory[begin.index()].take().unwrap_or_default()); let resources = coalesce_hoisted_resources(resources); if resources.is_empty() && memory_barriers.is_empty() { continue; } if let Some(index) = barrier_index_by_command[begin.index()] { barriers[index].memory_barriers.extend(memory_barriers); barriers[index].resources.extend(resources); } else { barriers.push(PipelineBarrier { before_command: begin, memory_barriers, resources: resources.into_iter().collect(), }); barrier_index_by_command[begin.index()] = Some(barriers.len() - 1); } } } fn coalesce_hoisted_resources(incoming: Vec) -> Vec { let mut resources = Vec::new(); let mut exact_image_indices = HashMap::new(); let mut target_image_indices = HashMap::new(); let mut target_range_indices: HashMap<(ResourceId, vk::ImageLayout), Vec> = HashMap::new(); let mut buffer_indices = HashMap::new(); for resource in incoming { coalesce_hoisted_image_barrier( &mut resources, &mut exact_image_indices, &mut target_image_indices, &mut target_range_indices, &mut buffer_indices, resource, ); } resources } fn coalesce_hoisted_image_barrier( resources: &mut Vec, exact_image_indices: &mut HashMap< ( ResourceId, (u32, u32, u32, u32, u32), vk::ImageLayout, vk::ImageLayout, BarrierOrigin, ), usize, >, target_image_indices: &mut HashMap< ( ResourceId, (u32, u32, u32, u32, u32), vk::ImageLayout, BarrierOrigin, ), Vec, >, target_range_indices: &mut HashMap<(ResourceId, vk::ImageLayout), Vec>, buffer_indices: &mut HashMap<(ResourceId, u64, u64, BarrierOrigin, u64, u64), usize>, mut incoming: BarrierResource, ) { let Subresource::Texture(incoming_range) = incoming.range else { let Subresource::Buffer(incoming_range) = incoming.range else { unreachable!() }; let key = ( incoming.id, incoming_range.offset, incoming_range.size, incoming.origin, incoming.src_stage.as_raw(), incoming.src_access.as_raw(), ); if let Some(&index) = buffer_indices.get(&key) { let existing = &mut resources[index]; existing.dst_stage |= incoming.dst_stage; existing.dst_access |= incoming.dst_access; } else { buffer_indices.insert(key, resources.len()); resources.push(incoming); } return; }; let (Some(old_layout), Some(new_layout)) = (incoming.old_layout, incoming.new_layout) else { resources.push(incoming); return; }; let range_key = image_range_key(incoming_range); let exact_key = ( incoming.id, range_key, old_layout, new_layout, incoming.origin, ); let existing_index = exact_image_indices.get(&exact_key).copied().or_else(|| { [new_layout, old_layout].into_iter().find_map(|layout| { target_image_indices .get(&(incoming.id, range_key, layout, incoming.origin)) .and_then(|indices| { indices.iter().copied().find(|&index| { resources[index].new_layout == Some(new_layout) && (resources[index].old_layout == Some(old_layout) || resources[index].new_layout == Some(old_layout)) }) }) }) }); if let Some(index) = existing_index { let existing = &mut resources[index]; existing.dst_stage |= incoming.dst_stage; existing.dst_access |= incoming.dst_access; return; } // Keep only subresources not already transitioned to this target layout. // Layout-region normalization normally makes these residual pieces empty or // disjoint, but retaining subtraction here protects partially overlapping ranges. let mut residual = vec![incoming_range]; for &index in target_range_indices .get(&(incoming.id, new_layout)) .into_iter() .flatten() { let existing = &resources[index]; let Subresource::Texture(covered) = existing.range else { continue; }; if same_image_subresource_range(covered, incoming_range) { continue; } residual = residual .into_iter() .flat_map(|piece| subtract_image_range(piece, covered).iter()) .collect(); if residual.is_empty() { break; } } for range in residual { incoming.range = Subresource::Texture(range); let key = ( incoming.id, image_range_key(range), old_layout, new_layout, incoming.origin, ); exact_image_indices.insert(key, resources.len()); target_image_indices .entry(( incoming.id, image_range_key(range), old_layout, incoming.origin, )) .or_default() .push(resources.len()); target_image_indices .entry(( incoming.id, image_range_key(range), new_layout, incoming.origin, )) .or_default() .push(resources.len()); target_range_indices .entry((incoming.id, new_layout)) .or_default() .push(resources.len()); resources.push(incoming); } } fn image_range_key(range: vk::ImageSubresourceRange) -> (u32, u32, u32, u32, u32) { ( range.aspect_mask.as_raw(), range.base_mip_level, range.level_count, range.base_array_layer, range.layer_count, ) } fn coalesce_hoisted_memory(incoming: Vec) -> Vec { let mut indices = HashMap::new(); let mut barriers = Vec::new(); for barrier in incoming { let key = ( barrier.src_stage, barrier.src_access, barrier.dst_stage, barrier.dst_access, ); if indices.contains_key(&key) { continue; } indices.insert(key, barriers.len()); barriers.push(barrier); } barriers } fn validate_intra_pass_barriers( barriers: &[PipelineBarrier], spans: &[RenderPassSpan], ) -> Result<(), GraphError> { if spans.is_empty() { return Ok(()); } for barrier in barriers { if is_render_pass_interior(barrier.before_command, spans) && barrier.resources.iter().any(|resource| { matches!((resource.old_layout, resource.new_layout), (Some(old), Some(new)) if old != new) }) { return Err(GraphError::UnsupportedIntraPassDependency); } } Ok(()) } fn is_render_pass_interior(command: CommandIndex, spans: &[RenderPassSpan]) -> bool { let span_index = spans.partition_point(|span| span.end < command); spans .get(span_index) .is_some_and(|span| span.interior_range().contains(&command.index())) } fn flatten_batches(batches: &[Vec]) -> Vec { batches.iter().flatten().copied().collect() } pub(crate) fn validate_render_pass_attachments( resources: &[ResourceDesc], desc: RenderPassDesc<'_>, ) -> Result<(), GraphError> { let mut seen_subresources = HashSet::new(); let attachments = desc .color .iter() .map(|attachment| { ( attachment.texture, &attachment.view, attachment.range, attachment.layout, true, ) }) .chain(desc.depth.into_iter().map(|attachment| { ( attachment.texture, &attachment.view, attachment.range, attachment.layout, false, ) })); for (texture, view, range, layout, is_color) in attachments { let id = texture.id(); let Some(resource) = resources.get(id.index()) else { return Err(GraphError::UnknownResource(id)); }; let reason = |message: &str| { GraphError::InvalidRenderPassAttachment(format!("texture {id:?}: {message}")) }; let counts = match resource { ResourceDesc::CreatedTexture(desc) => Some((desc.mip_levels, desc.array_layers)), ResourceDesc::ImportedTexture { .. } => None, ResourceDesc::CreatedBuffer(_) | ResourceDesc::ImportedBuffer { .. } => { return Err(reason("attachment resource is not a texture")); } }; if let AttachmentView::Imported(view) = view && let ResourceDesc::ImportedTexture { resource, .. } = resource && view.image().as_ref() != resource.as_ref() { return Err(reason("imported view refers to a different image")); } if let AttachmentView::Created(view) = view && (view.mip_range.end == vk::REMAINING_MIP_LEVELS || view.layer_range.end == vk::REMAINING_ARRAY_LAYERS) { return Err(reason("created view uses a remaining-range sentinel")); } if let AttachmentView::Created(view) = view && (view.mip_range.end < view.mip_range.start || view.layer_range.end < view.layer_range.start) { return Err(reason("created view range is reversed")); } let view_range = match view { AttachmentView::Imported(view) => view.subresource_range(), AttachmentView::Created(view) => vk::ImageSubresourceRange::default() .aspect_mask(view.aspect) .base_mip_level(view.mip_range.start) .level_count(view.mip_range.end - view.mip_range.start) .base_array_layer(view.layer_range.start) .layer_count(view.layer_range.end - view.layer_range.start), }; if !same_image_subresource_range(range, view_range) { return Err(reason("declared range does not match the attachment view")); } if !is_legal_attachment_layout(layout, is_color) { return Err(reason("layout is not legal for this attachment type")); } if range.level_count == vk::REMAINING_MIP_LEVELS || range.layer_count == vk::REMAINING_ARRAY_LAYERS || range.level_count == 0 || range.layer_count == 0 { return Err(reason("attachment range must be concrete and non-empty")); } let Some(mip_end) = range.base_mip_level.checked_add(range.level_count) else { return Err(reason("mip range overflows")); }; let Some(layer_end) = range.base_array_layer.checked_add(range.layer_count) else { return Err(reason("layer range overflows")); }; if let Some((mip_count, layer_count)) = counts && (mip_end > mip_count || layer_end > layer_count) { return Err(reason("attachment range exceeds texture mip/layer counts")); } let aspects = [ vk::ImageAspectFlags::COLOR, vk::ImageAspectFlags::DEPTH, vk::ImageAspectFlags::STENCIL, ] .into_iter() .filter(|aspect| range.aspect_mask.intersects(*aspect)) .collect::>(); let known_aspects = aspects .iter() .copied() .fold(vk::ImageAspectFlags::empty(), |mask, aspect| mask | aspect); if aspects.is_empty() || known_aspects != range.aspect_mask { return Err(reason("attachment range has unsupported aspect bits")); } for aspect in aspects { for mip in range.base_mip_level..mip_end { for layer in range.base_array_layer..layer_end { if !seen_subresources.insert((id, aspect.as_raw(), mip, layer)) { return Err(reason("duplicate attachment subresource")); } } } } } Ok(()) } fn same_image_subresource_range( a: vk::ImageSubresourceRange, b: vk::ImageSubresourceRange, ) -> bool { a.aspect_mask == b.aspect_mask && a.base_mip_level == b.base_mip_level && a.level_count == b.level_count && a.base_array_layer == b.base_array_layer && a.layer_count == b.layer_count } fn is_legal_attachment_layout(layout: vk::ImageLayout, is_color: bool) -> bool { if is_color { matches!( layout, vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL | vk::ImageLayout::GENERAL ) } else { matches!( layout, vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL | vk::ImageLayout::DEPTH_READ_ONLY_OPTIMAL | vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL | vk::ImageLayout::DEPTH_ATTACHMENT_STENCIL_READ_ONLY_OPTIMAL | vk::ImageLayout::GENERAL ) } } struct GraphCompiler<'a> { resources: &'a [ResourceDesc], accesses: &'a [Vec], command_count: usize, node_of_command: Vec, node_representatives: Vec, node_count: usize, lifetime_only: BTreeSet<(usize, ResourceId)>, } impl<'a> GraphCompiler<'a> { fn new_with_nodes( resources: &'a [ResourceDesc], accesses: &'a [Vec], command_count: usize, side_effects: &SideEffectMap, node_of_command: Vec, node_representatives: Vec, spans: &[RenderPassSpan], ) -> Self { let node_count = node_representatives.len(); Self { resources, accesses, command_count, node_of_command, node_representatives, node_count, lifetime_only: if spans.is_empty() { BTreeSet::new() } else { accesses .iter() .enumerate() .flat_map(|(command, accesses)| { accesses.iter().filter_map(move |access| { side_effects .is_lifetime_only(command.into(), access.resource) .then_some((command, access.resource)) }) }) .collect() }, } } #[cfg(test)] fn new( resources: &'a [ResourceDesc], accesses: &'a [Vec], command_count: usize, side_effects: &SideEffectMap, spans: &[RenderPassSpan], ) -> Self { let mut node_of_command = vec![usize::MAX; command_count]; let mut node_representatives = Vec::new(); let mut span_index = 0; let mut command = 0; while command < command_count { let node = node_representatives.len(); if let Some(span) = spans.get(span_index) && span.begin.index() == command { node_representatives.push(span.begin); node_of_command[command..=span.end.index()].fill(node); command = span.end.index() + 1; span_index += 1; } else { node_representatives.push(CommandIndex(command)); node_of_command[command] = node; command += 1; } } Self::new_with_nodes( resources, accesses, command_count, side_effects, node_of_command, node_representatives, spans, ) } fn is_lifetime_only(&self, command: CommandIndex, resource: ResourceId) -> bool { self.lifetime_only.contains(&(command.index(), resource)) } fn validate_resources(&self) -> Result<(), GraphError> { for command_accesses in self.accesses { for access in command_accesses { if access.resource.index() >= self.resources.len() { return Err(GraphError::UnknownResource(access.resource)); } } } Ok(()) } fn build_barriers(&self, execution_order: &[CommandIndex]) -> Vec { let initial_states = self .resources .iter() .map(ResourceDesc::initial_state) .collect::>(); self.build_barriers_with_initial_states(execution_order, &initial_states) } fn build_barriers_with_initial_states( &self, execution_order: &[CommandIndex], initial_states: &[crate::resources::ResourceState], ) -> Vec { let mut execution_positions = vec![usize::MAX; self.command_count]; for (position, command) in execution_order.iter().enumerate() { execution_positions[command.index()] = position; } let mut result: Vec = Vec::with_capacity(self.command_count); let mut prior_accesses_by_resource: Vec< Vec<(CommandIndex, crate::recorder::ResourceAccess)>, > = (0..self.resources.len()).map(|_| Vec::new()).collect(); let mut prior_access_summaries = (0..self.resources.len()) .map(|_| None::) .collect::>(); let mut current_layouts: Vec> = (0..self.resources.len()).map(|_| Vec::new()).collect(); for &command_index in execution_order { let mut barrier_resources = tinyvec::TinyVec::new(); for current in &self.accesses[command_index.index()] { if self.is_lifetime_only(command_index, current.resource) { continue; } // The most recent conflicting access is the complete source scope // for this barrier; prior barrier masks are not carried forward. let read_only = crate::resources::access_flag_is_read(current.scope.access) && !access_flag_is_write(current.scope.access); let can_skip_history_scan = read_only && prior_access_summaries[current.resource.index()] .as_ref() .is_some_and(|summary| { !summary.has_write && !summary.has_non_read_only && match current.scope.subresource { SubresourceScopeInner::Buffer(_) => true, SubresourceScopeInner::Texture { layout, .. } => { summary.uniform_layout == Some(layout) } } }); let previous = (!can_skip_history_scan) .then(|| &prior_accesses_by_resource[current.resource.index()]) .and_then(|prior_accesses| { prior_accesses .iter() .rev() .find(|(_, prior)| prior.scope.conflicts(¤t.scope)) }) .map(|(command, access)| (*command, *access)); if let Some((previous_command, previous)) = previous { debug_assert!( execution_positions[previous_command.index()] < execution_positions[command_index.index()] ); let (fallback_layout, new_layout) = Self::layouts(previous.scope, current.scope); let pieces = match (scope_range(current.scope), fallback_layout) { (Subresource::Texture(range), Some(fallback)) => layout_pieces( range, fallback, ¤t_layouts[current.resource.index()], ) .into_iter() .map(|(range, layout)| (Subresource::Texture(range), Some(layout))) .collect::>(), (range, old_layout) => vec![(range, old_layout)], }; for (range, old_layout) in pieces { barrier_resources.push(BarrierResource { id: current.resource, range, origin: BarrierOrigin::Command(previous_command), src_stage: previous.scope.stage, src_access: previous.scope.access, dst_stage: current.scope.stage, dst_access: current.scope.access, old_layout, new_layout, queue_transfer: None, }); if let (Subresource::Texture(range), Some(old), Some(new)) = (range, old_layout, new_layout) && old != new { update_layout_regions( &mut current_layouts[current.resource.index()], range, new, ); } } continue; } let desc = &self.resources[current.resource.index()]; let initial = initial_states[current.resource.index()]; let (old_layout, new_layout) = Self::layouts_from_state(initial, current.scope); let layout_transition = old_layout != new_layout && new_layout.is_some(); let initial_hazard = matches!( desc, ResourceDesc::ImportedBuffer { .. } | ResourceDesc::ImportedTexture { .. } ) && !initial.access.is_empty() && (access_flag_is_write(initial.access) || access_flag_is_write(current.scope.access)); if layout_transition || initial_hazard { let pieces = match (scope_range(current.scope), old_layout) { (Subresource::Texture(range), Some(fallback)) => layout_pieces( range, fallback, ¤t_layouts[current.resource.index()], ) .into_iter() .map(|(range, layout)| (Subresource::Texture(range), Some(layout))) .collect::>(), (range, old_layout) => vec![(range, old_layout)], }; for (range, old_layout) in pieces { barrier_resources.push(BarrierResource { id: current.resource, range, origin: BarrierOrigin::Initial, src_stage: initial.stage, src_access: initial.access, dst_stage: current.scope.stage, dst_access: current.scope.access, old_layout, new_layout, queue_transfer: None, }); if let (Subresource::Texture(range), Some(old), Some(new)) = (range, old_layout, new_layout) && old != new { update_layout_regions( &mut current_layouts[current.resource.index()], range, new, ); } } } } if !barrier_resources.is_empty() { result.push(PipelineBarrier { before_command: command_index, memory_barriers: Vec::new(), resources: barrier_resources, }); } for access in &self.accesses[command_index.index()] { if self.is_lifetime_only(command_index, access.resource) { continue; } prior_accesses_by_resource[access.resource.index()].push((command_index, *access)); let summary = prior_access_summaries[access.resource.index()] .get_or_insert_with(PriorAccessSummary::default); let is_read = crate::resources::access_flag_is_read(access.scope.access); let is_write = access_flag_is_write(access.scope.access); summary.has_write |= is_write; summary.has_non_read_only |= !is_read || is_write; if let SubresourceScopeInner::Texture { layout, .. } = access.scope.subresource { if let Some(uniform_layout) = summary.uniform_layout && uniform_layout != layout { summary.uniform_layout = None; summary.texture_layouts_diverged = true; } else if summary.uniform_layout.is_none() && !summary.texture_layouts_diverged { summary.uniform_layout = Some(layout); } } } } result } fn build_alias_barriers( &self, barriers: &mut Vec, lifetimes: &HashMap, frontiers: &ResourceFrontiers, happens_before: &HappensBefore, ) { let mut barrier_index_by_command = vec![None; self.command_count]; for (index, barrier) in barriers.iter().enumerate() { barrier_index_by_command[barrier.before_command.index()].get_or_insert(index); } let mut resources_by_pool: HashMap> = HashMap::new(); for (&id, lifetime) in lifetimes { if let Some(pool) = lifetime.placement.pool { resources_by_pool.entry(pool).or_default().push(id); } } let mut new_resources = lifetimes .iter() .filter_map(|(&id, lifetime)| { (lifetime.placement.aliased && !lifetime.imported).then_some(id) }) .collect::>(); new_resources.sort_unstable(); for new_id in new_resources { let Some(new_lifetime) = lifetimes.get(&new_id) else { continue; }; let new_first_uses = frontiers.alias_initial(new_id); if new_first_uses.is_empty() { continue; } let mut old_resources = new_lifetime .placement .pool .and_then(|pool| resources_by_pool.get(&pool)) .into_iter() .flatten() .filter_map(|&old_id| { let old_lifetime = lifetimes.get(&old_id)?; (old_id != new_id && !old_lifetime.imported && placement_ranges_overlap(old_lifetime.placement, new_lifetime.placement)) .then_some(old_id) }) .collect::>(); old_resources.sort_unstable(); let old_terminal_uses = old_resources .iter() .map(|&old_id| frontiers.alias_terminal(old_id)) .collect::>(); for first_use in new_first_uses.iter() { let first_use_command = ResourceUse { command: first_use.first_command(), stage: first_use.stage(), access: first_use.access(), }; let mut old_stage = vk::PipelineStageFlags2::NONE; let mut old_access = vk::AccessFlags2::NONE; let mut old_image_writes = false; let mut has_old_image = false; let mut merged_src_stage = vk::PipelineStageFlags2::NONE; let mut merged_src_access = vk::AccessFlags2::NONE; let mut has_old_buffer = false; for old_terminal_uses in &old_terminal_uses { if old_terminal_uses.is_empty() || !old_terminal_uses.iter().all(|terminal| { happens_before .happens_before(terminal.last_command(), first_use.first_command()) }) { continue; } let (src_stage, src_access) = aggregate_alias_uses(old_terminal_uses.iter()); match &self.resources[new_id.index()] { ResourceDesc::CreatedBuffer(_) => { has_old_buffer = true; merged_src_stage |= src_stage; merged_src_access |= if old_terminal_uses .iter() .any(|use_| access_flag_is_write(use_.access())) { src_access } else { vk::AccessFlags2::NONE }; } ResourceDesc::CreatedTexture(_) => { has_old_image = true; old_stage |= src_stage; old_access |= src_access; old_image_writes |= old_terminal_uses .iter() .any(|use_| access_flag_is_write(use_.access())); } ResourceDesc::ImportedBuffer { .. } | ResourceDesc::ImportedTexture { .. } => {} } } if has_old_image { let barrier = pipeline_barrier_at_command_indexed( barriers, &mut barrier_index_by_command, first_use.first_command(), ); let mut found_image_barrier = false; for image_barrier in barrier .resources .iter_mut() .filter(|resource| resource.id == new_id) { found_image_barrier = true; image_barrier.src_stage = old_stage; image_barrier.src_access = if old_image_writes { old_access } else { vk::AccessFlags2::NONE }; image_barrier.dst_stage = first_use.stage(); image_barrier.dst_access = first_use.access(); image_barrier.old_layout = Some(vk::ImageLayout::UNDEFINED); image_barrier.origin = BarrierOrigin::Initial; } if !found_image_barrier { let first_scope = first_use_scope(&first_use_command, self.accesses, new_id); barrier.resources.push(BarrierResource { id: new_id, range: scope_range(first_scope), origin: BarrierOrigin::Initial, src_stage: old_stage, src_access: if old_image_writes { old_access } else { vk::AccessFlags2::NONE }, dst_stage: first_use.stage(), dst_access: first_use.access(), old_layout: Some(vk::ImageLayout::UNDEFINED), new_layout: match first_scope.subresource { SubresourceScopeInner::Texture { layout, .. } => Some(layout), SubresourceScopeInner::Buffer(_) => None, }, queue_transfer: None, }); } } if has_old_buffer { let barrier = pipeline_barrier_at_command_indexed( barriers, &mut barrier_index_by_command, first_use.first_command(), ); // Vulkan aliasing needs one dependency from all outgoing terminal uses // to this incoming first use. Availability (src masks) is unscoped for // a buffer barrier, while visibility (dst masks) is scoped to its range; // targeting the new owner is therefore sufficient. A global barrier is // valid but strictly coarser. Keep the union from every overlapping // occupant, including execution-only src masks for read-only terminals. let mut found_buffer_barrier = false; for buffer_barrier in barrier .resources .iter_mut() .filter(|resource| resource.id == new_id) { found_buffer_barrier = true; buffer_barrier.src_stage |= merged_src_stage; buffer_barrier.src_access |= merged_src_access; // Preserve the entry's destination scope and origin: a command-origin // entry inside a pass remains there, unlike a newly added Initial entry. } if !found_buffer_barrier { // Initial origin gives newly added alias dependencies the same // span-begin hoisting behavior as the former global barriers. // Each graph buffer owns a distinct VkBuffer handle, even when its // allocation is pooled at Placement::offset. Barrier ranges are // relative to that handle, so cover it fully rather than using the // backing allocation's placement offset/size. barrier.resources.push(BarrierResource { id: new_id, range: Subresource::Buffer(BufferRange::full()), origin: BarrierOrigin::Initial, src_stage: merged_src_stage, src_access: merged_src_access, dst_stage: first_use.stage(), dst_access: first_use.access(), old_layout: None, new_layout: None, queue_transfer: None, }); } } } } } fn layouts( previous: SubresourceScope, current: SubresourceScope, ) -> (Option, Option) { let layout = |scope: SubresourceScope| match scope.subresource { SubresourceScopeInner::Texture { layout, .. } => Some(layout), SubresourceScopeInner::Buffer(_) => None, }; (layout(previous), layout(current)) } fn layouts_from_state( initial: crate::resources::ResourceState, current: SubresourceScope, ) -> (Option, Option) { let current_layout = match current.subresource { SubresourceScopeInner::Texture { layout, .. } => Some(layout), SubresourceScopeInner::Buffer(_) => None, }; let initial_layout = match current.subresource { SubresourceScopeInner::Texture { .. } => { Some(initial.layout.unwrap_or(vk::ImageLayout::UNDEFINED)) } SubresourceScopeInner::Buffer(_) => None, }; (initial_layout, current_layout) } fn bind_lifetimes( &self, execution_order: &[CommandIndex], batches: &[Batch], frontiers: &ResourceFrontiers, happens_before: &HappensBefore, uses_by_resource: &HashMap>, spans: &[RenderPassSpan], ) -> (HashMap, ResourceInterference) { let mut batch_for_node = vec![usize::MAX; self.node_count]; for (batch_index, batch) in batches.iter().enumerate() { // Span interiors map back to their begin node and are always in the // same execution batch as that begin command. for &command in &execution_order[batch.start..batch.start + batch.count] { let node = self.node_of_command[command.index()]; batch_for_node[node] = batch_index; } } let mut lifetimes = HashMap::new(); for (command, command_accesses) in self.accesses.iter().enumerate() { for access in command_accesses { lifetimes .entry(access.resource) .and_modify(|lifetime: &mut ResourceLifetime| { let batch = batch_for_node[self.node_of_command[command]]; lifetime.first_batch = lifetime.first_batch.min(batch); lifetime.last_batch = lifetime.last_batch.max(batch); }) .or_insert(ResourceLifetime { first_batch: batch_for_node[self.node_of_command[command]], last_batch: batch_for_node[self.node_of_command[command]], placement: Placement { offset: 0, size: match &self.resources[access.resource.index()] { ResourceDesc::CreatedBuffer(_) | ResourceDesc::CreatedTexture(_) => 0, ResourceDesc::ImportedBuffer { .. } | ResourceDesc::ImportedTexture { .. } => 0, }, alignment: 1, pool: None, aliased: false, }, imported: !self.resources[access.resource.index()].is_created(), }); } } let mut interference = ResourceInterference::from_frontiers(frontiers, happens_before); let mut pass_commands = vec![false; self.command_count]; for span in spans { pass_commands[span.begin.index()..=span.end.index()].fill(true); } let pass_resources = uses_by_resource .iter() .filter_map(|(&id, uses)| { (self.resources[id.index()].is_created() && uses.iter().any(|use_| pass_commands[use_.command.index()])) .then_some(id) }) .collect::>(); for (index, &a) in pass_resources.iter().enumerate() { for &b in &pass_resources[index + 1..] { // Different pool classes cannot share memory, so pass-boundary // alias transitions only constrain resources of the same class. if pool_class_for_resource(&self.resources[a.index()]) != pool_class_for_resource(&self.resources[b.index()]) { continue; } interference.edges[a.index()].insert(b); interference.edges[b.index()].insert(a); } } (lifetimes, interference) } #[cfg(test)] fn bind_lifetimes_for_test( &self, batches: &[Vec], frontiers: &ResourceFrontiers, happens_before: &HappensBefore, uses_by_resource: &HashMap>, spans: &[RenderPassSpan], ) -> (HashMap, ResourceInterference) { let execution_order = flatten_batches(batches); let mut range_batches = Vec::new(); let mut start = 0; for batch in batches { range_batches.push(Batch { start, count: batch.len(), }); start += batch.len(); } self.bind_lifetimes( &execution_order, &range_batches, frontiers, happens_before, uses_by_resource, spans, ) } fn resource_uses(&self) -> HashMap> { let mut uses_by_resource: HashMap> = HashMap::new(); for (command, command_accesses) in self.accesses.iter().enumerate() { for access in command_accesses { let resource_uses = uses_by_resource.entry(access.resource).or_default(); if let Some(resource_use) = resource_uses.last_mut() && resource_use.command.index() == command { resource_use.stage |= access.scope.stage; resource_use.access |= access.scope.access; } else { resource_uses.push(ResourceUse { command: command.into(), stage: access.scope.stage, access: access.scope.access, }); } } } uses_by_resource } /// Collapse each resource's command-level scopes to one first/last representative per DAG node. fn collapsed_resource_uses( &self, uses_by_resource: &HashMap>, ) -> HashMap> { collapse_uses(uses_by_resource, &self.node_of_command) } } #[derive(Default)] struct PriorAccessSummary { has_write: bool, has_non_read_only: bool, uniform_layout: Option, texture_layouts_diverged: bool, } impl ResourceInterference { fn from_frontiers(frontiers: &ResourceFrontiers, happens_before: &HappensBefore) -> Self { let mut result = Self { edges: vec![ BTreeSet::new(); frontiers.uses.len().max(frontiers.collapsed.len()).max(1) ], }; let resource_ids = if frontiers.command_level_alias_frontiers { frontiers .uses .iter() .enumerate() .filter_map(|(index, frontier)| { frontier .as_ref() .map(|_| ResourceId::from_u32(index as u32)) }) .collect::>() } else { frontiers .collapsed .iter() .enumerate() .filter_map(|(index, frontier)| { frontier .as_ref() .map(|_| ResourceId::from_u32(index as u32)) }) .collect::>() }; for (index, &a) in resource_ids.iter().enumerate() { for &b in &resource_ids[index + 1..] { if !frontiers.alias_in_order_either(a, b, happens_before) { result.edges[a.index()].insert(b); result.edges[b.index()].insert(a); } } } result } } fn terminal_uses<'a>( uses: &'a [ResourceUse], happens_before: &HappensBefore, ) -> Vec<&'a ResourceUse> { uses.iter() .filter(|candidate| { !uses.iter().any(|other| { candidate.command != other.command && happens_before.contains(candidate.command.index(), other.command.index()) }) }) .collect() } fn initial_uses<'a>( uses: &'a [ResourceUse], happens_before: &HappensBefore, ) -> Vec<&'a ResourceUse> { uses.iter() .filter(|candidate| { !uses.iter().any(|other| { candidate.command != other.command && happens_before.contains(other.command.index(), candidate.command.index()) }) }) .collect() } /// Returns whether every terminal use of A precedes every initial use of B. fn resources_alias_in_order( uses_a: &[ResourceUse], uses_b: &[ResourceUse], happens_before: &HappensBefore, ) -> bool { aliasing_permitted(uses_a, uses_b, happens_before) } /// Returns whether every terminal use of A precedes every initial use of B. /// Empty use sets are rejected rather than treated as vacuously safe. This /// ordering proves alias safety only under the current single-queue submission /// contract; if command-to-queue assignment is added, cross-queue alias /// transitions must also be semaphore edges. /// /// The graph currently records all commands into one command buffer. It has no /// command-to-queue assignment, so cross-queue aliasing is out of scope. pub fn aliasing_permitted( uses_a: &[ResourceUse], uses_b: &[ResourceUse], happens_before: &HappensBefore, ) -> bool { let terminal_a = terminal_uses(uses_a, happens_before); let initial_b = initial_uses(uses_b, happens_before); resource_use_ref_slices_alias_in_order(&terminal_a, &initial_b, happens_before) } fn resource_use_ref_slices_alias_in_order( terminal_a: &[&ResourceUse], initial_b: &[&ResourceUse], happens_before: &HappensBefore, ) -> bool { !terminal_a.is_empty() && !initial_b.is_empty() && terminal_a.iter().all(|terminal| { initial_b .iter() .all(|initial| happens_before.happens_before(terminal.command, initial.command)) }) } fn aggregate_resource_uses(uses: &[ResourceUse]) -> (vk::PipelineStageFlags2, vk::AccessFlags2) { uses.iter().fold( (vk::PipelineStageFlags2::NONE, vk::AccessFlags2::NONE), |(stage, access), resource_use| (stage | resource_use.stage, access | resource_use.access), ) } fn aggregate_alias_uses<'a>( uses: impl Iterator>, ) -> (vk::PipelineStageFlags2, vk::AccessFlags2) { uses.fold( (vk::PipelineStageFlags2::NONE, vk::AccessFlags2::NONE), |(stage, access), resource_use| { (stage | resource_use.stage(), access | resource_use.access()) }, ) } fn placement_ranges_overlap(a: Placement, b: Placement) -> bool { if a.pool.is_none() || a.pool != b.pool || a.size == 0 || b.size == 0 { return false; } let (Some(a_end), Some(b_end)) = (a.offset.checked_add(a.size), b.offset.checked_add(b.size)) else { return false; }; a.offset < b_end && b.offset < a_end } fn is_host_visible_location(location: MemoryLocation) -> bool { matches!( location, MemoryLocation::CpuToGpu | MemoryLocation::GpuToCpu ) } fn is_poolable( location: MemoryLocation, strategy: AllocationStrategy, requirements: renderer_resources::memory::MemoryRequirements, ) -> bool { !is_host_visible_location(location) && strategy != AllocationStrategy::Dedicated && !requirements.requires_dedicated_allocation } fn allocation_strategy_for( requirements: renderer_resources::memory::MemoryRequirements, ) -> AllocationStrategy { if requirements.requires_dedicated_allocation || requirements.size >= DEDICATED_ALLOCATION_THRESHOLD { AllocationStrategy::Dedicated } else { AllocationStrategy::AllocatorManaged } } fn allocation_strategy_for_size( requested_size: u64, requirements: renderer_resources::memory::MemoryRequirements, ) -> AllocationStrategy { if requested_size >= DEDICATED_ALLOCATION_THRESHOLD { AllocationStrategy::Dedicated } else { allocation_strategy_for(requirements) } } fn pool_class_for_image(tiling: vk::ImageTiling) -> PoolClass { if tiling == vk::ImageTiling::LINEAR { PoolClass::Linear } else { PoolClass::Optimal } } fn pool_class_for_resource(resource: &ResourceDesc) -> PoolClass { match resource { ResourceDesc::CreatedBuffer(_) => PoolClass::Linear, ResourceDesc::CreatedTexture(desc) => pool_class_for_image(desc.tiling), ResourceDesc::ImportedBuffer { .. } | ResourceDesc::ImportedTexture { .. } => { unreachable!("imported resources are not transient placement candidates") } } } fn validate_pooled_image_layout( id: ResourceId, desc: &GraphTextureDesc, requirements: renderer_resources::memory::MemoryRequirements, ) -> Result<(), GraphError> { if is_poolable( desc.mem_location, allocation_strategy_for(requirements), requirements, ) && desc.layout != vk::ImageLayout::UNDEFINED { return Err(GraphError::InvalidPooledImageLayout(id)); } Ok(()) } fn memory_property_flags(location: MemoryLocation) -> vk::MemoryPropertyFlags { match location { MemoryLocation::CpuToGpu | MemoryLocation::GpuToCpu => { vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT } MemoryLocation::GpuOnly => vk::MemoryPropertyFlags::DEVICE_LOCAL, MemoryLocation::Unknown => vk::MemoryPropertyFlags::empty(), } } fn pool_accepts_member( pool: &PlacementPool, class: PoolClass, property_flags: vk::MemoryPropertyFlags, requirements: renderer_resources::memory::MemoryRequirements, ) -> bool { pool.class == class && pool.property_flags == property_flags && requirements.memory_type_bits & (1 << pool.memory_type_index) != 0 } fn should_plan_created_resource( id: ResourceId, lifetimes: &HashMap, resolved: &HashMap, ) -> bool { lifetimes .get(&id) .is_some_and(|lifetime| !lifetime.imported) && !resolved.contains_key(&id) } fn set_placement( lifetimes: &mut HashMap, id: ResourceId, requirements: renderer_resources::memory::MemoryRequirements, offset: u64, pool: Option, aliased: bool, ) { if let Some(lifetime) = lifetimes.get_mut(&id) { lifetime.placement = Placement { offset, size: requirements.size, alignment: requirements.alignment, pool, aliased, }; } } fn byte_ranges_overlap(a_offset: u64, a_size: u64, b_offset: u64, b_size: u64) -> bool { let (Some(a_end), Some(b_end)) = (a_offset.checked_add(a_size), b_offset.checked_add(b_size)) else { return true; }; a_offset < b_end && b_offset < a_end } fn checked_alignment_lcm(lhs: u64, rhs: u64) -> Option { if lhs == 0 || rhs == 0 { return None; } let (mut a, mut b) = (lhs, rhs); while b != 0 { (a, b) = (b, a % b); } lhs.checked_div(a)?.checked_mul(rhs) } fn first_fit_placement( id: ResourceId, size: u64, alignment: u64, occupied: &[(ResourceId, u64)], lifetimes: &HashMap, interference: &ResourceInterference, frontiers: &ResourceFrontiers, happens_before: &HappensBefore, ) -> Option { if alignment == 0 || size == 0 { return None; } // Validate all lifetimes up front to preserve the missing-lifetime failure behavior. for (other_id, _) in occupied { lifetimes.get(other_id)?; } let mut candidate = 0_u64; let mut range_index = 0; while range_index < occupied.len() { let aligned = candidate .checked_add(alignment - 1)? .checked_div(alignment)? .checked_mul(alignment)?; let candidate_end = aligned.checked_add(size)?; let (other_id, other_offset) = occupied[range_index]; let other_size = lifetimes.get(&other_id)?.placement.size; let other_end = other_offset.checked_add(other_size); if other_end.is_some_and(|end| end <= aligned) { range_index += 1; continue; } if other_offset >= candidate_end && other_end.is_some() { return Some(aligned); } if !byte_ranges_overlap(aligned, size, other_offset, other_size) { range_index += 1; continue; } let ordered = !interference.interferes(id, other_id) && frontiers.alias_in_order_either(id, other_id, happens_before); if ordered { range_index += 1; continue; } let next_offset = other_end?; if next_offset <= aligned { return None; } candidate = next_offset; } let aligned = candidate .checked_add(alignment - 1)? .checked_div(alignment)? .checked_mul(alignment)?; aligned.checked_add(size)?; Some(aligned) } fn image_desc_for_graph_texture( desc: &GraphTextureDesc, accesses: &[crate::recorder::ResourceAccess], queue_families: QueueFlags, alloc_scheme: AllocationStrategy, flags: vk::ImageCreateFlags, ) -> ImageDesc { ImageDesc { flags, name: desc.name.clone(), format: desc.format, kind: desc.kind, mip_levels: desc.mip_levels, array_layers: desc.array_layers, samples: desc.samples, extent: desc.extent, tiling: desc.tiling, usage: derive_image_usage(accesses, desc.extra_usage), queue_families, layout: desc.layout, mem_location: desc.mem_location, alloc_scheme, } } fn first_use_scope<'a>( resource_use: &ResourceUse, accesses: &'a [Vec], resource: ResourceId, ) -> SubresourceScope { accesses[resource_use.command.index()] .iter() .find(|access| access.resource == resource) .expect("resource use has a matching declared access") .scope } fn pipeline_barrier_at_command( barriers: &mut Vec, command: CommandIndex, ) -> &mut PipelineBarrier { let index = barriers .iter() .position(|barrier| barrier.before_command == command) .unwrap_or_else(|| { barriers.push(PipelineBarrier { before_command: command, memory_barriers: Vec::new(), resources: tinyvec::TinyVec::new(), }); barriers.len() - 1 }); &mut barriers[index] } fn pipeline_barrier_at_command_indexed<'a>( barriers: &'a mut Vec, index_by_command: &mut [Option], command: CommandIndex, ) -> &'a mut PipelineBarrier { let index = *index_by_command[command.index()].get_or_insert_with(|| { barriers.push(PipelineBarrier { before_command: command, memory_barriers: Vec::new(), resources: tinyvec::TinyVec::new(), }); barriers.len() - 1 }); &mut barriers[index] } impl Default for RenderGraph { fn default() -> Self { Self::new() } } #[derive(Clone)] enum StagedImportResource { Buffer(Arc), Texture(Arc), } impl RenderGraph { // Import staging is transactional for ordinary errors. Panics are intentionally not caught; // callers must not rely on graph rollback after unwinding. fn import_transaction( &mut self, operation: impl FnOnce(&mut Self) -> Result, ) -> Result { let resources_len = self.resources.len(); let next_id = self.next_id; let buffer_ptrs = self.imported_buffers_by_ptr.clone(); let buffer_raw = self.imported_buffers_by_raw.clone(); let texture_ptrs = self.imported_textures_by_ptr.clone(); let texture_raw = self.imported_textures_by_raw.clone(); match operation(self) { Ok(value) => Ok(value), Err(error) => { self.resources.truncate(resources_len); self.next_id = next_id; self.imported_buffers_by_ptr = buffer_ptrs; self.imported_buffers_by_raw = buffer_raw; self.imported_textures_by_ptr = texture_ptrs; self.imported_textures_by_raw = texture_raw; Err(error) } } } fn find_imported( &self, resource: &StagedImportResource, ) -> Option<(ResourceId, ResourceState)> { let id = match resource { StagedImportResource::Buffer(resource) => self .imported_buffers_by_ptr .get(&(Arc::as_ptr(resource) as usize)) .copied() .or_else(|| self.imported_buffers_by_raw.get(&resource.raw()).copied()), StagedImportResource::Texture(resource) => self .imported_textures_by_ptr .get(&(Arc::as_ptr(resource) as usize)) .copied() .or_else(|| self.imported_textures_by_raw.get(&resource.raw()).copied()), }?; Some((id, self.resources[id.index()].initial_state())) } } fn map_descriptor_set_error( error: renderer_resources::descriptors::DescriptorSetError, ) -> GraphError { match error { renderer_resources::descriptors::DescriptorSetError::StaleSet { .. } => { GraphError::StaleDescriptorSet } error => GraphError::DescriptorImport(error.to_string()), } } fn same_state(a: ResourceState, b: ResourceState) -> bool { a.stage == b.stage && a.access == b.access && a.layout == b.layout } fn same_imported_identity(a: &StagedImportResource, b: &StagedImportResource) -> bool { match (a, b) { (StagedImportResource::Buffer(a), StagedImportResource::Buffer(b)) => { Arc::ptr_eq(a, b) || a.raw() == b.raw() } (StagedImportResource::Texture(a), StagedImportResource::Texture(b)) => { Arc::ptr_eq(a, b) || a.raw() == b.raw() } _ => false, } } fn effective_buffer_range( buffer_len: u64, base_offset: u64, range: u64, dynamic_offset: u64, ) -> Result { if base_offset > buffer_len { return Err("descriptor base offset exceeds buffer length"); } let base_range = if range == vk::WHOLE_SIZE { buffer_len - base_offset } else { range }; if base_offset .checked_add(base_range) .is_none_or(|end| end > buffer_len) { return Err("descriptor base range exceeds buffer length"); } let offset = base_offset .checked_add(dynamic_offset) .ok_or("effective descriptor offset overflow")?; if offset .checked_add(base_range) .is_none_or(|end| end > buffer_len) { return Err("effective descriptor range exceeds buffer length"); } Ok(BufferRange { offset, size: base_range, }) } fn dynamic_offset_slots( layout: &[renderer_resources::pipeline::DescriptorSetLayoutBindingDesc], ) -> Vec<(u32, u32, vk::DescriptorType)> { let mut dynamic_bindings = layout.iter().collect::>(); dynamic_bindings.sort_by_key(|binding| binding.binding); dynamic_bindings .into_iter() .filter(|binding| { matches!( binding.kind, vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC | vk::DescriptorType::STORAGE_BUFFER_DYNAMIC ) }) .flat_map(|binding| { (0..binding.count).map(move |element| (binding.binding, element, binding.kind)) }) .collect() } fn is_dynamic_offset_aligned(offset: u32, alignment: u64) -> bool { alignment != 0 && u64::from(offset).is_multiple_of(alignment) } fn texture_layouts_conflict( a: vk::ImageSubresourceRange, layout_a: vk::ImageLayout, b: vk::ImageSubresourceRange, layout_b: vk::ImageLayout, ) -> bool { layout_a != layout_b && a.intersects(&b) } fn descriptor_access( kind: vk::DescriptorType, ) -> Option<(vk::AccessFlags2, Option)> { match kind { vk::DescriptorType::UNIFORM_BUFFER | vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC => { Some((vk::AccessFlags2::UNIFORM_READ, None)) } vk::DescriptorType::STORAGE_BUFFER | vk::DescriptorType::STORAGE_BUFFER_DYNAMIC => Some(( vk::AccessFlags2::SHADER_STORAGE_READ | vk::AccessFlags2::SHADER_STORAGE_WRITE, None, )), vk::DescriptorType::SAMPLED_IMAGE | vk::DescriptorType::COMBINED_IMAGE_SAMPLER => { Some((vk::AccessFlags2::SHADER_SAMPLED_READ, None)) } vk::DescriptorType::STORAGE_IMAGE => Some(( vk::AccessFlags2::SHADER_STORAGE_READ | vk::AccessFlags2::SHADER_STORAGE_WRITE, Some(vk::ImageLayout::GENERAL), )), vk::DescriptorType::UNIFORM_TEXEL_BUFFER => Some((vk::AccessFlags2::SHADER_READ, None)), vk::DescriptorType::STORAGE_TEXEL_BUFFER => Some(( vk::AccessFlags2::SHADER_STORAGE_READ | vk::AccessFlags2::SHADER_STORAGE_WRITE, None, )), _ => None, } } fn state_for_identity( states: &[(DescriptorResourceIdentity, ResourceState)], resource: &StagedImportResource, ) -> Option { let matches = states.iter().filter_map(|(identity, state)| { let same = match (resource, identity) { (StagedImportResource::Buffer(resource), DescriptorResourceIdentity::Buffer(other)) => { Arc::ptr_eq(resource, other) || resource.raw() == other.raw() } ( StagedImportResource::Texture(resource), DescriptorResourceIdentity::Texture(other), ) => Arc::ptr_eq(resource, other) || resource.raw() == other.raw(), _ => false, }; same.then_some(*state) }); let mut matches = matches; let first = matches.next()?; matches .all(|state| same_state(first, state)) .then_some(first) } fn shader_stages_to_pipeline(stages: vk::ShaderStageFlags) -> Option { if stages == vk::ShaderStageFlags::ALL { return Some(vk::PipelineStageFlags2::ALL_COMMANDS); } let mut pipeline = vk::PipelineStageFlags2::NONE; let mappings = [ ( vk::ShaderStageFlags::VERTEX, vk::PipelineStageFlags2::VERTEX_SHADER, ), ( vk::ShaderStageFlags::TESSELLATION_CONTROL, vk::PipelineStageFlags2::TESSELLATION_CONTROL_SHADER, ), ( vk::ShaderStageFlags::TESSELLATION_EVALUATION, vk::PipelineStageFlags2::TESSELLATION_EVALUATION_SHADER, ), ( vk::ShaderStageFlags::GEOMETRY, vk::PipelineStageFlags2::GEOMETRY_SHADER, ), ( vk::ShaderStageFlags::FRAGMENT, vk::PipelineStageFlags2::FRAGMENT_SHADER, ), ( vk::ShaderStageFlags::COMPUTE, vk::PipelineStageFlags2::COMPUTE_SHADER, ), ( vk::ShaderStageFlags::TASK_EXT, vk::PipelineStageFlags2::TASK_SHADER_EXT, ), ( vk::ShaderStageFlags::MESH_EXT, vk::PipelineStageFlags2::MESH_SHADER_EXT, ), ]; let mut recognized = vk::ShaderStageFlags::empty(); if stages.contains(vk::ShaderStageFlags::ALL_GRAPHICS) { recognized |= vk::ShaderStageFlags::ALL_GRAPHICS; pipeline |= vk::PipelineStageFlags2::ALL_GRAPHICS; } for (shader, stage) in mappings { if stages.intersects(shader) { recognized |= shader; pipeline |= stage; } } (recognized == stages && !pipeline.is_empty()).then_some(pipeline) } /// A compiled graph retains the resource-hazard DAG's transitive closure for /// lifetime/alias decisions. After resolution, alias placements are based on /// memory requirements and split into linear and optimal pools, with barriers /// at each new occupant's first use. Those guarantees assume the recorded /// command buffer is submitted to one queue; multi-queue use requires semaphore /// edges for alias transitions. pub struct CompiledGraph { resources: Vec, resolved: HashMap, views: HashMap<(ResourceId, ImageViewDesc), Arc>, created_view_bindings: Vec<(ResourceId, ImageViewDesc, Arc)>, has_created_views: bool, placements_resolved: bool, commands: CommandList, batches: Vec, execution_order: Vec, barriers: Vec, barrier_indices_by_command: BarriersByCommand, ordinary_barrier_count: usize, lifetimes: HashMap, interference: ResourceInterference, dependency_edges: Vec>, node_of_command: Vec, node_representatives: Vec, happens_before: HappensBefore, uses_by_resource: HashMap>, frontiers: ResourceFrontiers, render_pass_spans: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum PoolClass { Linear, Optimal, } struct PendingPlacement { id: ResourceId, resource: ResolvedResource, requirements: renderer_resources::memory::MemoryRequirements, class: PoolClass, property_flags: vk::MemoryPropertyFlags, } struct PlacementPool { id: u32, class: PoolClass, memory_type_index: u32, property_flags: vk::MemoryPropertyFlags, size: u64, members: Vec<(ResourceId, u64)>, occupied: Vec<(ResourceId, u64)>, } enum ResolvedResource { Buffer(Arc), Image(Arc), } impl CompiledGraph { /// Topological batches of commands. Each returned slice references a /// contiguous range of the full execution order. pub fn batches(&self) -> impl Iterator { self.batches .iter() .map(|batch| &self.execution_order[batch.start..batch.start + batch.count]) } /// Full recording order, with each pass expanded as begin, interiors in /// original record order, then end. pub fn execution_order(&self) -> &[CommandIndex] { &self.execution_order } /// Render-pass spans in original command-recording order. pub fn render_pass_spans(&self) -> &[RenderPassSpan] { &self.render_pass_spans } pub fn barriers(&self) -> &[PipelineBarrier] { &self.barriers } /// Returns a cached view created for a resolved graph texture. pub fn created_view( &self, texture: ResourceId, desc: &ImageViewDesc, ) -> Option<&Arc> { self.views.get(&(texture, desc.clone())) } pub fn resource_lifetimes(&self) -> &HashMap { &self.lifetimes } pub fn interference(&self) -> &ResourceInterference { &self.interference } /// Whether command `a` is transitively ordered before command `b` by /// resource-hazard edges. Commands in the same render-pass span map to one /// node and are deliberately unordered for aliasing/interference analysis. pub fn happens_before(&self, a: CommandIndex, b: CommandIndex) -> bool { self.happens_before.contains(a.index(), b.index()) } /// The compile-time transitive closure of resource-hazard ordering. pub fn happens_before_closure(&self) -> &HappensBefore { &self.happens_before } /// Node-space adjacency over DAG-node representatives. Render-pass edges /// refer to the node represented by its begin command index. pub fn dependency_edges(&self) -> &[Vec] { &self.dependency_edges } /// Per-resource command uses, with stage/access masks ORed across accesses /// to that resource within each command. pub fn resource_uses(&self, resource: ResourceId) -> Option<&[ResourceUse]> { self.uses_by_resource.get(&resource).map(Vec::as_slice) } /// Replace placement-specific alias barriers while retaining compile-time ordinary barriers. fn rebuild_barriers(&mut self) -> Result<(), GraphError> { let compiler = GraphCompiler::new_with_nodes( &self.resources, self.commands.side_effects.command_accesses(), self.commands.len(), &self.commands.side_effects, self.node_of_command.clone(), self.node_representatives.clone(), &self.render_pass_spans, ); self.barriers.truncate(self.ordinary_barrier_count); compiler.build_alias_barriers( &mut self.barriers, &self.lifetimes, &self.frontiers, &self.happens_before, ); hoist_span_barriers( &mut self.barriers, &self.render_pass_spans, self.commands.len(), ); validate_intra_pass_barriers(&self.barriers, &self.render_pass_spans)?; self.barrier_indices_by_command = grouped_barrier_indices(&self.barriers, self.commands.len()); Ok(()) } /// Tests whether two use sets can safely reuse a placement in the supplied /// direction. Stage/access masks are retained for alias barriers; ordering /// is proven from command ids through the dependency closure. pub fn aliasing_permitted(&self, uses_a: &[ResourceUse], uses_b: &[ResourceUse]) -> bool { resources_alias_in_order(uses_a, uses_b, &self.happens_before) } /// Tests whether two resources can reuse a placement in either safe order. pub fn resources_can_alias(&self, a: ResourceId, b: ResourceId) -> bool { let (Some(uses_a), Some(uses_b)) = (self.resource_uses(a), self.resource_uses(b)) else { return false; }; self.aliasing_permitted(uses_a, uses_b) || self.aliasing_permitted(uses_b, uses_a) } /// Create and bind used transient resources and finalize alias barriers. /// /// Aliasing safety assumes all graph commands are submitted from one command /// buffer to one queue. Cross-queue aliasing is unsupported; when /// per-command queue assignment exists, alias transitions between queues /// must become semaphore edges as well as the corresponding memory barriers. pub fn resolve(&mut self, device: &Device) -> Result<(), GraphError> { if self.plan_placements(device)? { self.rebuild_barriers()?; } self.resolve_created_views()?; Ok(()) } fn resolve_created_views(&mut self) -> Result<(), GraphError> { if !self.has_created_views { return Ok(()); } let mut staged = HashMap::new(); let mut staged_bindings = Vec::new(); for command in &self.commands.commands { let Some(desc) = command.render_pass_desc() else { continue; }; let attachments = desc .color .iter() .map(|attachment| (attachment.texture.id(), &attachment.view)) .chain( desc.depth .into_iter() .map(|attachment| (attachment.texture.id(), &attachment.view)), ); for (texture, view) in attachments { let AttachmentView::Created(view_desc) = view else { continue; }; if !self.lifetimes.contains_key(&texture) || self.views.contains_key(&(texture, view_desc.clone())) || staged.contains_key(&(texture, view_desc.clone())) { continue; } let image = match self.resources.get(texture.index()) { Some(ResourceDesc::ImportedTexture { resource, .. }) => resource, _ => match self.resolved.get(&texture) { Some(ResolvedResource::Image(resource)) => resource, _ => { return Err(GraphError::ViewCreationFailed(format!( "created attachment texture {texture:?} is unresolved" ))); } }, }; let view = image .create_view(view_desc.clone()) .map(Arc::new) .map_err(|error| GraphError::ViewCreationFailed(error.to_string()))?; if view.image().as_ref() != image.as_ref() { return Err(GraphError::ViewCreationFailed(format!( "created view for texture {texture:?} refers to a different image" ))); } staged.insert((texture, view_desc.clone()), view.clone()); let mut binding_desc = view_desc.clone(); binding_desc.name = None; staged_bindings.push((texture, binding_desc, view)); } } self.views.extend(staged); self.created_view_bindings.extend(staged_bindings); Ok(()) } /// Stage resource creation and placements locally; publish only after all binds succeed. fn plan_placements(&mut self, device: &Device) -> Result { if self.placements_resolved { let has_unresolved_created_resource = self.resources.iter().enumerate().any(|(index, desc)| { matches!( desc, ResourceDesc::CreatedBuffer(_) | ResourceDesc::CreatedTexture(_) ) && self .lifetimes .contains_key(&ResourceId::from_u32(index as u32)) && !self .resolved .contains_key(&ResourceId::from_u32(index as u32)) }); if !has_unresolved_created_resource { return Ok(false); } } let command_accesses = self.commands.side_effects.command_accesses(); let mut accesses_by_resource = vec![Vec::new(); self.resources.len()]; for (command, accesses) in command_accesses.iter().enumerate() { for access in accesses { if !self .commands .side_effects .is_lifetime_only(command.into(), access.resource) { accesses_by_resource[access.resource.index()].push(*access); } } } let mut staged_resources = Vec::new(); let mut staged_lifetimes = self.lifetimes.clone(); let mut pooled = Vec::new(); // Create only resources with actual graph uses; unused transients have no lifetime. for (index, desc) in self.resources.iter().enumerate() { let id = ResourceId::from_u32(index as u32); if !should_plan_created_resource(id, &staged_lifetimes, &self.resolved) { continue; } let accesses = accesses_by_resource[id.index()].as_slice(); match desc { ResourceDesc::CreatedBuffer(desc) => { let desc = BufferDesc { flags: vk::BufferCreateFlags::empty(), name: desc.name.clone(), size: desc.size, usage: derive_buffer_usage(accesses, desc.extra_usage), queue_families: actual_queue_families( device, derive_queue_capabilities(accesses), ), mem_location: desc.mem_location, // Explicit allocation strategy is selected after the requirements query. alloc_scheme: AllocationStrategy::AllocatorManaged, }; let resource = BufferResource::new_unallocated(device.clone(), desc.clone()) .map_err(|_| GraphError::PlacementFailed(id))?; let requirements = resource .memory_requirements() .map_err(|_| GraphError::PlacementFailed(id))?; if !is_poolable( desc.mem_location, allocation_strategy_for_size(desc.size, requirements), requirements, ) { drop(resource); let resource = BufferResource::new( device.clone(), BufferDesc { alloc_scheme: allocation_strategy_for_size(desc.size, requirements), ..desc }, ) .map_err(|_| GraphError::PlacementFailed(id))?; set_placement(&mut staged_lifetimes, id, requirements, 0, None, false); staged_resources.push((id, ResolvedResource::Buffer(Arc::new(resource)))); continue; } pooled.push(PendingPlacement { id, resource: ResolvedResource::Buffer(Arc::new(resource)), requirements, class: PoolClass::Linear, property_flags: memory_property_flags(desc.mem_location), }); continue; } ResourceDesc::CreatedTexture(desc) => { let image_desc = ImageDesc { // Candidates start alias-capable; non-poolable candidates are recreated // without the alias flag below. flags: vk::ImageCreateFlags::ALIAS, name: desc.name.clone(), format: desc.format, kind: desc.kind, mip_levels: desc.mip_levels, array_layers: desc.array_layers, samples: desc.samples, extent: desc.extent, tiling: desc.tiling, usage: derive_image_usage(accesses, desc.extra_usage), queue_families: actual_queue_families( device, derive_queue_capabilities(accesses), ), // Query requirements from a Vulkan-valid undefined candidate; pooled // eligibility requires the graph's declared initial layout be undefined. layout: vk::ImageLayout::UNDEFINED, mem_location: desc.mem_location, alloc_scheme: AllocationStrategy::AllocatorManaged, }; let candidate = ImageResource::new_unallocated(device.clone(), image_desc) .map_err(|_| GraphError::PlacementFailed(id))?; let requirements = candidate .memory_requirements() .map_err(|_| GraphError::PlacementFailed(id))?; if !is_poolable( desc.mem_location, allocation_strategy_for(requirements), requirements, ) { // TODO: Consider reusing the candidate drop(candidate); let resource = ImageResource::new( device.clone(), image_desc_for_graph_texture( desc, accesses, actual_queue_families(device, derive_queue_capabilities(accesses)), allocation_strategy_for_size( desc.extent.width.into(), requirements, ), vk::ImageCreateFlags::empty(), ), ) .map_err(|_| GraphError::PlacementFailed(id))?; let requirements = resource .memory_requirements() .map_err(|_| GraphError::PlacementFailed(id))?; set_placement(&mut staged_lifetimes, id, requirements, 0, None, false); staged_resources.push((id, ResolvedResource::Image(Arc::new(resource)))); continue; } validate_pooled_image_layout(id, desc, requirements)?; pooled.push(PendingPlacement { id, resource: ResolvedResource::Image(Arc::new(candidate)), requirements, class: pool_class_for_image(desc.tiling), property_flags: memory_property_flags(desc.mem_location), }); continue; } ResourceDesc::ImportedBuffer { .. } | ResourceDesc::ImportedTexture { .. } => { continue; } } } // Add resources to class/type-specific pools with intersection semantics. let mut pools: Vec = Vec::new(); pooled.sort_by_key(|member| member.id); for member in pooled { let compatible_pool = pools.iter().position(|pool| { pool_accepts_member( pool, member.class, member.property_flags, member.requirements, ) }); let pool_index = if let Some(pool_index) = compatible_pool { pool_index } else { let memory_type_index = device .find_memory_type_index(&member.requirements, member.property_flags) .map_err(|_| GraphError::PlacementFailed(member.id))?; let id = u32::try_from(pools.len()) .map_err(|_| GraphError::PlacementFailed(member.id))?; pools.push(PlacementPool { id, class: member.class, memory_type_index, property_flags: member.property_flags, size: 0, members: Vec::new(), occupied: Vec::new(), }); pools.len() - 1 }; let offset = first_fit_placement( member.id, member.requirements.size, member.requirements.alignment, &pools[pool_index].occupied, &staged_lifetimes, &self.interference, &self.frontiers, &self.happens_before, ) .ok_or(GraphError::PlacementFailed(member.id))?; let end = offset .checked_add(member.requirements.size) .ok_or(GraphError::PlacementFailed(member.id))?; let pool = &mut pools[pool_index]; pool.size = pool.size.max(end); let mut intersects_unsafe_lifetime = false; let overlapping = pool .members .iter() .filter_map(|(other_id, other_offset)| { let overlaps = byte_ranges_overlap( offset, member.requirements.size, *other_offset, staged_lifetimes[other_id].placement.size, ); if !overlaps { return None; } let is_ordered = !self.interference.interferes(member.id, *other_id) && self.frontiers.alias_in_order_either( member.id, *other_id, &self.happens_before, ); if !is_ordered { intersects_unsafe_lifetime = true; } Some(*other_id) }) .collect::>(); if intersects_unsafe_lifetime { return Err(GraphError::PlacementFailed(member.id)); } pool.members.push((member.id, offset)); let occupied_index = pool .occupied .binary_search_by_key(&(offset, member.id), |(id, offset)| (*offset, *id)) .unwrap_or_else(|index| index); pool.occupied.insert(occupied_index, (member.id, offset)); for other_id in &overlapping { if let Some(lifetime) = staged_lifetimes.get_mut(other_id) { lifetime.placement.aliased = true; } } set_placement( &mut staged_lifetimes, member.id, member.requirements, offset, Some(pool.id), !overlapping.is_empty(), ); staged_resources.push((member.id, member.resource)); } // Allocate one shared block per pool, then transfer references to every bound member. let mut staged_indices = HashMap::with_capacity(staged_resources.len()); for (index, (id, _)) in staged_resources.iter().enumerate() { staged_indices.insert(*id, index); } for pool in pools { let first_id = pool.members[0].0; let alignment = pool .members .iter() .try_fold(1_u64, |alignment, (id, _)| { checked_alignment_lcm(alignment, staged_lifetimes[id].placement.alignment) }) .ok_or(GraphError::PlacementFailed(first_id))?; let requirements = renderer_resources::memory::MemoryRequirements { size: pool.size, alignment, memory_type_bits: 1 << pool.memory_type_index, requires_dedicated_allocation: false, prefers_dedicated_allocation: false, }; let allocation = device .allocate_memory(&requirements, pool.property_flags) .map_err(|_| GraphError::PlacementFailed(first_id))?; for (id, _) in &pool.members { let resource_index = *staged_indices.get(id).expect("every pool member is staged"); let resource = &mut staged_resources[resource_index]; match &mut resource.1 { ResolvedResource::Buffer(buffer) => Arc::get_mut(buffer) .expect("new pooled buffer has a unique owner") .bind_allocation(allocation.clone(), staged_lifetimes[id].placement.offset) .map_err(|error| GraphError::from((*id, error)))?, ResolvedResource::Image(image) => Arc::get_mut(image) .expect("new pooled image has a unique owner") .bind_allocation(allocation.clone(), staged_lifetimes[id].placement.offset) .map_err(|error| GraphError::from((*id, error)))?, } } } // Commit only after every creation, allocation, and bind has succeeded. let changed = !self.placements_resolved || !staged_resources.is_empty(); for (id, resource) in staged_resources { self.resolved.insert(id, resource); } self.lifetimes = staged_lifetimes; self.placements_resolved = true; Ok(changed) } /// Record the compiled order into a real Vulkan command buffer. Imported /// handles are automatically bound. Created resources must first be /// resolved with a device. pub fn record_commands( &mut self, recorder: &mut CommandRecorder<'_>, ) -> Result<(), RecordError> { // Validate every wrapper before resource binding or barriers can emit Vulkan commands. self.commands.preflight()?; for (index, desc) in self.resources.iter().enumerate() { let id = ResourceId::from_u32(index as u32); if !self.lifetimes.contains_key(&id) { continue; } match desc { ResourceDesc::ImportedBuffer { resource, .. } => { recorder.buffers.insert(id, resource.raw()); } ResourceDesc::ImportedTexture { resource, .. } => { recorder.images.insert(id, resource.raw()); } ResourceDesc::CreatedBuffer(_) => match self.resolved.get(&id) { Some(ResolvedResource::Buffer(resource)) => { recorder.buffers.insert(id, resource.raw()); } _ => return Err(RecordError::UnresolvedCreatedBuffer(id)), }, ResourceDesc::CreatedTexture(_) => match self.resolved.get(&id) { Some(ResolvedResource::Image(resource)) => { recorder.images.insert(id, resource.raw()); } _ => return Err(RecordError::UnresolvedCreatedImage(id)), }, } } for (texture, desc, view) in &self.created_view_bindings { recorder .views .insert((*texture, desc.clone()), view.clone()); } // Batches currently record into one command buffer, which must be // submitted to one queue for current aliasing safety. TODO: separate // per-batch command buffers may be submitted sequentially on that queue; // parallel/cross-queue execution requires command-to-queue assignment // and semaphore edges for every cross-queue alias transition. let mut in_render_pass = false; for step in recording_steps_cached( &self.execution_order, &self.barriers, &self.barrier_indices_by_command, ) { match step { // TODO: Merge `PipelineBarrier.resources` across commands or a batch to reduce vkCmdPipelineBarrier2 calls. RecordingStep::Barrier(barrier) => recorder.pipeline_barrier(barrier), RecordingStep::Command(command_index) => { if let Err(error) = self.commands.commands[command_index.index()].apply(recorder) { if std::mem::replace(&mut in_render_pass, false) { unsafe { recorder.device.cmd_end_rendering(recorder.command_buffer); } } return Err(error); } match self.commands.commands[command_index.index()].kind() { CommandKind::Begin => in_render_pass = true, CommandKind::End => in_render_pass = false, _ => {} } } } } Ok(()) } } enum RecordingStep<'a> { Barrier(&'a PipelineBarrier), Command(CommandIndex), } fn recording_steps<'a>( execution_order: &'a [CommandIndex], barriers: &'a [PipelineBarrier], ) -> impl Iterator> + 'a { let command_count = execution_order .iter() .map(|command| command.index()) .chain( barriers .iter() .map(|barrier| barrier.before_command.index()), ) .max() .unwrap_or_default() + 1; let mut barriers_by_command: Vec> = (0..command_count).map(|_| Vec::new()).collect(); for barrier in barriers { barriers_by_command[barrier.before_command.index()].push(barrier); } execution_order.iter().flat_map(move |&command_index| { std::mem::take(&mut barriers_by_command[command_index.index()]) .into_iter() .map(RecordingStep::Barrier) .chain(std::iter::once(RecordingStep::Command(command_index))) }) } fn grouped_barrier_indices( barriers: &[PipelineBarrier], command_count: usize, ) -> BarriersByCommand { let mut barriers_by_command = BarriersByCommand::new(command_count); for (index, barrier) in barriers.iter().enumerate() { barriers_by_command.entries[barrier.before_command.index()].push(index); } barriers_by_command } fn recording_steps_cached<'a>( execution_order: &'a [CommandIndex], barriers: &'a [PipelineBarrier], barrier_indices_by_command: &'a BarriersByCommand, ) -> impl Iterator> + 'a { execution_order.iter().flat_map(move |&command_index| { barrier_indices_by_command .get(command_index) .iter() .map(move |&index| RecordingStep::Barrier(&barriers[index])) .chain(std::iter::once(RecordingStep::Command(command_index))) }) } /// Very large resources bypass transient pools and use a dedicated allocation. const DEDICATED_ALLOCATION_THRESHOLD: u64 = 64 * 1024 * 1024; fn accesses_for_resource( id: ResourceId, command_accesses: &[Vec], side_effects: &SideEffectMap, ) -> Vec { command_accesses .iter() .enumerate() .flat_map(|(command, accesses)| { accesses.iter().filter_map(move |access| { (access.resource == id && !side_effects.is_lifetime_only(command.into(), access.resource)) .then_some(*access) }) }) .collect() } /// Derive buffer usages from declared access masks. Unknown access bits add no /// usage; callers can use `extra_usage` for uses outside the graph's knowledge. pub fn derive_buffer_usage( accesses: &[crate::recorder::ResourceAccess], extra_usage: vk::BufferUsageFlags, ) -> vk::BufferUsageFlags { accesses.iter().fold(extra_usage, |usage, access| { let mask = access.scope.access; let mut usage = usage; if mask.intersects(vk::AccessFlags2::TRANSFER_READ) { usage |= vk::BufferUsageFlags::TRANSFER_SRC; } if mask.intersects(vk::AccessFlags2::TRANSFER_WRITE) { usage |= vk::BufferUsageFlags::TRANSFER_DST; } if mask.intersects( vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_STORAGE_READ | vk::AccessFlags2::SHADER_STORAGE_WRITE, ) { // Shader reads are conservatively treated as storage-buffer reads; // add UNIFORM_BUFFER explicitly when the shader uses uniform data. usage |= vk::BufferUsageFlags::STORAGE_BUFFER; } if mask.intersects(vk::AccessFlags2::UNIFORM_READ) { usage |= vk::BufferUsageFlags::UNIFORM_BUFFER; } if mask.intersects(vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::SHADER_STORAGE_WRITE) { usage |= vk::BufferUsageFlags::STORAGE_BUFFER; } if mask.intersects(vk::AccessFlags2::VERTEX_ATTRIBUTE_READ) { usage |= vk::BufferUsageFlags::VERTEX_BUFFER; } if mask.intersects(vk::AccessFlags2::INDEX_READ) { usage |= vk::BufferUsageFlags::INDEX_BUFFER; } if mask.intersects(vk::AccessFlags2::INDIRECT_COMMAND_READ) { usage |= vk::BufferUsageFlags::INDIRECT_BUFFER; } usage }) } /// Derive image usages from declared access masks. Shader reads require a /// sampled or storage image descriptor, so both flags are enabled; unknown /// access bits add no usage and may be supplied through `extra_usage`. pub fn derive_image_usage( accesses: &[crate::recorder::ResourceAccess], extra_usage: vk::ImageUsageFlags, ) -> vk::ImageUsageFlags { accesses.iter().fold(extra_usage, |usage, access| { let mask = access.scope.access; let mut usage = usage; if mask.intersects(vk::AccessFlags2::TRANSFER_READ) { usage |= vk::ImageUsageFlags::TRANSFER_SRC; } if mask.intersects(vk::AccessFlags2::TRANSFER_WRITE) { usage |= vk::ImageUsageFlags::TRANSFER_DST; } if mask.intersects( vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_SAMPLED_READ | vk::AccessFlags2::SHADER_STORAGE_READ, ) { usage |= vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::STORAGE; } if mask.intersects(vk::AccessFlags2::SHADER_WRITE | vk::AccessFlags2::SHADER_STORAGE_WRITE) { usage |= vk::ImageUsageFlags::STORAGE; } if mask.intersects(vk::AccessFlags2::INPUT_ATTACHMENT_READ) { usage |= vk::ImageUsageFlags::INPUT_ATTACHMENT; } if mask.intersects( vk::AccessFlags2::COLOR_ATTACHMENT_READ | vk::AccessFlags2::COLOR_ATTACHMENT_WRITE, ) { usage |= vk::ImageUsageFlags::COLOR_ATTACHMENT; } if mask.intersects( vk::AccessFlags2::DEPTH_STENCIL_ATTACHMENT_READ | vk::AccessFlags2::DEPTH_STENCIL_ATTACHMENT_WRITE, ) { usage |= vk::ImageUsageFlags::DEPTH_STENCIL_ATTACHMENT; } usage }) } /// Map pipeline-stage usage to desired queue capabilities. Graphics stages /// imply GRAPHICS, compute stages imply ASYNC_COMPUTE, and copy/clear/transfer /// stages imply TRANSFER. NONE and unrecognized stages contribute nothing. pub fn derive_queue_capabilities(accesses: &[crate::recorder::ResourceAccess]) -> QueueFlags { accesses .iter() .fold(QueueFlags::empty(), |mut flags, access| { let stages = access.scope.stage; if stages.intersects( vk::PipelineStageFlags2::DRAW_INDIRECT | vk::PipelineStageFlags2::VERTEX_INPUT | vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::TESSELLATION_CONTROL_SHADER | vk::PipelineStageFlags2::TESSELLATION_EVALUATION_SHADER | vk::PipelineStageFlags2::GEOMETRY_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER | vk::PipelineStageFlags2::EARLY_FRAGMENT_TESTS | vk::PipelineStageFlags2::LATE_FRAGMENT_TESTS | vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT | vk::PipelineStageFlags2::ALL_GRAPHICS | vk::PipelineStageFlags2::PRE_RASTERIZATION_SHADERS, ) { flags |= QueueFlags::GRAPHICS; } if stages.intersects(vk::PipelineStageFlags2::COMPUTE_SHADER) { flags |= QueueFlags::ASYNC_COMPUTE; } if stages.intersects( vk::PipelineStageFlags2::TRANSFER | vk::PipelineStageFlags2::COPY | vk::PipelineStageFlags2::CLEAR, ) { flags |= QueueFlags::TRANSFER; } flags }) } /// Convert inferred capabilities into the device's queue-role selectors, /// retaining only roles whose selected family actually supports the inferred /// capability. If no role matches (or nothing was inferred), use graphics. fn actual_queue_families(device: &Device, capabilities: QueueFlags) -> QueueFlags { use renderer_device::queue::QueueFlags as FamilyFlags; let mut selected = QueueFlags::empty(); if capabilities.contains(QueueFlags::GRAPHICS) && device .queues .graphics() .family .flags .contains(FamilyFlags::GRAPHICS) { selected |= QueueFlags::GRAPHICS; } if capabilities.contains(QueueFlags::ASYNC_COMPUTE) && device .queues .compute() .family .flags .contains(FamilyFlags::COMPUTE) { selected |= QueueFlags::ASYNC_COMPUTE; } if capabilities.contains(QueueFlags::TRANSFER) && device .queues .transfer() .family .flags .contains(FamilyFlags::TRANSFER) { selected |= QueueFlags::TRANSFER; } if selected.is_empty() { QueueFlags::GRAPHICS } else { selected } } #[cfg(test)] mod tests { use super::*; use crate::{ commands::{ AttachmentView, BeginRendering, ClearTexture, ColorAttachment, CopyBuffer, Draw, DrawArgs, EndRendering, }, resources::{ BufferRange, BufferSlice, DescriptorResourceIdentity, GraphBufferDesc, GraphTextureDesc, IndexBinding, IndexFormat, IndirectBinding, ResourceDesc, ResourceState, VertexBinding, }, }; use ash::vk::Handle; use renderer_resources::{ buffers::Buffer as BufferResource, descriptors::{DescriptorResourceBinding, TrackedDescriptorWrite}, images::{Image as ImageResource, ImageDesc, ImageView}, pipeline::{DescriptorSet, DescriptorSetLayoutBindingDesc, Sampler}, }; fn ordering_from_accesses( resources: &[ResourceDesc], accesses: &[Vec], ) -> CommandOrderings { let mut commands = CommandList::new(); for command_accesses in accesses { let command = AllowedAccess( command_accesses .iter() .map(|access| (access.resource, access.scope)) .collect(), ); commands.push(command); } build_command_orderings(resources, &commands.commands, &mut commands.side_effects).unwrap() } fn topo_batches_for_test( compiler: &GraphCompiler<'_>, ) -> Result>, GraphError> { let orderings = ordering_from_accesses(compiler.resources, compiler.accesses); Ok(orderings .batches .iter() .map(|batch| orderings.execution_order[batch.start..batch.start + batch.count].to_vec()) .collect()) } static RECORDING_MARKER_APPLIED: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); struct RecordingMarker(Buffer); impl crate::commands::Command for RecordingMarker { fn side_effects( &self, command_index: CommandIndex, map: &mut crate::recorder::SideEffectMap, ) { map.insert( command_index, self.0.id(), SubresourceScope { stage: vk::PipelineStageFlags2::TRANSFER, access: vk::AccessFlags2::TRANSFER_READ, subresource: SubresourceScopeInner::Buffer(BufferRange { offset: 0, size: 32, }), }, ); } } impl crate::recorder::RecordedCommand for RecordingMarker { fn apply(&mut self, _recorder: &mut CommandRecorder<'_>) -> Result<(), RecordError> { RECORDING_MARKER_APPLIED.fetch_add(1, std::sync::atomic::Ordering::SeqCst); Ok(()) } } struct AllowedAccess(Vec<(ResourceId, SubresourceScope)>); impl crate::commands::Command for AllowedAccess { fn side_effects(&self, command_index: CommandIndex, map: &mut SideEffectMap) { for &(resource, scope) in &self.0 { map.insert(command_index, resource, scope); } } } impl crate::recorder::RecordedCommand for AllowedAccess { fn allowed_in_render_pass(&self) -> bool { true } fn apply(&mut self, _recorder: &mut CommandRecorder<'_>) -> Result<(), RecordError> { Ok(()) } } fn test_buffer(handle: u64, size: u64) -> Arc { Arc::new(BufferResource::for_testing( vk::Buffer::from_raw(handle), size, )) } fn test_image(handle: u64) -> Arc { Arc::new(ImageResource::for_testing( vk::Image::from_raw(handle), ImageDesc { format: vk::Format::R8G8B8A8_UNORM, extent: vk::Extent3D { width: 16, height: 16, depth: 1, }, usage: vk::ImageUsageFlags::SAMPLED, ..Default::default() }, )) } fn empty_render_pass() -> BeginRendering { BeginRendering { color: Vec::new(), depth: None, } } fn created_color_attachment(texture: Texture, layout: vk::ImageLayout) -> ColorAttachment { ColorAttachment { texture, view: AttachmentView::Created(renderer_resources::images::ImageViewDesc::color_2d()), layout, load_op: vk::AttachmentLoadOp::DONT_CARE, store_op: vk::AttachmentStoreOp::STORE, clear_value: vk::ClearValue::default(), range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), } } fn test_view(image: Arc) -> Arc { Arc::new(ImageView::for_testing( vk::ImageView::from_raw(1), image, texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), )) } fn layout_binding(binding: u32, kind: vk::DescriptorType) -> DescriptorSetLayoutBindingDesc { DescriptorSetLayoutBindingDesc { binding, count: 1, kind, stage: vk::ShaderStageFlags::FRAGMENT, flags: None, } } fn test_set( handle: u64, layout: Vec, bindings: Vec, ) -> Arc { Arc::new(DescriptorSet::for_testing( vk::DescriptorSet::from_raw(handle), layout, 0, 0, bindings, )) } #[test] fn pool_classes_separate_linear_buffers_from_optimal_images() { let pool = PlacementPool { id: 0, class: PoolClass::Linear, memory_type_index: 0, property_flags: vk::MemoryPropertyFlags::DEVICE_LOCAL, size: 64, members: Vec::new(), occupied: Vec::new(), }; let requirements = renderer_resources::memory::MemoryRequirements { size: 16, alignment: 8, memory_type_bits: 1, requires_dedicated_allocation: false, prefers_dedicated_allocation: false, }; assert_eq!( pool_class_for_image(vk::ImageTiling::LINEAR), PoolClass::Linear ); assert_eq!( pool_class_for_image(vk::ImageTiling::OPTIMAL), PoolClass::Optimal ); assert_ne!(PoolClass::Linear, PoolClass::Optimal); assert!(!pool_accepts_member( &pool, pool_class_for_image(vk::ImageTiling::OPTIMAL), vk::MemoryPropertyFlags::DEVICE_LOCAL, requirements, )); } #[test] fn pool_groups_require_a_common_memory_type_for_every_member() { let requirements = |bits| renderer_resources::memory::MemoryRequirements { size: 16, alignment: 8, memory_type_bits: bits, requires_dedicated_allocation: false, prefers_dedicated_allocation: false, }; let pool = PlacementPool { id: 0, class: PoolClass::Linear, memory_type_index: 1, property_flags: vk::MemoryPropertyFlags::DEVICE_LOCAL, size: 16, members: vec![(ResourceId::from_u32(0), 0)], occupied: vec![(ResourceId::from_u32(0), 0)], }; assert!(pool_accepts_member( &pool, PoolClass::Linear, vk::MemoryPropertyFlags::DEVICE_LOCAL, requirements(0b0010), )); assert!(!pool_accepts_member( &pool, PoolClass::Linear, vk::MemoryPropertyFlags::DEVICE_LOCAL, requirements(0b0001), )); } #[test] fn first_fit_checks_every_occupied_range_and_finds_earliest_gap() { let id = ResourceId::from_u32(2); let occupied = [(ResourceId::from_u32(0), 0), (ResourceId::from_u32(1), 16)]; let lifetimes = HashMap::from([ (ResourceId::from_u32(0), placed_lifetime(0, 32, false)), (ResourceId::from_u32(1), placed_lifetime(16, 32, false)), (id, placed_lifetime(0, 0, false)), ]); let uses = HashMap::from([ ( ResourceId::from_u32(0), vec![ResourceUse { command: CommandIndex(0), stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, }], ), ( ResourceId::from_u32(1), vec![ResourceUse { command: CommandIndex(1), stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, }], ), ( id, vec![ResourceUse { command: CommandIndex(2), stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, }], ), ]); let closure = HappensBefore::from_command_dag( &[vec![2], vec![2], vec![]], &[ vec![CommandIndex(0), CommandIndex(1)], vec![CommandIndex(2)], ], ); let mut interference = ResourceInterference { edges: vec![BTreeSet::new(); 3], }; for old in [ResourceId::from_u32(0), ResourceId::from_u32(1)] { interference.edges[id.index()].insert(old); interference.edges[old.index()].insert(id); } assert_eq!( first_fit_placement( id, 32, 16, &occupied, &lifetimes, &interference, &ResourceFrontiers::from_uses(&uses, &closure), &closure, ), Some(48) ); } #[test] fn first_fit_advances_past_all_unsafe_intersecting_ranges() { let candidate = ResourceId::from_u32(2); let occupied = [(ResourceId::from_u32(0), 0), (ResourceId::from_u32(1), 16)]; let lifetimes = HashMap::from([ (ResourceId::from_u32(0), placed_lifetime(0, 32, false)), (ResourceId::from_u32(1), placed_lifetime(16, 32, false)), (candidate, placed_lifetime(0, 0, false)), ]); let uses = HashMap::from([ ( ResourceId::from_u32(0), vec![ResourceUse { command: CommandIndex(0), stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, }], ), ( ResourceId::from_u32(1), vec![ResourceUse { command: CommandIndex(1), stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, }], ), ( candidate, vec![ResourceUse { command: CommandIndex(2), stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, }], ), ]); let closure = HappensBefore::from_command_dag( &[vec![], vec![], vec![]], &[vec![CommandIndex(0), CommandIndex(1), CommandIndex(2)]], ); let mut interference = ResourceInterference { edges: vec![BTreeSet::new(); 3], }; for old in [ResourceId::from_u32(0), ResourceId::from_u32(1)] { interference.edges[candidate.index()].insert(old); interference.edges[old.index()].insert(candidate); } assert_eq!( first_fit_placement( candidate, 16, 16, &occupied, &lifetimes, &interference, &ResourceFrontiers::from_uses(&uses, &closure), &closure, ), Some(48) ); } #[test] fn dedicated_and_host_visible_resources_are_not_poolable() { let requirements = renderer_resources::memory::MemoryRequirements { size: 128, alignment: 16, memory_type_bits: 1, requires_dedicated_allocation: false, prefers_dedicated_allocation: false, }; assert!(!is_poolable( MemoryLocation::GpuOnly, AllocationStrategy::Dedicated, requirements, )); assert!(!is_poolable( MemoryLocation::CpuToGpu, AllocationStrategy::AllocatorManaged, requirements, )); assert!(!is_poolable( MemoryLocation::GpuToCpu, AllocationStrategy::AllocatorManaged, requirements, )); assert!(!is_poolable( MemoryLocation::GpuOnly, AllocationStrategy::AllocatorManaged, renderer_resources::memory::MemoryRequirements { requires_dedicated_allocation: true, ..requirements }, )); assert!(is_poolable( MemoryLocation::GpuOnly, AllocationStrategy::AllocatorManaged, requirements, )); } #[test] fn allocator_managed_strategy_remains_poolable_until_requirements_select_dedicated() { let requirements = renderer_resources::memory::MemoryRequirements { size: 128, alignment: 16, memory_type_bits: 1, requires_dedicated_allocation: false, prefers_dedicated_allocation: false, }; assert_eq!( allocation_strategy_for(requirements), AllocationStrategy::AllocatorManaged ); assert_eq!( allocation_strategy_for(renderer_resources::memory::MemoryRequirements { requires_dedicated_allocation: true, ..requirements }), AllocationStrategy::Dedicated ); assert_eq!( allocation_strategy_for(renderer_resources::memory::MemoryRequirements { size: DEDICATED_ALLOCATION_THRESHOLD, ..requirements }), AllocationStrategy::Dedicated ); } #[test] fn unused_transients_have_no_lifetime_and_are_not_candidates() { let unused = ResourceId::from_u32(0); let used = ResourceId::from_u32(1); let resources = [ ResourceDesc::CreatedBuffer(GraphBufferDesc { size: 1024, ..Default::default() }), ResourceDesc::CreatedBuffer(GraphBufferDesc::default()), ]; let accesses = vec![vec![resource_access_for(used)]]; let orderings = ordering_from_accesses(&resources, &accesses); let batches = orderings .batches .iter() .map(|batch| orderings.execution_order[batch.start..batch.start + batch.count].to_vec()) .collect::>(); let closure = HappensBefore::from_command_dag(&[vec![]], &batches); let compiler = GraphCompiler::new_with_nodes( &resources, &accesses, 1, &SideEffectMap::default(), orderings.node_of_command, orderings.node_representatives, &orderings.spans, ); let uses = compiler.resource_uses(); let frontiers = ResourceFrontiers::from_uses(&uses, &closure); let uses_by_resource = compiler.resource_uses(); let (lifetimes, _) = compiler.bind_lifetimes_for_test( &batches, &frontiers, &closure, &uses_by_resource, &[], ); assert!(!lifetimes.contains_key(&unused)); assert!(lifetimes.contains_key(&used)); assert!(!should_plan_created_resource( unused, &lifetimes, &HashMap::new(), )); assert!(should_plan_created_resource( used, &lifetimes, &HashMap::new(), )); } #[test] fn pooled_image_layout_validation_rejects_non_undefined_but_allows_dedicated() { let id = ResourceId::from_u32(7); let desc = GraphTextureDesc { layout: vk::ImageLayout::GENERAL, ..Default::default() }; let requirements = renderer_resources::memory::MemoryRequirements { size: 128, alignment: 16, memory_type_bits: 1, requires_dedicated_allocation: false, prefers_dedicated_allocation: false, }; assert_eq!( validate_pooled_image_layout(id, &desc, requirements), Err(GraphError::InvalidPooledImageLayout(id)) ); validate_pooled_image_layout( id, &desc, renderer_resources::memory::MemoryRequirements { size: DEDICATED_ALLOCATION_THRESHOLD, ..requirements }, ) .expect("non-pooled images retain the existing initial-layout behavior"); } fn buffer_entry( binding: u32, kind: vk::DescriptorType, buffer: Arc, offset: u64, range: u64, ) -> TrackedDescriptorWrite { TrackedDescriptorWrite { binding, array_element: 0, descriptor_type: kind, stages: vk::ShaderStageFlags::FRAGMENT, resource: DescriptorResourceBinding::Buffer { buffer, offset, range, dynamic: matches!( kind, vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC | vk::DescriptorType::STORAGE_BUFFER_DYNAMIC ), }, } } fn image_entry( binding: u32, kind: vk::DescriptorType, view: Arc, layout: vk::ImageLayout, ) -> TrackedDescriptorWrite { TrackedDescriptorWrite { binding, array_element: 0, descriptor_type: kind, stages: vk::ShaderStageFlags::FRAGMENT, resource: DescriptorResourceBinding::Image { range: view.subresource_range(), view, sampler: None, layout, }, } } fn import_buffer_state( buffer: &Arc, state: ResourceState, ) -> (DescriptorResourceIdentity, ResourceState) { (DescriptorResourceIdentity::Buffer(buffer.clone()), state) } fn import_texture_state( image: &Arc, state: ResourceState, ) -> (DescriptorResourceIdentity, ResourceState) { (DescriptorResourceIdentity::Texture(image.clone()), state) } #[test] fn managed_descriptor_import_transitively_retains_buffer_and_image_owners() { let buffer = test_buffer(101, 128); let image = test_image(201); let view = test_view(image.clone()); let buffer_weak = Arc::downgrade(&buffer); let image_weak = Arc::downgrade(&image); let view_weak = Arc::downgrade(&view); let set = test_set( 1, vec![ layout_binding(0, vk::DescriptorType::UNIFORM_BUFFER), layout_binding(1, vk::DescriptorType::SAMPLED_IMAGE), ], vec![ buffer_entry(0, vk::DescriptorType::UNIFORM_BUFFER, buffer.clone(), 8, 32), image_entry( 1, vk::DescriptorType::SAMPLED_IMAGE, view.clone(), vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, ), ], ); drop(buffer); drop(image); drop(view); let buffer = buffer_weak.upgrade().unwrap(); let image = image_weak.upgrade().unwrap(); let state = ResourceState::default(); let mut graph = RenderGraph::new(); let imported = graph .import_descriptor_set( set.clone(), &[ import_buffer_state(&buffer, state), import_texture_state(&image, state), ], &[], ) .unwrap(); assert!(buffer_weak.strong_count() >= 2); assert!(image_weak.strong_count() >= 2); assert!(view_weak.strong_count() >= 1); assert_eq!(graph.resources.len(), 2); assert_eq!(imported.accesses.len(), 2); assert!(graph.resources.iter().any(|desc| matches!( desc, ResourceDesc::ImportedBuffer { resource, .. } if Arc::ptr_eq(resource, &buffer) ))); assert!(graph.resources.iter().any(|desc| matches!( desc, ResourceDesc::ImportedTexture { resource, .. } if Arc::ptr_eq(resource, &image) ))); } #[test] fn managed_descriptor_import_deduplicates_buffer_in_both_import_orders() { let state = ResourceState::default(); let buffer = test_buffer(102, 64); let set = test_set( 2, vec![layout_binding(0, vk::DescriptorType::UNIFORM_BUFFER)], vec![buffer_entry( 0, vk::DescriptorType::UNIFORM_BUFFER, buffer.clone(), 0, 32, )], ); let mut graph = RenderGraph::new(); let direct = graph.import_buffer(buffer.clone(), state).unwrap(); let imported = graph .import_descriptor_set(set, &[import_buffer_state(&buffer, state)], &[]) .unwrap(); let DescriptorResourceAccess::Buffer { buffer: transitive, .. } = imported.accesses[0] else { panic!("buffer descriptor import produces a buffer scope"); }; assert_eq!(direct, transitive); assert_eq!(graph.resources.len(), 1); let buffer = test_buffer(103, 64); let set = test_set( 3, vec![layout_binding(0, vk::DescriptorType::UNIFORM_BUFFER)], vec![buffer_entry( 0, vk::DescriptorType::UNIFORM_BUFFER, buffer.clone(), 0, 32, )], ); let mut graph = RenderGraph::new(); let imported = graph .import_descriptor_set(set, &[import_buffer_state(&buffer, state)], &[]) .unwrap(); let DescriptorResourceAccess::Buffer { buffer: transitive, .. } = imported.accesses[0] else { panic!("buffer descriptor import produces a buffer scope"); }; let direct = graph.import_buffer(buffer, state).unwrap(); assert_eq!(direct, transitive); assert_eq!(graph.resources.len(), 1); } #[test] fn managed_descriptor_import_deduplicates_texture_in_both_import_orders() { let state = ResourceState::default(); let image = test_image(202); let view = test_view(image.clone()); let set = test_set( 4, vec![layout_binding(0, vk::DescriptorType::SAMPLED_IMAGE)], vec![image_entry( 0, vk::DescriptorType::SAMPLED_IMAGE, view, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, )], ); let mut graph = RenderGraph::new(); let direct = graph.import_texture(image.clone(), state).unwrap(); let imported = graph .import_descriptor_set(set, &[import_texture_state(&image, state)], &[]) .unwrap(); let DescriptorResourceAccess::Texture { texture: transitive, .. } = imported.accesses[0] else { panic!("image descriptor import produces a texture scope"); }; assert_eq!(direct, transitive); assert_eq!(graph.resources.len(), 1); let image = test_image(203); let view = test_view(image.clone()); let set = test_set( 5, vec![layout_binding(0, vk::DescriptorType::SAMPLED_IMAGE)], vec![image_entry( 0, vk::DescriptorType::SAMPLED_IMAGE, view, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, )], ); let mut graph = RenderGraph::new(); let imported = graph .import_descriptor_set(set, &[import_texture_state(&image, state)], &[]) .unwrap(); let DescriptorResourceAccess::Texture { texture: transitive, .. } = imported.accesses[0] else { panic!("image descriptor import produces a texture scope"); }; let direct = graph.import_texture(image, state).unwrap(); assert_eq!(direct, transitive); assert_eq!(graph.resources.len(), 1); } #[test] fn descriptor_import_rejects_conflicting_initial_state_in_both_orders() { let buffer_state = ResourceState::default(); let conflicting = ResourceState { stage: vk::PipelineStageFlags2::HOST, access: vk::AccessFlags2::HOST_WRITE, layout: None, }; let buffer = test_buffer(104, 64); let set = test_set( 6, vec![layout_binding(0, vk::DescriptorType::UNIFORM_BUFFER)], vec![buffer_entry( 0, vk::DescriptorType::UNIFORM_BUFFER, buffer.clone(), 0, 32, )], ); let mut graph = RenderGraph::new(); let direct = graph.import_buffer(buffer.clone(), buffer_state).unwrap(); assert!(matches!( graph.import_descriptor_set( set, &[import_buffer_state(&buffer, conflicting)], &[], ), Err(GraphError::ConflictingInitialState(id)) if id == direct.id() )); let buffer = test_buffer(105, 64); let set = test_set( 7, vec![layout_binding(0, vk::DescriptorType::UNIFORM_BUFFER)], vec![buffer_entry( 0, vk::DescriptorType::UNIFORM_BUFFER, buffer.clone(), 0, 32, )], ); let mut graph = RenderGraph::new(); let imported = graph .import_descriptor_set(set, &[import_buffer_state(&buffer, buffer_state)], &[]) .unwrap(); let DescriptorResourceAccess::Buffer { buffer: id, .. } = imported.accesses[0] else { panic!("buffer descriptor import produces a buffer scope"); }; assert_eq!( graph.import_buffer(buffer, conflicting), Err(GraphError::ConflictingInitialState(id.id())) ); } #[test] fn stale_managed_descriptor_set_is_rejected_at_import() { let buffer = test_buffer(106, 64); let set = Arc::new(DescriptorSet::for_testing( vk::DescriptorSet::from_raw(8), vec![layout_binding(0, vk::DescriptorType::UNIFORM_BUFFER)], 3, 4, vec![buffer_entry( 0, vk::DescriptorType::UNIFORM_BUFFER, buffer.clone(), 0, 32, )], )); let mut graph = RenderGraph::new(); assert!(matches!( graph.import_descriptor_set( set, &[import_buffer_state(&buffer, ResourceState::default())], &[], ), Err(GraphError::StaleDescriptorSet) )); } #[test] fn recording_preflight_rejects_stale_draw_before_graph_recording_steps() { let buffer = test_buffer(109, 64); let state = ResourceState { stage: vk::PipelineStageFlags2::HOST, access: vk::AccessFlags2::HOST_WRITE, layout: None, }; let mut graph = RenderGraph::new(); let imported = graph.import_buffer(buffer.clone(), state).unwrap(); let stale_set = Arc::new(DescriptorSet::for_testing( vk::DescriptorSet::from_raw(9), vec![layout_binding(0, vk::DescriptorType::UNIFORM_BUFFER)], 1, 2, vec![buffer_entry( 0, vk::DescriptorType::UNIFORM_BUFFER, buffer, 0, 32, )], )); graph.push(RecordingMarker(imported)); graph.push(BeginRendering { color: Vec::new(), depth: None, }); graph.push(Draw { args: DrawArgs::Direct { vertex_count: 3, instance_count: 1, first_vertex: 0, first_instance: 0, }, pipeline_layout: vk::PipelineLayout::null(), vertices: Vec::new(), index: None, indirect: None, descriptor_sets: vec![GraphDescriptorSet { raw: vk::DescriptorSet::from_raw(9), accesses: Vec::new(), managed: Some(stale_set), dynamic_offsets: Vec::new(), }], push_constants: Vec::new(), }); graph.push(EndRendering); let mut compiled = graph.compile().unwrap(); assert!(compiled.resource_lifetimes()[&imported.id()].imported); assert!(compiled.barriers().iter().any(|barrier| { barrier.before_command == CommandIndex(0) && !barrier.resources.is_empty() })); static BARRIER_CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); unsafe extern "system" fn count_barrier( _device: vk::Device, _command_buffer: vk::CommandBuffer, _dependency: *const vk::DependencyInfo<'_>, ) { BARRIER_CALLS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); } let device = unsafe { ash::Device::load_with( |name| { if name.to_bytes() == b"vkCmdPipelineBarrier2" { count_barrier as *const () as *const std::ffi::c_void } else { std::ptr::null() } }, vk::Device::null(), ) }; RECORDING_MARKER_APPLIED.store(0, std::sync::atomic::Ordering::SeqCst); BARRIER_CALLS.store(0, std::sync::atomic::Ordering::SeqCst); let mut recorder = CommandRecorder::new(&device, vk::CommandBuffer::null()); assert_eq!( compiled.record_commands(&mut recorder), Err(RecordError::StaleDescriptorSet) ); assert_eq!(BARRIER_CALLS.load(std::sync::atomic::Ordering::SeqCst), 0); assert_eq!( RECORDING_MARKER_APPLIED.load(std::sync::atomic::Ordering::SeqCst), 0 ); } #[test] fn failed_descriptor_import_does_not_append_earlier_valid_resources() { let buffer = test_buffer(110, 64); let set = test_set( 16, vec![ layout_binding(0, vk::DescriptorType::UNIFORM_BUFFER), layout_binding(1, vk::DescriptorType::INPUT_ATTACHMENT), ], vec![ buffer_entry(0, vk::DescriptorType::UNIFORM_BUFFER, buffer.clone(), 0, 32), TrackedDescriptorWrite { binding: 1, array_element: 0, descriptor_type: vk::DescriptorType::INPUT_ATTACHMENT, stages: vk::ShaderStageFlags::FRAGMENT, resource: DescriptorResourceBinding::Sampler { sampler: Arc::new(Sampler::for_testing(vk::Sampler::from_raw(3))), }, }, ], ); let mut graph = RenderGraph::new(); assert!(matches!( graph.import_descriptor_set( set, &[import_buffer_state(&buffer, ResourceState::default())], &[], ), Err(GraphError::DescriptorImport(_)) )); assert!(graph.resources.is_empty()); assert_eq!(graph.next_id, 0); assert!(graph.imported_buffers_by_ptr.is_empty()); assert!(graph.imported_buffers_by_raw.is_empty()); } #[test] fn failed_import_transaction_restores_exact_raw_identity_mapping() { let first = test_buffer(111, 64); let second = test_buffer(111, 64); assert!(!Arc::ptr_eq(&first, &second)); let state = ResourceState::default(); let mut graph = RenderGraph::new(); let first_id = graph.import_buffer(first.clone(), state).unwrap().id(); // Model an existing graph state with two pointer identities sharing a raw handle; the // raw-key choice is deliberately not the resource-order reconstruction's last entry. let second_id = graph.add_resource(ResourceDesc::ImportedBuffer { resource: second.clone(), initial: state, }); graph .imported_buffers_by_ptr .insert(Arc::as_ptr(&second) as usize, second_id); graph.imported_buffers_by_raw.insert(first.raw(), first_id); let ptr_before = graph.imported_buffers_by_ptr.clone(); let raw_before = graph.imported_buffers_by_raw.clone(); let resources_before = graph.resources.len(); let failed = graph.import_transaction::<()>(|graph| { // Pointer identity selects the second resource and updates its shared raw-key entry; // the subsequent import error must restore the prior raw-key choice exactly. graph.import_buffer(second.clone(), state)?; Err(GraphError::ConflictingDescriptorInitialStates) }); assert!(matches!( failed, Err(GraphError::ConflictingDescriptorInitialStates) )); assert_eq!(graph.resources.len(), resources_before); assert_eq!(graph.imported_buffers_by_ptr, ptr_before); assert_eq!(graph.imported_buffers_by_raw, raw_before); assert_eq!( graph.imported_buffers_by_raw.get(&first.raw()), Some(&first_id) ); // Pointer identity still selects the second resource, while a fresh Arc using the raw // handle follows the exact pre-transaction raw index. let fresh = test_buffer(111, 64); assert_eq!(graph.import_buffer(fresh, state).unwrap().id(), first_id); assert_eq!(graph.import_buffer(second, state).unwrap().id(), second_id); } #[test] fn descriptor_bound_uniform_copy_orders_before_draw_and_infers_usage() { let buffer = test_buffer(107, 64); let set = test_set( 9, vec![layout_binding(0, vk::DescriptorType::UNIFORM_BUFFER)], vec![buffer_entry( 0, vk::DescriptorType::UNIFORM_BUFFER, buffer.clone(), 0, 32, )], ); let state = ResourceState::default(); let mut graph = RenderGraph::new(); let destination = graph .import_descriptor_set(set.clone(), &[import_buffer_state(&buffer, state)], &[]) .unwrap(); let DescriptorResourceAccess::Buffer { buffer: uniform, .. } = destination.accesses[0] else { panic!("uniform descriptor maps to buffer access"); }; let source = graph.create_buffer(GraphBufferDesc { size: 64, ..Default::default() }); let range = BufferRange { offset: 0, size: 32, }; graph.push(CopyBuffer { src: source, src_range: range, dst: uniform, dst_range: range, }); graph.push(BeginRendering { color: Vec::new(), depth: None, }); graph.push(Draw { args: DrawArgs::Direct { vertex_count: 3, instance_count: 1, first_vertex: 0, first_instance: 0, }, pipeline_layout: vk::PipelineLayout::null(), vertices: Vec::new(), index: None, indirect: None, descriptor_sets: vec![destination], push_constants: Vec::new(), }); graph.push(EndRendering); let accesses = graph.commands.side_effects.command_accesses(); assert!( derive_buffer_usage( &accesses_for_resource(uniform.id(), accesses, &graph.commands.side_effects), vk::BufferUsageFlags::empty() ) .contains(vk::BufferUsageFlags::UNIFORM_BUFFER) ); let compiled = graph.compile().unwrap(); let batches = compiled.batches().collect::>(); assert_eq!( batches, [ &[CommandIndex(0)][..], &[CommandIndex(1), CommandIndex(2), CommandIndex(3)][..], ] ); let barrier = compiled .barriers() .iter() .find(|barrier| barrier.before_command == CommandIndex(1)) .unwrap(); assert!(barrier.resources.iter().any(|resource| { resource.id == uniform.id() && resource.dst_access == vk::AccessFlags2::UNIFORM_READ && resource.dst_stage == vk::PipelineStageFlags2::FRAGMENT_SHADER })); } #[test] fn sampled_image_import_sets_usage_and_layout_barrier() { let image = test_image(204); let view = test_view(image.clone()); let set = test_set( 10, vec![layout_binding(0, vk::DescriptorType::SAMPLED_IMAGE)], vec![image_entry( 0, vk::DescriptorType::SAMPLED_IMAGE, view, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, )], ); let initial = ResourceState { stage: vk::PipelineStageFlags2::TRANSFER, access: vk::AccessFlags2::TRANSFER_WRITE, layout: Some(vk::ImageLayout::GENERAL), }; let mut graph = RenderGraph::new(); let descriptor_set = graph .import_descriptor_set(set, &[import_texture_state(&image, initial)], &[]) .unwrap(); let DescriptorResourceAccess::Texture { texture, .. } = descriptor_set.accesses[0] else { panic!("sampled image maps to texture access"); }; graph.push(BeginRendering { color: Vec::new(), depth: None, }); graph.push(Draw { args: DrawArgs::Direct { vertex_count: 3, instance_count: 1, first_vertex: 0, first_instance: 0, }, pipeline_layout: vk::PipelineLayout::null(), vertices: Vec::new(), index: None, indirect: None, descriptor_sets: vec![descriptor_set], push_constants: Vec::new(), }); graph.push(EndRendering); let accesses = graph.commands.side_effects.command_accesses(); assert!( derive_image_usage( &accesses_for_resource(texture.id(), accesses, &graph.commands.side_effects), vk::ImageUsageFlags::empty() ) .contains(vk::ImageUsageFlags::SAMPLED) ); let compiled = graph.compile().unwrap(); let barrier = compiled .barriers() .iter() .find(|barrier| barrier.before_command == CommandIndex(0)) .expect("initial sampled-image transition is hoisted before BeginRendering"); assert_eq!(barrier.resources.len(), 1); let barrier = &barrier.resources[0]; assert_eq!(barrier.old_layout, Some(vk::ImageLayout::GENERAL)); assert_eq!( barrier.new_layout, Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) ); assert_eq!(barrier.dst_access, vk::AccessFlags2::SHADER_SAMPLED_READ); } #[test] fn sampler_only_managed_binding_has_no_graph_resource_or_memory_access() { let sampler = Arc::new(Sampler::for_testing(vk::Sampler::from_raw(1))); let set = test_set( 11, vec![layout_binding(0, vk::DescriptorType::SAMPLER)], vec![TrackedDescriptorWrite { binding: 0, array_element: 0, descriptor_type: vk::DescriptorType::SAMPLER, stages: vk::ShaderStageFlags::FRAGMENT, resource: DescriptorResourceBinding::Sampler { sampler }, }], ); let mut graph = RenderGraph::new(); let imported = graph.import_descriptor_set(set, &[], &[]).unwrap(); assert!(imported.accesses.is_empty()); assert!(graph.resources.is_empty()); } #[test] fn dynamic_offsets_across_sets_follow_binding_order_and_validate_effective_ranges() { let buffer = test_buffer(108, 256); let first = test_set( 12, vec![ DescriptorSetLayoutBindingDesc { binding: 8, count: 1, ..layout_binding(8, vk::DescriptorType::STORAGE_BUFFER_DYNAMIC) }, DescriptorSetLayoutBindingDesc { binding: 2, count: 1, ..layout_binding(2, vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC) }, ], vec![ buffer_entry( 2, vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC, buffer.clone(), 16, 32, ), buffer_entry( 8, vk::DescriptorType::STORAGE_BUFFER_DYNAMIC, buffer.clone(), 64, 32, ), ], ); let second = test_set( 13, vec![layout_binding( 1, vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC, )], vec![buffer_entry( 1, vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC, buffer.clone(), 128, 32, )], ); let state = ResourceState::default(); let mut graph = RenderGraph::new(); let initial = [import_buffer_state(&buffer, state)]; let first = graph .import_descriptor_set(first, &initial, &[8, 16]) .unwrap(); let second = graph .import_descriptor_set(second, &initial, &[24]) .unwrap(); assert_eq!(first.dynamic_offsets(), &[8, 16]); assert_eq!(second.dynamic_offsets(), &[24]); assert_eq!( first .dynamic_offsets() .iter() .chain(second.dynamic_offsets()) .copied() .collect::>(), [8, 16, 24] ); let ranges = first .accesses .iter() .chain(&second.accesses) .map(|access| match access { DescriptorResourceAccess::Buffer { range, .. } => *range, _ => panic!("dynamic buffer descriptor maps to buffer range"), }) .collect::>(); assert_eq!( ranges, [ BufferRange { offset: 24, size: 32 }, BufferRange { offset: 80, size: 32 }, BufferRange { offset: 152, size: 32 }, ] ); let out_of_bounds = test_set( 14, vec![layout_binding( 0, vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC, )], vec![buffer_entry( 0, vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC, buffer.clone(), 224, 32, )], ); assert!( graph .import_descriptor_set(out_of_bounds, &initial, &[8]) .is_err() ); } #[test] fn managed_descriptor_import_rejects_unsupported_descriptor_kind() { let set = test_set( 15, vec![layout_binding(0, vk::DescriptorType::INPUT_ATTACHMENT)], vec![TrackedDescriptorWrite { binding: 0, array_element: 0, descriptor_type: vk::DescriptorType::INPUT_ATTACHMENT, stages: vk::ShaderStageFlags::FRAGMENT, resource: DescriptorResourceBinding::Sampler { sampler: Arc::new(Sampler::for_testing(vk::Sampler::from_raw(2))), }, }], ); assert!(matches!( RenderGraph::new().import_descriptor_set(set, &[], &[]), Err(GraphError::DescriptorImport(_)) )); } #[test] fn kahn_emits_dag_in_deterministic_order() { let mut kahn = Kahn::new(4); kahn.add_edge(0, 2); kahn.add_edge(0, 2); kahn.add_edge(1, 2); kahn.add_edge(2, 3); let mut batches = Vec::new(); while !kahn.is_complete() { let batch = kahn.pop_batch(); assert!(!batch.is_empty()); for &node in &batch { kahn.complete(node); } batches.push(batch); } assert_eq!(batches, [vec![0, 1], vec![2], vec![3]]); assert!(kahn.is_complete()); } #[test] fn kahn_detects_cycles() { let mut kahn = Kahn::new(3); kahn.add_edge(0, 1); kahn.add_edge(1, 2); kahn.add_edge(2, 1); for node in kahn.pop_batch() { kahn.complete(node); } assert_eq!(kahn.emitted, 1); assert!(!kahn.is_complete()); } #[test] fn kahn_handles_holes() { let mut kahn = Kahn::new(6); kahn.add_edge(3, 2); kahn.add_edge(4, 2); kahn.add_edge(2, 1); kahn.add_node(5); assert_eq!(vec![3, 4, 5], kahn.pop_batch()); for node in [3, 4, 5] { kahn.complete(node); } assert_eq!(vec![2], kahn.pop_batch()); for node in [2] { kahn.complete(node); } assert_eq!(vec![1], kahn.pop_batch()); for node in [1] { kahn.complete(node); } dbg!(&kahn); assert!(kahn.is_complete()); } #[test] fn interference_tracks_overlapping_inclusive_lifetimes() { let lifetimes = HashMap::from([ ( ResourceId::from_u32(0), ResourceLifetime { first_batch: 0, last_batch: 2, placement: Placement { offset: 0, size: 1, alignment: 1, pool: None, aliased: false, }, imported: false, }, ), ( ResourceId::from_u32(1), ResourceLifetime { first_batch: 2, last_batch: 3, placement: Placement { offset: 0, size: 1, alignment: 1, pool: None, aliased: false, }, imported: false, }, ), ( ResourceId::from_u32(2), ResourceLifetime { first_batch: 4, last_batch: 5, placement: Placement { offset: 0, size: 1, alignment: 1, pool: None, aliased: false, }, imported: false, }, ), ]); let interference = ResourceInterference::from_lifetimes( lifetimes.iter().map(|(&id, &lifetime)| (id, lifetime)), ); assert!(interference.interferes(ResourceId::from_u32(0), ResourceId::from_u32(1))); assert!(interference.interferes(ResourceId::from_u32(1), ResourceId::from_u32(0))); assert!(!interference.interferes(ResourceId::from_u32(0), ResourceId::from_u32(2))); assert!(!interference.interferes(ResourceId::from_u32(1), ResourceId::from_u32(2))); } #[test] fn lifetime_analysis_tracks_batches_and_command_level_interference() { let resources = vec![ ResourceDesc::CreatedBuffer(GraphBufferDesc { size: 8, ..Default::default() }), ResourceDesc::CreatedBuffer(GraphBufferDesc { size: 8, ..Default::default() }), ResourceDesc::CreatedBuffer(GraphBufferDesc { size: 8, ..Default::default() }), ]; let accesses = vec![ vec![resource_access_for(ResourceId::from_u32(0))], vec![ resource_access_for(ResourceId::from_u32(0)), resource_access_for(ResourceId::from_u32(1)), ], vec![resource_access_for(ResourceId::from_u32(0))], vec![resource_access_for(ResourceId::from_u32(2))], ]; let orderings = ordering_from_accesses(&resources, &accesses); let batches = orderings .batches .iter() .map(|batch| orderings.execution_order[batch.start..batch.start + batch.count].to_vec()) .collect::>(); let happens_before = HappensBefore::from_command_dag(&orderings.dependency_edges, &batches); let compiler = GraphCompiler::new_with_nodes( &resources, &accesses, accesses.len(), &SideEffectMap::default(), orderings.node_of_command, orderings.node_representatives, &orderings.spans, ); let uses_by_resource = compiler.resource_uses(); let (lifetimes, interference) = compiler.bind_lifetimes_for_test( &batches, &ResourceFrontiers::from_uses(&uses_by_resource, &happens_before), &happens_before, &uses_by_resource, &[], ); assert_eq!(lifetimes[&ResourceId::from_u32(0)].first_batch, 0); assert_eq!(lifetimes[&ResourceId::from_u32(0)].last_batch, 2); assert!(interference.interferes(ResourceId::from_u32(0), ResourceId::from_u32(1))); assert!(interference.interferes(ResourceId::from_u32(0), ResourceId::from_u32(2))); assert!(interference.interferes(ResourceId::from_u32(1), ResourceId::from_u32(2))); } #[test] fn happens_before_closure_covers_direct_and_transitive_edges() { let edges = vec![vec![1], vec![2], vec![], vec![]]; let batches = vec![ vec![CommandIndex(0), CommandIndex(3)], vec![CommandIndex(1)], vec![CommandIndex(2)], ]; let happens_before = HappensBefore::from_command_dag(&edges, &batches); assert!(happens_before.happens_before(CommandIndex(0), CommandIndex(1))); assert!(happens_before.happens_before(CommandIndex(0), CommandIndex(2))); assert!(happens_before.happens_before(CommandIndex(1), CommandIndex(2))); assert!(!happens_before.happens_before(CommandIndex(2), CommandIndex(0))); assert!(!happens_before.happens_before(CommandIndex(0), CommandIndex(3))); assert!(!happens_before.happens_before(CommandIndex(0), CommandIndex(0))); } #[test] fn same_batch_resource_uses_interfere() { let edges = vec![vec![], vec![]]; let batches = vec![vec![CommandIndex(0), CommandIndex(1)]]; let happens_before = HappensBefore::from_command_dag(&edges, &batches); let a = [ResourceUse { command: CommandIndex(0), stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, }]; let b = [ResourceUse { command: CommandIndex(1), stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, }]; let use_sets = HashMap::from([ (ResourceId::from_u32(0), a.to_vec()), (ResourceId::from_u32(1), b.to_vec()), ]); let interference = ResourceInterference::from_frontiers( &ResourceFrontiers::from_uses(&use_sets, &happens_before), &happens_before, ); assert!(interference.interferes(ResourceId::from_u32(0), ResourceId::from_u32(1))); assert!(!resources_alias_in_order(&a, &b, &happens_before)); assert!(!resources_alias_in_order(&b, &a, &happens_before)); } #[test] fn unordered_distinct_batches_interfere_even_without_interval_overlap() { // This reproduces the plan's ready-set shape: 0 and 2 are batch-ready; // 1 depends on 0, but 2 has no edge to 1. let edges = vec![vec![1], vec![], vec![]]; let batches = vec![ vec![CommandIndex(0), CommandIndex(2)], vec![CommandIndex(1)], ]; let happens_before = HappensBefore::from_command_dag(&edges, &batches); let resource_a = [ResourceUse { command: CommandIndex(2), stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, }]; let resource_b = [ResourceUse { command: CommandIndex(1), stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, }]; assert!(!happens_before.happens_before(CommandIndex(2), CommandIndex(1))); assert!(!resources_alias_in_order( &resource_a, &resource_b, &happens_before, )); let use_sets = HashMap::from([ (ResourceId::from_u32(0), resource_a.to_vec()), (ResourceId::from_u32(1), resource_b.to_vec()), ]); let interference = ResourceInterference::from_frontiers( &ResourceFrontiers::from_uses(&use_sets, &happens_before), &happens_before, ); assert!(interference.interferes(ResourceId::from_u32(0), ResourceId::from_u32(1))); } #[test] fn dependency_chain_proves_alias_reuse_safe() { let resources = (0..5) .map(|_| { ResourceDesc::CreatedBuffer(GraphBufferDesc { size: 8, ..Default::default() }) }) .collect::>(); let accesses = vec![ vec![ resource_access_for(ResourceId::from_u32(0)), resource_access_for(ResourceId::from_u32(2)), ], vec![ resource_access_for(ResourceId::from_u32(2)), resource_access_for(ResourceId::from_u32(3)), ], vec![ resource_access_for(ResourceId::from_u32(3)), resource_access_for(ResourceId::from_u32(1)), ], ]; let orderings = ordering_from_accesses(&resources, &accesses); let batches = orderings .batches .iter() .map(|batch| orderings.execution_order[batch.start..batch.start + batch.count].to_vec()) .collect::>(); let happens_before = HappensBefore::from_command_dag(&orderings.dependency_edges, &batches); let compiler = GraphCompiler::new_with_nodes( &resources, &accesses, accesses.len(), &SideEffectMap::default(), orderings.node_of_command, orderings.node_representatives, &orderings.spans, ); let uses = compiler.resource_uses(); assert!(happens_before.happens_before(CommandIndex(0), CommandIndex(2))); assert!(resources_alias_in_order( &uses[&ResourceId::from_u32(0)], &uses[&ResourceId::from_u32(1)], &happens_before, )); let uses_a = &uses[&ResourceId::from_u32(0)]; let uses_b = &uses[&ResourceId::from_u32(1)]; assert!(aliasing_permitted(uses_a, uses_b, &happens_before)); } #[test] fn copy_buffer_compiles_with_dependency_and_barrier() { let mut graph = RenderGraph::new(); let src = graph.create_buffer(GraphBufferDesc { size: 64, ..Default::default() }); let dst = graph.create_buffer(GraphBufferDesc { size: 64, ..Default::default() }); graph.push(CopyBuffer { src, src_range: BufferRange { offset: 0, size: 64, }, dst, dst_range: BufferRange { offset: 0, size: 64, }, }); let compiled = graph.compile().expect("graph compiles"); assert_eq!( compiled.batches().flatten().copied().collect::>(), vec![CommandIndex(0)] ); assert!(compiled.barriers().is_empty()); assert_eq!(compiled.resource_lifetimes().len(), 2); assert!(compiled.interference().interferes(src.id(), dst.id())); assert!(!compiled.resource_lifetimes()[&src.id()].imported); } #[test] fn render_pass_spans_validate_classification_and_balance() { let mut graph = RenderGraph::new(); graph.push(BeginRendering { color: Vec::new(), depth: None, }); graph.push(EndRendering); let compiled = graph.compile().unwrap(); assert_eq!(compiled.render_pass_spans().len(), 1); assert_eq!(compiled.render_pass_spans()[0].begin, CommandIndex(0)); assert_eq!(compiled.render_pass_spans()[0].end, CommandIndex(1)); let mut unbalanced = RenderGraph::new(); unbalanced.push(EndRendering); assert!(matches!( unbalanced.compile(), Err(GraphError::UnbalancedRenderPass) )); let mut unclosed = RenderGraph::new(); unclosed.push(empty_render_pass()); assert!(matches!( unclosed.compile(), Err(GraphError::UnbalancedRenderPass) )); let mut nested = RenderGraph::new(); nested.push(empty_render_pass()); nested.push(empty_render_pass()); assert!(matches!( nested.compile(), Err(GraphError::NestedRenderPass) )); } #[test] fn render_pass_span_is_one_node_and_expands_in_record_order() { let mut graph = RenderGraph::new(); let source = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let shared = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let range = BufferRange { offset: 0, size: 8 }; graph.push(CopyBuffer { src: source, src_range: range, dst: shared, dst_range: range, }); graph.push(empty_render_pass()); graph.push(Draw { args: DrawArgs::Direct { vertex_count: 3, instance_count: 1, first_vertex: 0, first_instance: 0, }, pipeline_layout: vk::PipelineLayout::null(), vertices: Vec::new(), index: None, indirect: None, descriptor_sets: Vec::new(), push_constants: Vec::new(), }); graph.push(EndRendering); graph.push(CopyBuffer { src: source, src_range: range, dst: shared, dst_range: range, }); graph.commands.side_effects.insert( CommandIndex(2), shared.id(), SubresourceScope { stage: vk::PipelineStageFlags2::VERTEX_SHADER, access: vk::AccessFlags2::SHADER_READ, subresource: SubresourceScopeInner::Buffer(range), }, ); let compiled = graph.compile().expect("graph compiles"); assert_eq!( compiled.execution_order(), [ CommandIndex(0), CommandIndex(1), CommandIndex(2), CommandIndex(3), CommandIndex(4), ] ); assert_eq!( compiled.batches().collect::>(), [ &[CommandIndex(0)][..], &[CommandIndex(1), CommandIndex(2), CommandIndex(3)][..], &[CommandIndex(4)][..], ] ); assert_eq!(compiled.dependency_edges().len(), 3); assert!(compiled.happens_before(CommandIndex(0), CommandIndex(2))); assert!(compiled.happens_before(CommandIndex(2), CommandIndex(4))); assert!(!compiled.happens_before(CommandIndex(1), CommandIndex(2))); assert!(!compiled.happens_before(CommandIndex(2), CommandIndex(3))); assert_eq!(compiled.resource_lifetimes()[&shared.id()].first_batch, 0); assert_eq!(compiled.resource_lifetimes()[&shared.id()].last_batch, 2); } #[test] fn render_pass_rejects_graphics_outside_and_transfer_inside() { let mut outside = RenderGraph::new(); outside.push(Draw { args: DrawArgs::Direct { vertex_count: 3, instance_count: 1, first_vertex: 0, first_instance: 0, }, pipeline_layout: vk::PipelineLayout::null(), vertices: Vec::new(), index: None, indirect: None, descriptor_sets: Vec::new(), push_constants: Vec::new(), }); assert!(matches!( outside.compile(), Err(GraphError::DrawOutsideRenderPass(CommandIndex(0))) )); let mut inside = RenderGraph::new(); let source = inside.create_buffer(GraphBufferDesc { size: 16, ..Default::default() }); let destination = inside.create_buffer(GraphBufferDesc { size: 16, ..Default::default() }); inside.push(empty_render_pass()); inside.push(CopyBuffer { src: source, src_range: BufferRange { offset: 0, size: 16, }, dst: destination, dst_range: BufferRange { offset: 0, size: 16, }, }); inside.push(EndRendering); assert!(matches!( inside.compile(), Err(GraphError::CommandNotAllowedInRenderPass(CommandIndex(1))) )); } #[test] fn render_pass_attachment_validation_and_lifetime_access() { let mut graph = RenderGraph::new(); let texture = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, ..Default::default() }); let mut begin = empty_render_pass(); begin.color.push(created_color_attachment( texture, vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, )); graph.push(begin); graph.push(EndRendering); let compiled = graph.compile().unwrap(); assert!( compiled .commands .side_effects .is_lifetime_only(CommandIndex(1), texture.id()) ); let uses = compiled.resource_uses(texture.id()).unwrap(); assert!(uses.iter().any(|use_| use_.command == CommandIndex(1))); assert!(!compiled.barriers().iter().any(|barrier| { barrier.before_command == CommandIndex(1) && barrier .resources .iter() .any(|resource| resource.id == texture.id()) })); let mut duplicate = RenderGraph::new(); let texture = duplicate.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, ..Default::default() }); let mut begin = empty_render_pass(); begin .color .push(created_color_attachment(texture, vk::ImageLayout::GENERAL)); begin .color .push(created_color_attachment(texture, vk::ImageLayout::GENERAL)); duplicate.push(begin); duplicate.push(EndRendering); assert!(matches!( duplicate.compile(), Err(GraphError::InvalidRenderPassAttachment(_)) )); let mut invalid_layout = RenderGraph::new(); let texture = invalid_layout.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, ..Default::default() }); let mut begin = empty_render_pass(); begin.color.push(created_color_attachment( texture, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, )); invalid_layout.push(begin); invalid_layout.push(EndRendering); assert!(matches!( invalid_layout.compile(), Err(GraphError::InvalidRenderPassAttachment(_)) )); let mut sentinel = RenderGraph::new(); let texture = sentinel.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, ..Default::default() }); let mut begin = empty_render_pass(); let mut attachment = created_color_attachment(texture, vk::ImageLayout::GENERAL); let AttachmentView::Created(view) = &mut attachment.view else { unreachable!(); }; view.mip_range.end = vk::REMAINING_MIP_LEVELS; begin.color.push(attachment); sentinel.push(begin); sentinel.push(EndRendering); assert!(matches!( sentinel.compile(), Err(GraphError::InvalidRenderPassAttachment(_)) )); let mut mismatch = RenderGraph::new(); let graph_image = test_image(602); let foreign_image = test_image(603); let texture = mismatch .import_texture(graph_image, ResourceState::default()) .unwrap(); let mut begin = empty_render_pass(); begin.color.push(ColorAttachment::imported( texture, test_view(foreign_image), vk::ImageLayout::GENERAL, vk::AttachmentLoadOp::DONT_CARE, vk::AttachmentStoreOp::STORE, vk::ClearValue::default(), )); mismatch.push(begin); mismatch.push(EndRendering); assert!(matches!( mismatch.compile(), Err(GraphError::InvalidRenderPassAttachment(_)) )); } #[test] fn render_pass_feedback_loop_rejects_overlapping_interior_texture_access() { let mut graph = RenderGraph::new(); let texture = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, mip_levels: 2, ..Default::default() }); let mut begin = empty_render_pass(); begin .color .push(created_color_attachment(texture, vk::ImageLayout::GENERAL)); graph.push(begin); struct TextureAccess(Texture); impl crate::commands::Command for TextureAccess { fn side_effects(&self, command: CommandIndex, map: &mut SideEffectMap) { map.insert( command, self.0.id(), SubresourceScope { stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, access: vk::AccessFlags2::SHADER_READ, subresource: SubresourceScopeInner::Texture { range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), layout: vk::ImageLayout::GENERAL, }, }, ); } } impl crate::recorder::RecordedCommand for TextureAccess { fn allowed_in_render_pass(&self) -> bool { true } fn apply(&mut self, _: &mut CommandRecorder<'_>) -> Result<(), RecordError> { Ok(()) } } graph.push(TextureAccess(texture)); graph.push(EndRendering); assert!(matches!( graph.compile(), Err(GraphError::RenderPassFeedbackLoop(_)) )); } #[test] fn intra_pass_same_layout_barriers_are_allowed_but_layout_transitions_are_rejected() { struct ImageUse { texture: Texture, layout: vk::ImageLayout, access: vk::AccessFlags2, } impl crate::commands::Command for ImageUse { fn side_effects(&self, command: CommandIndex, map: &mut SideEffectMap) { map.insert( command, self.texture.id(), SubresourceScope { stage: vk::PipelineStageFlags2::COMPUTE_SHADER, access: self.access, subresource: SubresourceScopeInner::Texture { range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), layout: self.layout, }, }, ); } } impl crate::recorder::RecordedCommand for ImageUse { fn allowed_in_render_pass(&self) -> bool { true } fn apply(&mut self, _: &mut CommandRecorder<'_>) -> Result<(), RecordError> { Ok(()) } } let make_graph = |second_layout| { let mut graph = RenderGraph::new(); let texture = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, ..Default::default() }); graph.push(empty_render_pass()); graph.push(ImageUse { texture, layout: vk::ImageLayout::GENERAL, access: vk::AccessFlags2::SHADER_STORAGE_WRITE, }); graph.push(ImageUse { texture, layout: second_layout, access: vk::AccessFlags2::SHADER_STORAGE_READ, }); graph.push(EndRendering); graph }; let same_layout = make_graph(vk::ImageLayout::GENERAL) .compile() .expect("same-layout in-pass barrier is supported"); assert!(same_layout.barriers().iter().any(|barrier| { barrier.before_command == CommandIndex(2) && barrier.resources.iter().any(|resource| { resource.old_layout == Some(vk::ImageLayout::GENERAL) && resource.new_layout == Some(vk::ImageLayout::GENERAL) }) })); assert!(matches!( make_graph(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL).compile(), Err(GraphError::UnsupportedIntraPassDependency) )); } #[test] fn hazard_free_foreign_command_is_scheduled_outside_render_pass_span() { let mut graph = RenderGraph::new(); graph.push(empty_render_pass()); graph.push(AllowedAccess(Vec::new())); graph.push(EndRendering); graph.push(AllowedAccess(Vec::new())); let compiled = graph.compile().expect("graph compiles"); let begin = compiled .execution_order() .iter() .position(|&command| command == CommandIndex(0)) .unwrap(); let end = compiled .execution_order() .iter() .position(|&command| command == CommandIndex(2)) .unwrap(); let foreign = compiled .execution_order() .iter() .position(|&command| command == CommandIndex(3)) .unwrap(); assert!(foreign < begin || foreign > end); } #[test] fn interior_access_barriers_are_hoisted_and_coalesced() { let mut graph = RenderGraph::new(); let source = graph.create_buffer(GraphBufferDesc { size: 32, ..Default::default() }); let produced = graph.create_buffer(GraphBufferDesc { size: 32, ..Default::default() }); let range = BufferRange { offset: 0, size: 32, }; let producer = graph.push(CopyBuffer { src: source, src_range: range, dst: produced, dst_range: range, }); graph.push(empty_render_pass()); graph.push(AllowedAccess(vec![( produced.id(), SubresourceScope { stage: vk::PipelineStageFlags2::VERTEX_SHADER, access: vk::AccessFlags2::SHADER_READ, subresource: SubresourceScopeInner::Buffer(range), }, )])); graph.push(AllowedAccess(vec![( produced.id(), SubresourceScope { stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, access: vk::AccessFlags2::UNIFORM_READ, subresource: SubresourceScopeInner::Buffer(range), }, )])); graph.push(EndRendering); let compiled = graph.compile().expect("graph compiles"); let begin = compiled .barriers() .iter() .find(|barrier| barrier.before_command == CommandIndex(1)) .expect("outside-producer dependency is hoisted to begin"); let resource = begin .resources .iter() .find(|resource| resource.id == produced.id()) .expect("producer buffer barrier is present"); assert_eq!(resource.origin, BarrierOrigin::Command(producer)); let consumers = begin .resources .iter() .filter(|resource| resource.id == produced.id()) .collect::>(); assert_eq!(consumers.len(), 1); assert_eq!( consumers[0].dst_stage, vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER ); assert_eq!( consumers[0].dst_access, vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::UNIFORM_READ ); } #[test] fn initial_storage_image_transition_is_hoisted_before_begin() { let mut graph = RenderGraph::new(); let texture = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, extra_usage: vk::ImageUsageFlags::STORAGE, ..Default::default() }); graph.push(empty_render_pass()); graph.push(AllowedAccess(vec![( texture.id(), SubresourceScope { stage: vk::PipelineStageFlags2::COMPUTE_SHADER, access: vk::AccessFlags2::SHADER_STORAGE_WRITE, subresource: SubresourceScopeInner::Texture { range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), layout: vk::ImageLayout::GENERAL, }, }, )])); graph.push(EndRendering); let compiled = graph.compile().expect("graph compiles"); let begin = compiled .barriers() .iter() .find(|barrier| barrier.before_command == CommandIndex(0)) .expect("initial image state transition is hoisted to begin"); let resource = begin .resources .iter() .find(|resource| resource.id == texture.id()) .expect("storage image transition is present"); assert_eq!(resource.origin, BarrierOrigin::Initial); assert_eq!(resource.old_layout, Some(vk::ImageLayout::UNDEFINED)); assert_eq!(resource.new_layout, Some(vk::ImageLayout::GENERAL)); } #[test] fn outside_image_transition_to_interior_readers_is_coalesced() { let mut graph = RenderGraph::new(); let texture = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, ..Default::default() }); let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); let producer = graph.push(AllowedAccess(vec![( texture.id(), SubresourceScope { stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, subresource: SubresourceScopeInner::Texture { range, layout: vk::ImageLayout::GENERAL, }, }, )])); graph.push(empty_render_pass()); graph.push(AllowedAccess(vec![( texture.id(), SubresourceScope { stage: vk::PipelineStageFlags2::VERTEX_SHADER, access: vk::AccessFlags2::SHADER_READ, subresource: SubresourceScopeInner::Texture { range, layout: vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, }, }, )])); graph.push(AllowedAccess(vec![( texture.id(), SubresourceScope { stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, access: vk::AccessFlags2::SHADER_SAMPLED_READ, subresource: SubresourceScopeInner::Texture { range, layout: vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, }, }, )])); graph.push(EndRendering); let compiled = graph.compile().expect("outside transition is hoisted"); let begin = compiled .barriers() .iter() .find(|barrier| barrier.before_command == CommandIndex(1)) .expect("transition is keyed at begin"); let resources = begin .resources .iter() .filter(|resource| resource.id == texture.id()) .collect::>(); assert_eq!(resources.len(), 1); assert_eq!(resources[0].origin, BarrierOrigin::Command(producer)); assert_eq!(resources[0].old_layout, Some(vk::ImageLayout::GENERAL)); assert_eq!( resources[0].new_layout, Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) ); assert_eq!( resources[0].dst_stage, vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER ); assert_eq!( resources[0].dst_access, vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_SAMPLED_READ ); } #[test] fn interior_storage_buffer_hazard_emits_barrier_without_splitting_pass() { let mut graph = RenderGraph::new(); let buffer = graph.create_buffer(GraphBufferDesc { size: 32, extra_usage: vk::BufferUsageFlags::STORAGE_BUFFER, ..Default::default() }); let range = BufferRange { offset: 0, size: 32, }; graph.push(empty_render_pass()); graph.push(AllowedAccess(vec![( buffer.id(), SubresourceScope { stage: vk::PipelineStageFlags2::COMPUTE_SHADER, access: vk::AccessFlags2::SHADER_STORAGE_WRITE, subresource: SubresourceScopeInner::Buffer(range), }, )])); graph.push(AllowedAccess(vec![( buffer.id(), SubresourceScope { stage: vk::PipelineStageFlags2::COMPUTE_SHADER, access: vk::AccessFlags2::SHADER_STORAGE_READ, subresource: SubresourceScopeInner::Buffer(range), }, )])); graph.push(EndRendering); let compiled = graph.compile().expect("same-pass buffer barrier is legal"); assert_eq!( compiled.batches().collect::>(), [&[ CommandIndex(0), CommandIndex(1), CommandIndex(2), CommandIndex(3), ][..]] ); assert_eq!(compiled.render_pass_spans().len(), 1); assert!(compiled.barriers().iter().any(|barrier| { barrier.before_command == CommandIndex(2) && barrier.resources.iter().any(|resource| { resource.id == buffer.id() && resource.src_access == vk::AccessFlags2::SHADER_STORAGE_WRITE && resource.dst_access == vk::AccessFlags2::SHADER_STORAGE_READ }) })); } #[test] fn interior_storage_image_same_layout_hazard_emits_image_barrier() { let mut graph = RenderGraph::new(); let texture = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, extra_usage: vk::ImageUsageFlags::STORAGE, ..Default::default() }); let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); graph.push(empty_render_pass()); graph.push(AllowedAccess(vec![( texture.id(), SubresourceScope { stage: vk::PipelineStageFlags2::COMPUTE_SHADER, access: vk::AccessFlags2::SHADER_STORAGE_WRITE, subresource: SubresourceScopeInner::Texture { range, layout: vk::ImageLayout::GENERAL, }, }, )])); graph.push(AllowedAccess(vec![( texture.id(), SubresourceScope { stage: vk::PipelineStageFlags2::COMPUTE_SHADER, access: vk::AccessFlags2::SHADER_STORAGE_READ, subresource: SubresourceScopeInner::Texture { range, layout: vk::ImageLayout::GENERAL, }, }, )])); graph.push(EndRendering); let compiled = graph.compile().expect("same-layout image barrier is legal"); assert!(compiled.barriers().iter().any(|barrier| { barrier.before_command == CommandIndex(2) && barrier.resources.iter().any(|resource| { resource.id == texture.id() && resource.old_layout == Some(vk::ImageLayout::GENERAL) && resource.new_layout == Some(vk::ImageLayout::GENERAL) }) })); } #[test] fn interior_only_resource_lifetime_maps_to_pass_batch() { let mut graph = RenderGraph::new(); let buffer = graph.create_buffer(GraphBufferDesc { size: 16, ..Default::default() }); graph.push(empty_render_pass()); graph.push(AllowedAccess(vec![( buffer.id(), SubresourceScope { stage: vk::PipelineStageFlags2::VERTEX_SHADER, access: vk::AccessFlags2::SHADER_READ, subresource: SubresourceScopeInner::Buffer(BufferRange { offset: 0, size: 16, }), }, )])); graph.push(EndRendering); let compiled = graph.compile().expect("graph compiles"); let batch = compiled .batches() .position(|commands| commands.contains(&CommandIndex(0))) .unwrap(); let lifetime = compiled.resource_lifetimes()[&buffer.id()]; assert_eq!(lifetime.first_batch, batch); assert_eq!(lifetime.last_batch, batch); assert_ne!(lifetime.first_batch, usize::MAX); } #[test] fn render_pass_attachment_and_interior_image_interfere_until_end() { let mut graph = RenderGraph::new(); let attachment = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, ..Default::default() }); let storage = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, extra_usage: vk::ImageUsageFlags::STORAGE, ..Default::default() }); let mut begin = empty_render_pass(); begin.color.push(created_color_attachment( attachment, vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, )); let begin_index = graph.push(begin); let interior = graph.push(AllowedAccess(vec![( storage.id(), SubresourceScope { stage: vk::PipelineStageFlags2::COMPUTE_SHADER, access: vk::AccessFlags2::SHADER_STORAGE_WRITE, subresource: SubresourceScopeInner::Texture { range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), layout: vk::ImageLayout::GENERAL, }, }, )])); let end_index = graph.push(EndRendering); let compiled = graph.compile().expect("graph compiles"); assert!( compiled .interference() .interferes(attachment.id(), storage.id()) ); assert!(!compiled.barriers().iter().any(|barrier| { barrier.before_command == interior && barrier .resources .iter() .any(|resource| resource.id == attachment.id() || resource.id == storage.id()) })); let uses = compiled.resource_uses(attachment.id()).unwrap(); assert!(uses.iter().any(|use_| use_.command == begin_index)); assert!(uses.iter().any(|use_| use_.command == end_index)); let execution_order = compiled.execution_order(); let end_position = execution_order .iter() .position(|&command| command == end_index) .unwrap(); let last_attachment_use = uses .iter() .map(|use_| { execution_order .iter() .position(|&command| command == use_.command) .unwrap() }) .max() .unwrap(); assert_eq!(last_attachment_use, end_position); } #[test] fn dependencies_order_commands_around_the_whole_pass_but_not_within_it() { let mut graph = RenderGraph::new(); let source = graph.create_buffer(GraphBufferDesc { size: 16, ..Default::default() }); let shared = graph.create_buffer(GraphBufferDesc { size: 16, ..Default::default() }); let range = BufferRange { offset: 0, size: 16, }; let producer = graph.push(CopyBuffer { src: source, src_range: range, dst: shared, dst_range: range, }); graph.push(empty_render_pass()); let interior_a = graph.push(AllowedAccess(vec![( shared.id(), SubresourceScope { stage: vk::PipelineStageFlags2::VERTEX_SHADER, access: vk::AccessFlags2::SHADER_READ, subresource: SubresourceScopeInner::Buffer(range), }, )])); let interior_b = graph.push(AllowedAccess(vec![( shared.id(), SubresourceScope { stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, access: vk::AccessFlags2::SHADER_READ, subresource: SubresourceScopeInner::Buffer(range), }, )])); graph.push(EndRendering); let consumer = graph.push(AllowedAccess(vec![( shared.id(), SubresourceScope { stage: vk::PipelineStageFlags2::COMPUTE_SHADER, access: vk::AccessFlags2::SHADER_STORAGE_WRITE, subresource: SubresourceScopeInner::Buffer(range), }, )])); let compiled = graph.compile().expect("graph compiles"); assert!(compiled.happens_before(producer, interior_a)); assert!(compiled.happens_before(interior_a, consumer)); assert!(!compiled.happens_before(interior_a, interior_b)); } #[test] fn draw_vertex_index_and_indirect_reads_follow_copies_and_derive_usage() { let mut graph = RenderGraph::new(); let source = graph.create_buffer(GraphBufferDesc { size: 64, ..Default::default() }); let vertex = graph.create_buffer(GraphBufferDesc { size: 64, ..Default::default() }); let index = graph.create_buffer(GraphBufferDesc { size: 64, ..Default::default() }); let indirect = graph.create_buffer(GraphBufferDesc { size: 64, ..Default::default() }); let range = BufferRange { offset: 0, size: 64, }; for dst in [vertex, index, indirect] { graph.push(CopyBuffer { src: source, src_range: range, dst, dst_range: range, }); } graph.push(BeginRendering { color: Vec::new(), depth: None, }); graph.push(Draw { args: DrawArgs::Indexed { index_count: 3, instance_count: 1, first_index: 0, vertex_offset: 0, first_instance: 0, }, pipeline_layout: vk::PipelineLayout::null(), vertices: vec![VertexBinding { binding: 0, slice: BufferSlice { buffer: vertex, range, }, }], index: Some(IndexBinding { slice: BufferSlice { buffer: index, range, }, format: IndexFormat::Uint16, }), indirect: Some(IndirectBinding { slice: BufferSlice { buffer: indirect, range, }, draw_count: 1, stride: 20, }), descriptor_sets: Vec::new(), push_constants: Vec::new(), }); graph.push(EndRendering); let accesses = graph.commands.side_effects.command_accesses(); assert_eq!( derive_buffer_usage( &accesses_for_resource(vertex.id(), accesses, &graph.commands.side_effects), vk::BufferUsageFlags::empty() ), vk::BufferUsageFlags::TRANSFER_DST | vk::BufferUsageFlags::VERTEX_BUFFER ); assert_eq!( derive_buffer_usage( &accesses_for_resource(index.id(), accesses, &graph.commands.side_effects), vk::BufferUsageFlags::empty() ), vk::BufferUsageFlags::TRANSFER_DST | vk::BufferUsageFlags::INDEX_BUFFER ); assert_eq!( derive_buffer_usage( &accesses_for_resource(indirect.id(), accesses, &graph.commands.side_effects), vk::BufferUsageFlags::empty() ), vk::BufferUsageFlags::TRANSFER_DST | vk::BufferUsageFlags::INDIRECT_BUFFER ); let compiled = graph.compile().expect("graph compiles"); let batches = compiled.batches().collect::>(); assert_eq!(batches.len(), 2); assert_eq!( batches[0], [CommandIndex(0), CommandIndex(1), CommandIndex(2)] ); assert_eq!( batches[1], [CommandIndex(3), CommandIndex(4), CommandIndex(5)] ); let draw_barrier = compiled .barriers() .iter() .find(|barrier| barrier.before_command == CommandIndex(3)) .expect("draw input barriers are generated"); assert_eq!(draw_barrier.resources.len(), 3); assert!( draw_barrier .resources .iter() .any(|resource| resource.id == vertex.id()) ); assert!( draw_barrier .resources .iter() .any(|resource| resource.id == index.id()) ); assert!( draw_barrier .resources .iter() .any(|resource| resource.id == indirect.id()) ); } #[test] fn descriptor_set_declarations_create_buffer_and_texture_accesses() { let mut graph = RenderGraph::new(); let source = graph.create_buffer(GraphBufferDesc { size: 32, ..Default::default() }); let uniform = graph.create_buffer(GraphBufferDesc { size: 32, ..Default::default() }); let texture = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, ..Default::default() }); let buffer_range = BufferRange { offset: 0, size: 32, }; graph.push(CopyBuffer { src: source, src_range: buffer_range, dst: uniform, dst_range: buffer_range, }); graph.push(ClearTexture { texture, range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), layout: vk::ImageLayout::TRANSFER_DST_OPTIMAL, color: [0.0; 4], }); let set = graph.track_descriptor_set( vk::DescriptorSet::null(), vec![ DescriptorResourceAccess::Buffer { buffer: uniform, range: buffer_range, stage: vk::PipelineStageFlags2::VERTEX_SHADER, access: vk::AccessFlags2::UNIFORM_READ, }, DescriptorResourceAccess::Texture { texture, range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), layout: vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, access: vk::AccessFlags2::SHADER_SAMPLED_READ, }, ], ); graph.push(BeginRendering { color: Vec::new(), depth: None, }); graph.push(Draw { args: DrawArgs::Direct { vertex_count: 3, instance_count: 1, first_vertex: 0, first_instance: 0, }, pipeline_layout: vk::PipelineLayout::null(), vertices: Vec::new(), index: None, indirect: None, descriptor_sets: vec![set], push_constants: Vec::new(), }); graph.push(EndRendering); let accesses = graph.commands.side_effects.command_accesses(); assert!( derive_buffer_usage( &accesses_for_resource(uniform.id(), accesses, &graph.commands.side_effects), vk::BufferUsageFlags::empty() ) .contains(vk::BufferUsageFlags::UNIFORM_BUFFER) ); assert!( derive_image_usage( &accesses_for_resource(texture.id(), accesses, &graph.commands.side_effects), vk::ImageUsageFlags::empty() ) .contains(vk::ImageUsageFlags::SAMPLED) ); let compiled = graph.compile().expect("graph compiles"); let draw_barrier = compiled .barriers() .iter() .find(|barrier| barrier.before_command == CommandIndex(2)) .expect("descriptor accesses produce barriers"); assert_eq!(draw_barrier.resources.len(), 2); let texture_barrier = draw_barrier .resources .iter() .find(|resource| resource.id == texture.id()) .expect("sampled texture barrier is present"); assert_eq!( texture_barrier.new_layout, Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) ); assert_eq!( texture_barrier.dst_access, vk::AccessFlags2::SHADER_SAMPLED_READ ); assert!(compiled.resource_lifetimes().contains_key(&uniform.id())); assert!(compiled.resource_lifetimes().contains_key(&texture.id())); assert!(!compiled.resource_lifetimes()[&uniform.id()].imported); assert!(!compiled.resource_lifetimes()[&texture.id()].imported); assert!( compiled .interference() .interferes(uniform.id(), texture.id()) ); } #[test] fn descriptor_import_helpers_validate_effective_ranges_and_dynamic_order() { assert_eq!( effective_buffer_range(128, 16, 32, 64), Ok(BufferRange { offset: 80, size: 32 }) ); assert!(effective_buffer_range(128, 96, 32, 16).is_err()); assert!(effective_buffer_range(128, u64::MAX, 16, 1).is_err()); assert!(effective_buffer_range(128, 16, vk::WHOLE_SIZE, 0).is_ok()); assert!(effective_buffer_range(128, 16, vk::WHOLE_SIZE, 4).is_err()); assert!(is_dynamic_offset_aligned(64, 32)); assert!(!is_dynamic_offset_aligned(16, 32)); assert!(!is_dynamic_offset_aligned(0, 0)); let layout = [ renderer_resources::pipeline::DescriptorSetLayoutBindingDesc { binding: 9, count: 2, kind: vk::DescriptorType::STORAGE_BUFFER_DYNAMIC, ..Default::default() }, renderer_resources::pipeline::DescriptorSetLayoutBindingDesc { binding: 2, count: 1, kind: vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC, ..Default::default() }, ]; assert_eq!( dynamic_offset_slots(&layout), [ (2, 0, vk::DescriptorType::UNIFORM_BUFFER_DYNAMIC), (9, 0, vk::DescriptorType::STORAGE_BUFFER_DYNAMIC), (9, 1, vk::DescriptorType::STORAGE_BUFFER_DYNAMIC), ] ); assert_eq!( descriptor_access(vk::DescriptorType::UNIFORM_BUFFER), Some((vk::AccessFlags2::UNIFORM_READ, None)) ); assert_eq!( descriptor_access(vk::DescriptorType::SAMPLED_IMAGE), Some((vk::AccessFlags2::SHADER_SAMPLED_READ, None)) ); assert_eq!( descriptor_access(vk::DescriptorType::STORAGE_IMAGE), Some(( vk::AccessFlags2::SHADER_STORAGE_READ | vk::AccessFlags2::SHADER_STORAGE_WRITE, Some(vk::ImageLayout::GENERAL), )) ); assert_eq!(descriptor_access(vk::DescriptorType::SAMPLER), None); assert_eq!( shader_stages_to_pipeline( vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::FRAGMENT ), Some(vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER) ); assert_eq!( shader_stages_to_pipeline(vk::ShaderStageFlags::ALL), Some(vk::PipelineStageFlags2::ALL_COMMANDS) ); assert!(texture_layouts_conflict( texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), vk::ImageLayout::GENERAL, texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, )); assert!(!texture_layouts_conflict( texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), vk::ImageLayout::GENERAL, texture_range(vk::ImageAspectFlags::COLOR, 1, 1, 0, 1), vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, )); } #[test] fn raw_and_managed_descriptor_binding_offsets_are_concatenated_by_set_order() { let first = GraphDescriptorSet { raw: vk::DescriptorSet::null(), accesses: Vec::new(), managed: None, dynamic_offsets: vec![4, 8], }; let second = GraphDescriptorSet { raw: vk::DescriptorSet::null(), accesses: Vec::new(), managed: None, dynamic_offsets: vec![12], }; let combined = [first, second] .iter() .flat_map(|set| set.dynamic_offsets().iter().copied()) .collect::>(); assert_eq!(combined, [4, 8, 12]); } #[test] fn write_then_read_creates_order_and_barrier() { let mut graph = RenderGraph::new(); let a = graph.create_buffer(GraphBufferDesc { size: 32, ..Default::default() }); let b = graph.create_buffer(GraphBufferDesc { size: 32, ..Default::default() }); let c = graph.create_buffer(GraphBufferDesc { size: 32, ..Default::default() }); graph.push(CopyBuffer { src: a, src_range: BufferRange { offset: 0, size: 32, }, dst: b, dst_range: BufferRange { offset: 0, size: 32, }, }); graph.push(CopyBuffer { src: b, src_range: BufferRange { offset: 0, size: 32, }, dst: c, dst_range: BufferRange { offset: 0, size: 32, }, }); let compiled = graph.compile().expect("graph compiles"); assert_eq!( compiled.batches().flatten().copied().collect::>(), vec![CommandIndex(0), CommandIndex(1)] ); assert_eq!(compiled.barriers().len(), 1); assert_eq!(compiled.barriers()[0].before_command, CommandIndex(1)); assert_eq!(compiled.barriers()[0].resources[0].id, b.id()); } #[test] fn write_read_write_emits_barriers_from_each_most_recent_conflict() { let resource = ResourceId::from_u32(0); let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { size: 64, ..Default::default() })]; let accesses = vec![ vec![scoped_access( resource, 0, 64, vk::PipelineStageFlags2::COPY, vk::AccessFlags2::TRANSFER_WRITE, )], vec![scoped_access( resource, 0, 64, vk::PipelineStageFlags2::VERTEX_SHADER, vk::AccessFlags2::SHADER_READ, )], vec![scoped_access( resource, 0, 64, vk::PipelineStageFlags2::COMPUTE_SHADER, vk::AccessFlags2::SHADER_WRITE, )], ]; let compiler = GraphCompiler::new( &resources, &accesses, accesses.len(), &SideEffectMap::default(), &[], ); let batches = topo_batches_for_test(&compiler).expect("graph is acyclic"); let execution_order = flatten_batches(&batches); let barriers = compiler.build_barriers(&execution_order); assert_eq!(barriers.len(), 2); assert_eq!(barriers[0].before_command, CommandIndex(1)); assert_eq!( barriers[0].resources[0].src_stage, vk::PipelineStageFlags2::COPY ); assert_eq!( barriers[0].resources[0].src_access, vk::AccessFlags2::TRANSFER_WRITE ); assert_eq!( barriers[0].resources[0].dst_stage, vk::PipelineStageFlags2::VERTEX_SHADER ); assert_eq!( barriers[0].resources[0].dst_access, vk::AccessFlags2::SHADER_READ ); let second_write_barrier = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(2)) .expect("second write has a barrier"); assert_eq!( second_write_barrier.resources[0].src_stage, vk::PipelineStageFlags2::VERTEX_SHADER ); assert_eq!( second_write_barrier.resources[0].src_access, vk::AccessFlags2::SHADER_READ ); } #[test] fn alias_buffer_barrier_aggregates_terminal_write_and_first_use_scopes() { let old = ResourceId::from_u32(0); let new = ResourceId::from_u32(1); let resources = [ ResourceDesc::CreatedBuffer(GraphBufferDesc { size: 64, ..Default::default() }), ResourceDesc::CreatedBuffer(GraphBufferDesc { size: 64, ..Default::default() }), ]; let accesses = vec![ vec![ scoped_access( old, 0, 64, vk::PipelineStageFlags2::COPY, vk::AccessFlags2::TRANSFER_WRITE, ), scoped_access( old, 0, 64, vk::PipelineStageFlags2::COMPUTE_SHADER, vk::AccessFlags2::SHADER_STORAGE_WRITE, ), ], vec![scoped_access( new, 0, 64, vk::PipelineStageFlags2::FRAGMENT_SHADER, vk::AccessFlags2::UNIFORM_READ, )], ]; let compiler = GraphCompiler::new( &resources, &accesses, accesses.len(), &SideEffectMap::default(), &[], ); let batches = vec![vec![CommandIndex(0)], vec![CommandIndex(1)]]; let happens_before = HappensBefore::from_command_dag(&[vec![1], vec![]], &batches); let uses = compiler.resource_uses(); let frontiers = ResourceFrontiers::from_uses(&uses, &happens_before); let lifetimes = HashMap::from([ (old, placed_lifetime(0, 64, false)), (new, placed_lifetime(0, 64, true)), ]); let mut barriers = compiler.build_barriers(&flatten_batches(&batches)); compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); let barrier = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(1)) .unwrap(); assert!(barrier.memory_barriers.is_empty()); let alias = barrier .resources .iter() .find(|resource| resource.id == new) .unwrap(); assert_eq!( alias.src_stage, vk::PipelineStageFlags2::COPY | vk::PipelineStageFlags2::COMPUTE_SHADER ); assert_eq!( alias.src_access, vk::AccessFlags2::TRANSFER_WRITE | vk::AccessFlags2::SHADER_STORAGE_WRITE ); assert_eq!(alias.dst_stage, vk::PipelineStageFlags2::FRAGMENT_SHADER); assert_eq!(alias.dst_access, vk::AccessFlags2::UNIFORM_READ); assert!(matches!( alias.range, Subresource::Buffer(BufferRange { offset: 0, size: vk::WHOLE_SIZE, }) )); assert_eq!(alias.old_layout, None); assert_eq!(alias.new_layout, None); assert_eq!(alias.queue_transfer, None); assert_eq!(alias.origin, BarrierOrigin::Initial); // Existing first-use resource dependencies retain their destination/range/origin, // while alias source masks are added to their own source masks. let existing_resource = BarrierResource { id: new, range: Subresource::Buffer(BufferRange { offset: 8, size: 16, }), origin: BarrierOrigin::Command(CommandIndex(0)), src_stage: vk::PipelineStageFlags2::VERTEX_SHADER, src_access: vk::AccessFlags2::SHADER_READ, dst_stage: vk::PipelineStageFlags2::COPY, dst_access: vk::AccessFlags2::TRANSFER_WRITE, old_layout: None, new_layout: None, queue_transfer: None, }; let mut barriers = vec![PipelineBarrier { before_command: CommandIndex(1), memory_barriers: Vec::new(), resources: tinyvec::tiny_vec!([BarrierResource; 2] => existing_resource), }]; compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); let alias = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(1)) .unwrap() .resources .iter() .find(|resource| resource.id == new) .unwrap(); assert_eq!( alias.src_stage, vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::COPY | vk::PipelineStageFlags2::COMPUTE_SHADER ); assert_eq!( alias.src_access, vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::TRANSFER_WRITE | vk::AccessFlags2::SHADER_STORAGE_WRITE ); assert_eq!(alias.dst_stage, vk::PipelineStageFlags2::COPY); assert_eq!(alias.dst_access, vk::AccessFlags2::TRANSFER_WRITE); assert!(matches!( alias.range, Subresource::Buffer(BufferRange { offset: 8, size: 16, }) )); assert_eq!(alias.origin, BarrierOrigin::Command(CommandIndex(0))); } #[test] fn alias_buffer_read_only_source_uses_execution_only_scoped_barrier() { let old = ResourceId::from_u32(0); let new = ResourceId::from_u32(1); let resources = [ ResourceDesc::CreatedBuffer(GraphBufferDesc::default()), ResourceDesc::CreatedBuffer(GraphBufferDesc::default()), ]; let accesses = vec![ vec![scoped_access( old, 0, 8, vk::PipelineStageFlags2::VERTEX_SHADER, vk::AccessFlags2::SHADER_READ, )], vec![scoped_access( new, 0, 8, vk::PipelineStageFlags2::COPY, vk::AccessFlags2::TRANSFER_WRITE, )], ]; let compiler = GraphCompiler::new( &resources, &accesses, accesses.len(), &SideEffectMap::default(), &[], ); let batches = vec![vec![CommandIndex(0)], vec![CommandIndex(1)]]; let happens_before = HappensBefore::from_command_dag(&[vec![1], vec![]], &batches); let uses = compiler.resource_uses(); let frontiers = ResourceFrontiers::from_uses(&uses, &happens_before); let lifetimes = HashMap::from([ (old, placed_lifetime(0, 8, false)), (new, placed_lifetime(0, 8, true)), ]); let mut barriers = compiler.build_barriers(&flatten_batches(&batches)); compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); let barrier = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(1)) .unwrap(); assert!(barrier.memory_barriers.is_empty()); let alias = barrier .resources .iter() .find(|resource| resource.id == new) .unwrap(); assert_eq!(alias.src_stage, vk::PipelineStageFlags2::VERTEX_SHADER); assert_eq!(alias.src_access, vk::AccessFlags2::NONE); assert_eq!(alias.dst_stage, vk::PipelineStageFlags2::COPY); assert_eq!(alias.dst_access, vk::AccessFlags2::TRANSFER_WRITE); } #[test] fn alias_image_scope_is_folded_into_undefined_layout_transition() { let old = ResourceId::from_u32(0); let new = ResourceId::from_u32(1); let resources = [ ResourceDesc::CreatedTexture(GraphTextureDesc::default()), ResourceDesc::CreatedTexture(GraphTextureDesc::default()), ]; let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); let accesses = vec![ vec![texture_scoped_access( old, range, vk::ImageLayout::GENERAL, vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT, vk::AccessFlags2::COLOR_ATTACHMENT_WRITE, )], vec![texture_scoped_access( new, range, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, vk::PipelineStageFlags2::FRAGMENT_SHADER, vk::AccessFlags2::SHADER_SAMPLED_READ, )], ]; let compiler = GraphCompiler::new( &resources, &accesses, accesses.len(), &SideEffectMap::default(), &[], ); let batches = vec![vec![CommandIndex(0)], vec![CommandIndex(1)]]; let happens_before = HappensBefore::from_command_dag(&[vec![1], vec![]], &batches); let uses = compiler.resource_uses(); let frontiers = ResourceFrontiers::from_uses(&uses, &happens_before); let lifetimes = HashMap::from([ (old, placed_lifetime(0, 128, false)), (new, placed_lifetime(0, 128, true)), ]); let mut barriers = compiler.build_barriers(&flatten_batches(&batches)); compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); let barrier = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(1)) .unwrap(); assert!(barrier.memory_barriers.is_empty()); let image = barrier .resources .iter() .find(|resource| resource.id == new) .unwrap(); assert_eq!(image.old_layout, Some(vk::ImageLayout::UNDEFINED)); assert_eq!(image.origin, BarrierOrigin::Initial); assert_eq!( image.src_stage, vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT ); assert_eq!(image.src_access, vk::AccessFlags2::COLOR_ATTACHMENT_WRITE); assert_eq!(image.dst_stage, vk::PipelineStageFlags2::FRAGMENT_SHADER); } #[test] fn alias_range_barriers_include_every_intersecting_occupant() { let old_ids = [ ResourceId::from_u32(0), ResourceId::from_u32(1), ResourceId::from_u32(2), ResourceId::from_u32(3), ]; let new = ResourceId::from_u32(4); let resources = (0..5) .map(|_| ResourceDesc::CreatedBuffer(GraphBufferDesc::default())) .collect::>(); let old_stages = [ vk::PipelineStageFlags2::VERTEX_SHADER, vk::PipelineStageFlags2::FRAGMENT_SHADER, vk::PipelineStageFlags2::COMPUTE_SHADER, vk::PipelineStageFlags2::COPY, ]; let mut accesses = old_ids .iter() .zip(old_stages) .map(|(&id, stage)| { vec![scoped_access( id, 0, 8, stage, vk::AccessFlags2::TRANSFER_WRITE, )] }) .collect::>(); accesses.push(vec![scoped_access( new, 0, 8, vk::PipelineStageFlags2::COMPUTE_SHADER, vk::AccessFlags2::SHADER_WRITE, )]); let compiler = GraphCompiler::new( &resources, &accesses, accesses.len(), &SideEffectMap::default(), &[], ); let batches = (0..5) .map(|index| vec![CommandIndex(index)]) .collect::>(); let edges = vec![vec![4], vec![4], vec![4], vec![4], vec![]]; let happens_before = HappensBefore::from_command_dag(&edges, &batches); let uses = compiler.resource_uses(); let frontiers = ResourceFrontiers::from_uses(&uses, &happens_before); let lifetimes = HashMap::from([ (old_ids[0], placed_lifetime(0, 8, false)), (old_ids[1], placed_lifetime(4, 8, false)), (old_ids[2], placed_lifetime(8, 8, false)), (old_ids[3], placed_lifetime(32, 8, false)), (new, placed_lifetime(8, 16, true)), ]); let mut barriers = compiler.build_barriers(&flatten_batches(&batches)); compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); let barrier = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(4)) .unwrap(); // Source masks are unioned across all intersecting occupants into one barrier. assert!(barrier.memory_barriers.is_empty()); let alias = barrier .resources .iter() .find(|resource| resource.id == new) .unwrap(); assert_eq!( alias.src_stage, vk::PipelineStageFlags2::FRAGMENT_SHADER | vk::PipelineStageFlags2::COMPUTE_SHADER ); assert_eq!(alias.src_access, vk::AccessFlags2::TRANSFER_WRITE); assert_eq!(alias.dst_stage, vk::PipelineStageFlags2::COMPUTE_SHADER); assert_eq!(alias.dst_access, vk::AccessFlags2::SHADER_WRITE); assert!(matches!( alias.range, Subresource::Buffer(BufferRange { offset: 0, size: vk::WHOLE_SIZE, }) )); } fn placed_lifetime(offset: u64, size: u64, aliased: bool) -> ResourceLifetime { ResourceLifetime { first_batch: 0, last_batch: 0, placement: Placement { offset, size, alignment: 1, pool: Some(0), aliased, }, imported: false, } } #[test] fn imported_texture_read_with_different_layout_gets_initial_transition() { let resource = ResourceId::from_u32(0); let resources = [ResourceDesc::CreatedTexture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, mip_levels: 2, array_layers: 2, ..Default::default() })]; let initial_states = [crate::resources::ResourceState { stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, access: vk::AccessFlags2::SHADER_READ, layout: Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL), }]; // Supply the imported image's initial state directly so this stays // device-free and does not need to construct a Vulkan Image wrapper. let accesses = vec![ vec![texture_scoped_access( resource, texture_range(vk::ImageAspectFlags::COLOR, 0, 2, 0, 2), vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, vk::PipelineStageFlags2::VERTEX_SHADER, vk::AccessFlags2::SHADER_READ, )], vec![texture_scoped_access( resource, texture_range(vk::ImageAspectFlags::COLOR, 1, 1, 1, 1), vk::ImageLayout::GENERAL, vk::PipelineStageFlags2::COMPUTE_SHADER, vk::AccessFlags2::SHADER_SAMPLED_READ, )], ]; let compiler = GraphCompiler::new( &resources, &accesses, accesses.len(), &SideEffectMap::default(), &[], ); let batches = topo_batches_for_test(&compiler).expect("graph is acyclic"); let barriers = compiler .build_barriers_with_initial_states(&flatten_batches(&batches), &initial_states); assert_eq!(barriers.len(), 1); assert_eq!(barriers[0].before_command, CommandIndex(1)); assert_eq!( barriers[0].resources[0].old_layout, Some(initial_states[0].layout.unwrap()) ); assert_eq!( barriers[0].resources[0].new_layout, Some(vk::ImageLayout::GENERAL) ); assert_eq!( barriers[0].resources[0].dst_access, vk::AccessFlags2::SHADER_SAMPLED_READ ); } #[test] fn overlapping_texture_reads_with_different_layouts_get_layout_transition() { let resource = ResourceId::from_u32(0); let resources = [ResourceDesc::CreatedTexture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, ..Default::default() })]; let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); let accesses = vec![ vec![texture_scoped_access( resource, range, vk::ImageLayout::GENERAL, vk::PipelineStageFlags2::COPY, vk::AccessFlags2::TRANSFER_WRITE, )], vec![texture_scoped_access( resource, range, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, vk::PipelineStageFlags2::VERTEX_SHADER, vk::AccessFlags2::SHADER_READ, )], vec![texture_scoped_access( resource, range, vk::ImageLayout::GENERAL, vk::PipelineStageFlags2::FRAGMENT_SHADER, vk::AccessFlags2::SHADER_SAMPLED_READ, )], ]; let compiler = GraphCompiler::new( &resources, &accesses, accesses.len(), &SideEffectMap::default(), &[], ); let batches = topo_batches_for_test(&compiler).expect("graph is acyclic"); let barriers = compiler.build_barriers(&flatten_batches(&batches)); assert_eq!(barriers.len(), 3); let current_read_barrier = &barriers[2]; assert_eq!(current_read_barrier.before_command, CommandIndex(2)); let barrier = ¤t_read_barrier.resources[0]; assert_eq!(barrier.dst_stage, vk::PipelineStageFlags2::FRAGMENT_SHADER); assert_eq!(barrier.dst_access, vk::AccessFlags2::SHADER_SAMPLED_READ); assert_eq!( barrier.old_layout, Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) ); assert_eq!(barrier.new_layout, Some(vk::ImageLayout::GENERAL)); } #[test] fn same_target_readers_chain_the_current_image_layout() { let resource = ResourceId::from_u32(0); let resources = [ResourceDesc::CreatedTexture(GraphTextureDesc::default())]; let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); let accesses = vec![ vec![texture_scoped_access( resource, range, vk::ImageLayout::GENERAL, vk::PipelineStageFlags2::COPY, vk::AccessFlags2::TRANSFER_WRITE, )], vec![texture_scoped_access( resource, range, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, vk::PipelineStageFlags2::VERTEX_SHADER, vk::AccessFlags2::SHADER_READ, )], vec![texture_scoped_access( resource, range, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, vk::PipelineStageFlags2::FRAGMENT_SHADER, vk::AccessFlags2::SHADER_READ, )], ]; let compiler = GraphCompiler::new( &resources, &accesses, accesses.len(), &SideEffectMap::default(), &[], ); let order = [CommandIndex(0), CommandIndex(1), CommandIndex(2)]; let barriers = compiler.build_barriers(&order); let second_reader = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(2)) .expect("the producer conflicts with the second reader"); assert_eq!(second_reader.resources.len(), 1); assert_eq!( second_reader.resources[0].old_layout, Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) ); assert_eq!( second_reader.resources[0].new_layout, Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) ); assert_eq!( second_reader.resources[0].origin, BarrierOrigin::Command(CommandIndex(0)) ); } #[test] fn hoisted_equal_image_transitions_union_destination_scopes_in_order() { let id = ResourceId::from_u32(0); let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); let mut barriers = vec![ PipelineBarrier { before_command: CommandIndex(2), memory_barriers: Vec::new(), resources: [BarrierResource { id, range: Subresource::Texture(range), origin: BarrierOrigin::Command(CommandIndex(0)), src_stage: vk::PipelineStageFlags2::TRANSFER, src_access: vk::AccessFlags2::TRANSFER_WRITE, dst_stage: vk::PipelineStageFlags2::VERTEX_SHADER, dst_access: vk::AccessFlags2::SHADER_READ, old_layout: Some(vk::ImageLayout::GENERAL), new_layout: Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL), queue_transfer: None, }] .into_iter() .collect(), }, PipelineBarrier { before_command: CommandIndex(3), memory_barriers: Vec::new(), resources: [BarrierResource { id, range: Subresource::Texture(range), origin: BarrierOrigin::Command(CommandIndex(0)), src_stage: vk::PipelineStageFlags2::TRANSFER, src_access: vk::AccessFlags2::TRANSFER_WRITE, dst_stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, dst_access: vk::AccessFlags2::SHADER_SAMPLED_READ, old_layout: Some(vk::ImageLayout::GENERAL), new_layout: Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL), queue_transfer: None, }] .into_iter() .collect(), }, ]; let spans = [RenderPassSpan { begin: CommandIndex(1), end: CommandIndex(4), }]; hoist_span_barriers(&mut barriers, &spans, 5); let merged = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(1)) .unwrap(); assert_eq!(merged.resources.len(), 1); assert_eq!( merged.resources[0].dst_stage, vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER ); assert_eq!( merged.resources[0].dst_access, vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_SAMPLED_READ ); } #[test] fn overlapping_pure_reads_without_layout_change_need_no_barrier() { let resource = ResourceId::from_u32(0); let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { size: 64, ..Default::default() })]; let accesses = vec![ vec![scoped_access( resource, 0, 64, vk::PipelineStageFlags2::VERTEX_SHADER, vk::AccessFlags2::SHADER_READ, )], vec![scoped_access( resource, 16, 16, vk::PipelineStageFlags2::FRAGMENT_SHADER, vk::AccessFlags2::UNIFORM_READ, )], ]; let compiler = GraphCompiler::new( &resources, &accesses, accesses.len(), &SideEffectMap::default(), &[], ); let batches = topo_batches_for_test(&compiler).expect("graph is acyclic"); assert!( compiler .build_barriers(&flatten_batches(&batches)) .is_empty() ); } #[test] fn write_before_contained_read_emits_barrier_before_read() { let resource = ResourceId::from_u32(0); let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { size: 64, ..Default::default() })]; let accesses = vec![ vec![scoped_access( resource, 0, 64, vk::PipelineStageFlags2::COPY, vk::AccessFlags2::TRANSFER_WRITE, )], vec![scoped_access( resource, 16, 16, vk::PipelineStageFlags2::FRAGMENT_SHADER, vk::AccessFlags2::UNIFORM_READ, )], ]; let compiler = GraphCompiler::new( &resources, &accesses, accesses.len(), &SideEffectMap::default(), &[], ); let batches = topo_batches_for_test(&compiler).expect("graph is acyclic"); let barriers = compiler.build_barriers(&flatten_batches(&batches)); assert_eq!(batches, [vec![CommandIndex(0)], vec![CommandIndex(1)]]); assert_eq!(barriers.len(), 1); assert_eq!(barriers[0].before_command, CommandIndex(1)); assert_eq!( barriers[0].resources[0].src_access, vk::AccessFlags2::TRANSFER_WRITE ); assert_eq!( barriers[0].resources[0].dst_stage, vk::PipelineStageFlags2::FRAGMENT_SHADER ); assert_eq!( barriers[0].resources[0].dst_access, vk::AccessFlags2::UNIFORM_READ ); } #[test] fn independent_commands_share_a_batch_and_dependencies_advance_to_later_batches() { let mut graph = RenderGraph::new(); let a = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let shared = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let b = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let independent_src = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let independent_dst = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); graph.push(CopyBuffer { src: a, src_range: BufferRange { offset: 0, size: 8 }, dst: shared, dst_range: BufferRange { offset: 0, size: 8 }, }); graph.push(CopyBuffer { src: shared, src_range: BufferRange { offset: 0, size: 8 }, dst: b, dst_range: BufferRange { offset: 0, size: 8 }, }); graph.push(CopyBuffer { src: independent_src, src_range: BufferRange { offset: 0, size: 8 }, dst: independent_dst, dst_range: BufferRange { offset: 0, size: 8 }, }); let compiled = graph.compile().expect("graph compiles"); assert_eq!( compiled.batches().collect::>(), [ &[CommandIndex(0), CommandIndex(2)][..], &[CommandIndex(1)][..] ] ); assert_eq!( compiled.batches().flatten().copied().collect::>(), vec![CommandIndex(0), CommandIndex(2), CommandIndex(1)] ); assert_eq!(compiled.barriers().len(), 1); assert_eq!(compiled.barriers()[0].before_command, CommandIndex(1)); let batch_for = |command| { compiled .batches() .position(|batch| batch.contains(&command)) .expect("command appears in a batch") }; for barrier in compiled.barriers() { assert!( batch_for(CommandIndex(0)) < batch_for(barrier.before_command), "barrier target follows its conflicting source in another batch" ); } } #[test] fn recording_steps_place_barriers_immediately_before_their_target_command() { let first_barrier = PipelineBarrier { before_command: CommandIndex(1), memory_barriers: Vec::new(), resources: tinyvec::TinyVec::new(), }; let second_barrier = PipelineBarrier { before_command: CommandIndex(1), memory_barriers: Vec::new(), resources: tinyvec::TinyVec::new(), }; let barriers = [first_barrier, second_barrier]; let batches = [vec![CommandIndex(0), CommandIndex(1)]]; let execution_order = flatten_batches(&batches); let steps = recording_steps(&execution_order, &barriers).collect::>(); assert!(matches!(steps[0], RecordingStep::Command(CommandIndex(0)))); assert!( matches!(steps[1], RecordingStep::Barrier(barrier) if std::ptr::eq(barrier, &barriers[0])) ); assert!( matches!(steps[2], RecordingStep::Barrier(barrier) if std::ptr::eq(barrier, &barriers[1])) ); assert!(matches!(steps[3], RecordingStep::Command(CommandIndex(1)))); } #[test] fn barrier_for_later_command_follows_independent_command_in_same_batch() { let mut graph = RenderGraph::new(); let first_src = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let first_shared = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let first_dst = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let second_src = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let second_shared = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let second_dst = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let copy = |src, dst| CopyBuffer { src, src_range: BufferRange { offset: 0, size: 8 }, dst, dst_range: BufferRange { offset: 0, size: 8 }, }; graph.push(copy(first_src, first_shared)); graph.push(copy(second_src, second_shared)); graph.push(copy(second_shared, second_dst)); graph.push(copy(first_shared, first_dst)); let compiled = graph.compile().expect("graph compiles"); assert_eq!( compiled.batches().collect::>(), [ &[CommandIndex(0), CommandIndex(1)][..], &[CommandIndex(2), CommandIndex(3)][..], ] ); let steps = recording_steps(compiled.execution_order(), compiled.barriers()).collect::>(); let command_two_position = steps .iter() .position(|step| matches!(step, RecordingStep::Command(CommandIndex(2)))) .expect("independent command is recorded"); let barrier_three_position = steps .iter() .position(|step| { matches!(step, RecordingStep::Barrier(barrier) if barrier.before_command == CommandIndex(3)) }) .expect("barrier for command three is recorded"); assert!(command_two_position < barrier_three_position); } #[test] fn dependency_chain_occupies_three_batches() { let mut graph = RenderGraph::new(); let a = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let b = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let c = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); let d = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); graph.push(CopyBuffer { src: a, src_range: BufferRange { offset: 0, size: 8 }, dst: b, dst_range: BufferRange { offset: 0, size: 8 }, }); graph.push(CopyBuffer { src: b, src_range: BufferRange { offset: 0, size: 8 }, dst: c, dst_range: BufferRange { offset: 0, size: 8 }, }); graph.push(CopyBuffer { src: c, src_range: BufferRange { offset: 0, size: 8 }, dst: d, dst_range: BufferRange { offset: 0, size: 8 }, }); let compiled = graph.compile().expect("graph compiles"); assert_eq!( compiled.batches().collect::>(), [ &[CommandIndex(0)][..], &[CommandIndex(1)][..], &[CommandIndex(2)][..], ] ); assert!(compiled.happens_before(CommandIndex(0), CommandIndex(2))); assert!(compiled.resources_can_alias(a.id(), d.id())); for barrier in compiled.barriers() { let source_command = CommandIndex(barrier.before_command.index() - 1); let source_batch = compiled .batches() .position(|batch| batch.contains(&source_command)) .expect("conflicting source appears in a batch"); let target_batch = compiled .batches() .position(|batch| batch.contains(&barrier.before_command)) .expect("barrier target appears in a batch"); assert!(source_batch < target_batch); } } #[test] fn texture_lifetime_size_waits_for_device_requirements() { let mut graph = RenderGraph::new(); let texture = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, extent: vk::Extent3D { width: 4, height: 4, depth: 1, }, ..Default::default() }); let buffer = graph.create_buffer(GraphBufferDesc { size: 8, ..Default::default() }); graph.push(ClearTexture { texture, range: vk::ImageSubresourceRange { aspect_mask: vk::ImageAspectFlags::COLOR, base_mip_level: 0, level_count: 1, base_array_layer: 0, layer_count: 1, }, layout: vk::ImageLayout::TRANSFER_DST_OPTIMAL, color: [0.0; 4], }); // Keep the buffer live in the same command, so its slot follows the image. graph.commands.side_effects.insert( CommandIndex(0), buffer.id(), SubresourceScope { stage: vk::PipelineStageFlags2::CLEAR, access: vk::AccessFlags2::TRANSFER_WRITE, subresource: SubresourceScopeInner::Buffer(BufferRange { offset: 0, size: 8 }), }, ); let compiled = graph.compile().expect("graph compiles"); let texture_placement = compiled.resource_lifetimes()[&texture.id()].placement; let buffer_placement = compiled.resource_lifetimes()[&buffer.id()].placement; assert_eq!(texture_placement.size, 0); assert_eq!(buffer_placement.size, 0); assert_eq!(texture_placement.offset, 0); assert_eq!(buffer_placement.offset, 0); } #[test] fn copy_buffer_derives_transfer_usages_for_source_and_destination() { let mut graph = RenderGraph::new(); let src = graph.create_buffer(GraphBufferDesc { size: 64, ..Default::default() }); let dst = graph.create_buffer(GraphBufferDesc { size: 64, ..Default::default() }); graph.push(CopyBuffer { src, src_range: BufferRange { offset: 0, size: 64, }, dst, dst_range: BufferRange { offset: 0, size: 64, }, }); graph.push(CopyBuffer { src: dst, src_range: BufferRange { offset: 0, size: 64, }, dst: src, dst_range: BufferRange { offset: 0, size: 64, }, }); let accesses = graph.commands.side_effects.command_accesses(); let src_accesses = accesses_for_resource(src.id(), accesses, &graph.commands.side_effects); let dst_accesses = accesses_for_resource(dst.id(), accesses, &graph.commands.side_effects); assert_eq!( derive_buffer_usage(&src_accesses, vk::BufferUsageFlags::empty()), vk::BufferUsageFlags::TRANSFER_SRC | vk::BufferUsageFlags::TRANSFER_DST ); assert_eq!( derive_buffer_usage(&dst_accesses, vk::BufferUsageFlags::empty()), vk::BufferUsageFlags::TRANSFER_SRC | vk::BufferUsageFlags::TRANSFER_DST ); } #[test] fn clear_texture_derives_transfer_destination_usage() { let mut graph = RenderGraph::new(); let texture = graph.create_texture(GraphTextureDesc { format: vk::Format::R8G8B8A8_UNORM, ..Default::default() }); graph.push(ClearTexture { texture, range: vk::ImageSubresourceRange { aspect_mask: vk::ImageAspectFlags::COLOR, base_mip_level: 0, level_count: 1, base_array_layer: 0, layer_count: 1, }, layout: vk::ImageLayout::TRANSFER_DST_OPTIMAL, color: [0.0; 4], }); let accesses = accesses_for_resource( texture.id(), graph.commands.side_effects.command_accesses(), &graph.commands.side_effects, ); assert_eq!( derive_image_usage(&accesses, vk::ImageUsageFlags::empty()), vk::ImageUsageFlags::TRANSFER_DST ); } #[test] fn extra_usage_is_preserved_and_combined_with_derived_usage() { let mut graph = RenderGraph::new(); let buffer = graph.create_buffer(GraphBufferDesc { size: 16, extra_usage: vk::BufferUsageFlags::UNIFORM_BUFFER, ..Default::default() }); graph.push(CopyBuffer { src: buffer, src_range: BufferRange { offset: 0, size: 16, }, dst: buffer, dst_range: BufferRange { offset: 0, size: 16, }, }); let accesses = accesses_for_resource( buffer.id(), graph.commands.side_effects.command_accesses(), &graph.commands.side_effects, ); assert_eq!( derive_buffer_usage(&accesses, vk::BufferUsageFlags::UNIFORM_BUFFER), vk::BufferUsageFlags::TRANSFER_SRC | vk::BufferUsageFlags::TRANSFER_DST | vk::BufferUsageFlags::UNIFORM_BUFFER ); } #[test] fn pipeline_stages_map_to_queue_capabilities() { let accesses = [ resource_access( vk::PipelineStageFlags2::VERTEX_SHADER, vk::AccessFlags2::SHADER_READ, ), resource_access( vk::PipelineStageFlags2::COMPUTE_SHADER, vk::AccessFlags2::SHADER_WRITE, ), resource_access( vk::PipelineStageFlags2::COPY | vk::PipelineStageFlags2::CLEAR, vk::AccessFlags2::TRANSFER_WRITE, ), resource_access(vk::PipelineStageFlags2::NONE, vk::AccessFlags2::NONE), ]; assert_eq!( derive_queue_capabilities(&accesses), QueueFlags::GRAPHICS | QueueFlags::ASYNC_COMPUTE | QueueFlags::TRANSFER ); } fn resource_access( stage: vk::PipelineStageFlags2, access: vk::AccessFlags2, ) -> crate::recorder::ResourceAccess { crate::recorder::ResourceAccess { resource: ResourceId::from_u32(0), scope: SubresourceScope { stage, access, subresource: SubresourceScopeInner::Buffer(BufferRange::full()), }, } } fn resource_access_for(resource: ResourceId) -> crate::recorder::ResourceAccess { crate::recorder::ResourceAccess { resource, scope: SubresourceScope { stage: vk::PipelineStageFlags2::COPY, access: vk::AccessFlags2::TRANSFER_WRITE, subresource: SubresourceScopeInner::Buffer(BufferRange::full()), }, } } fn scoped_access( resource: ResourceId, offset: u64, size: u64, stage: vk::PipelineStageFlags2, access: vk::AccessFlags2, ) -> crate::recorder::ResourceAccess { crate::recorder::ResourceAccess { resource, scope: SubresourceScope { stage, access, subresource: SubresourceScopeInner::Buffer(BufferRange { offset, size }), }, } } fn texture_range( aspect_mask: vk::ImageAspectFlags, base_mip_level: u32, level_count: u32, base_array_layer: u32, layer_count: u32, ) -> vk::ImageSubresourceRange { vk::ImageSubresourceRange { aspect_mask, base_mip_level, level_count, base_array_layer, layer_count, } } fn texture_scoped_access( resource: ResourceId, range: vk::ImageSubresourceRange, layout: vk::ImageLayout, stage: vk::PipelineStageFlags2, access: vk::AccessFlags2, ) -> crate::recorder::ResourceAccess { crate::recorder::ResourceAccess { resource, scope: SubresourceScope { stage, access, subresource: SubresourceScopeInner::Texture { range, layout }, }, } } }