From ac1b6c9cc8f8d250ea7f7b8f9aeb769bb484a073 Mon Sep 17 00:00:00 2001 From: janis Date: Sun, 25 Oct 2026 21:44:21 +0100 Subject: [PATCH] idk? barrier stuff --- crates/renderer_rendergraph/src/graph.rs | 871 ++++++++++++++++++- crates/renderer_rendergraph/src/recorder.rs | 194 ++++- crates/renderer_rendergraph/src/resources.rs | 183 ++-- crates/renderer_resources/src/images.rs | 107 +++ 4 files changed, 1278 insertions(+), 77 deletions(-) diff --git a/crates/renderer_rendergraph/src/graph.rs b/crates/renderer_rendergraph/src/graph.rs index 347d512..92944a2 100644 --- a/crates/renderer_rendergraph/src/graph.rs +++ b/crates/renderer_rendergraph/src/graph.rs @@ -19,7 +19,8 @@ use crate::{ recorder::{CommandIndex, CommandList, CommandRecorder, RecordError, scope_range}, resources::{ Buffer, GraphBufferDesc, GraphTextureDesc, ResourceDesc, ResourceId, Subresource, - SubresourceScope, SubresourceScopeInner, Texture, access_flag_is_write, + SubresourceScope, SubresourceScopeInner, Texture, access_flag_is_read, + access_flag_is_write, }, }; @@ -54,6 +55,40 @@ pub struct PipelineBarrier { pub resources: Vec, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct AccessRef { + ordered_index: usize, + access_index: usize, +} + +#[derive(Debug, Clone, Copy)] +struct BarrierResourceRef { + barrier_index: usize, + resource_index: usize, +} + +impl BarrierResourceRef { + fn get(self, barriers: &[PipelineBarrier]) -> &BarrierResource { + &barriers[self.barrier_index].resources[self.resource_index] + } + + fn widen( + self, + barriers: &mut [PipelineBarrier], + stage: vk::PipelineStageFlags2, + access: vk::AccessFlags2, + ) { + let resource = &mut barriers[self.barrier_index].resources[self.resource_index]; + resource.dst_stage |= stage; + resource.dst_access |= access; + } +} + +struct EmittedBarrierEntry { + access: AccessRef, + resource: BarrierResourceRef, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ResourceLifetime { pub first_command: CommandIndex, @@ -341,6 +376,20 @@ impl<'a> GraphCompiler<'a> { &self, order: &[CommandIndex], batches: &[Vec], + ) -> Vec { + let initial_states = self + .resources + .iter() + .map(ResourceDesc::initial_state) + .collect::>(); + self.build_barriers_with_initial_states(order, batches, &initial_states) + } + + fn build_barriers_with_initial_states( + &self, + order: &[CommandIndex], + batches: &[Vec], + initial_states: &[crate::resources::ResourceState], ) -> Vec { let mut batch_indices = vec![usize::MAX; self.command_count]; for (batch_index, batch) in batches.iter().enumerate() { @@ -349,31 +398,138 @@ impl<'a> GraphCompiler<'a> { } } - batches.iter().enumerate().flat_map(|(batch_index, batch)| { - batch - .iter() - .enumerate() - .map(move |(in_batch_index, &cmd)| (batch_index, in_batch_index, cmd)) - }); + let mut result: Vec = Vec::new(); + + // Associates each access that produced a barrier with its + // BarrierResource, so a later contained read can widen its dst masks. + // This empirically-driven optimization reduces barrier calls, but is + // not guaranteed to be cost-free: wider stage/access masks can increase + // cache-visibility or stage-synchronization costs on some vendors, and + // the later read waits at the earlier barrier point. + let mut barrier_entries: HashMap = HashMap::new(); - let mut result = Vec::new(); for (ordered_index, &command_index) in order.iter().enumerate() { let mut barrier_resources = Vec::new(); + let mut emitted_entries = Vec::new(); - for current in &self.accesses[command_index.index()] { - let previous = order[..ordered_index] - .iter() - .rev() - .find_map(|&prior_index| { - self.accesses[prior_index.index()] + for (current_access_index, current) in + self.accesses[command_index.index()].iter().enumerate() + { + let current_is_pure_read = access_flag_is_read(current.scope.access) + && !access_flag_is_write(current.scope.access); + + // Only reads can fold: an overlapping write breaks the read + // chain for the affected subresources, while disjoint writes + // leave it intact. + if current_is_pure_read { + let mut folded = false; + + 'prior_accesses: for prior_order_index in (0..ordered_index).rev() { + let prior_command = order[prior_order_index]; + for (prior_access_index, prior) in self.accesses[prior_command.index()] .iter() + .enumerate() .rev() - .find(|previous| { - previous.resource == current.resource - && previous.scope.conflicts(¤t.scope) - }) - .map(|access| (prior_index, access)) - }); + .filter(|(_, prior)| prior.resource == current.resource) + { + if !access_flag_is_write(prior.scope.access) { + if access_flag_is_read(prior.scope.access) + && prior.scope.has_same_access_polarity(¤t.scope) + && prior.scope.contains_subresource_range(¤t.scope) + { + let prior_access = AccessRef { + ordered_index: prior_order_index, + access_index: prior_access_index, + }; + if let Some(barrier_resource) = + barrier_entries.get(&prior_access).copied() + { + let prior_barrier_resource = barrier_resource.get(&result); + // Folding is valid only if the earlier barrier covers + // this entire read; widening masks cannot extend its range + // or change the layout established at that barrier. + if Self::can_fold_read( + prior.scope, + current.scope, + prior_barrier_resource, + ) { + barrier_resource.widen( + &mut result, + current.scope.stage, + current.scope.access, + ); + folded = true; + } + } + + if folded { + break 'prior_accesses; + } + + // This is the nearest containing read. If it + // cannot fold, normal hazard/initial-state + // handling must decide what barrier is needed. + break 'prior_accesses; + } + } else if prior.scope.overlaps_subresource(¤t.scope) { + // Only a write overlapping this read's range breaks + // its chain; disjoint writes do not affect containment. + break 'prior_accesses; + } + } + } + + if folded { + continue; + } + } + + let mut previous = None; + let mut src_stage = vk::PipelineStageFlags2::NONE; + let mut src_access = vk::AccessFlags2::NONE; + + // Accumulate every outstanding conflicting source, newest first. + // A prior barrier is a chain point only for the subresources its + // range covers. For partial coverage, conservatively keep scanning + // and include every older conflicting access; source masks do not + // encode which part of the current range they made available. + 'prior_barriers: for prior_order_index in (0..ordered_index).rev() { + let prior_command = order[prior_order_index]; + for (prior_access_index, prior) in self.accesses[prior_command.index()] + .iter() + .enumerate() + .rev() + .filter(|(_, prior)| { + prior.resource == current.resource + && prior.scope.conflicts(¤t.scope) + }) + { + if previous.is_none() { + previous = Some((prior_command, prior)); + } + + src_stage |= prior.scope.stage; + src_access |= prior.scope.access; + + let prior_access = AccessRef { + ordered_index: prior_order_index, + access_index: prior_access_index, + }; + + if let Some(barrier_resource) = barrier_entries.get(&prior_access).copied() + { + // The prior barrier's source masks carry the older + // availability chain; include them in this source scope. + let prior_barrier = barrier_resource.get(&result); + src_stage |= prior_barrier.src_stage; + src_access |= prior_barrier.src_access; + + if prior_barrier.range.contains_scope(¤t.scope) { + break 'prior_barriers; + } + } + } + } if let Some((previous_command, previous)) = previous { debug_assert!( @@ -381,13 +537,17 @@ impl<'a> GraphCompiler<'a> { < batch_indices[command_index.index()] ); + // Layouts follow the latest conflicting access, which is + // the state the resource actually reached most recently. + // TODO: Define a range-aware policy if accumulated sources + // with different layouts can overlap only partially. let (old_layout, new_layout) = Self::layouts(previous.scope, current.scope); barrier_resources.push(BarrierResource { id: current.resource, range: scope_range(current.scope), - src_stage: previous.scope.stage, - src_access: previous.scope.access, + src_stage, + src_access, dst_stage: current.scope.stage, dst_access: current.scope.access, old_layout, @@ -395,11 +555,22 @@ impl<'a> GraphCompiler<'a> { queue_transfer: None, }); + emitted_entries.push(EmittedBarrierEntry { + access: AccessRef { + ordered_index, + access_index: current_access_index, + }, + resource: BarrierResourceRef { + barrier_index: result.len(), + resource_index: barrier_resources.len() - 1, + }, + }); + continue; } let desc = &self.resources[current.resource.0 as usize]; - let initial = desc.initial_state(); + let initial = initial_states[current.resource.0 as usize]; let (old_layout, new_layout) = Self::layouts_from_state(initial, current.scope); @@ -424,10 +595,24 @@ impl<'a> GraphCompiler<'a> { new_layout, queue_transfer: None, }); + + emitted_entries.push(EmittedBarrierEntry { + access: AccessRef { + ordered_index, + access_index: current_access_index, + }, + resource: BarrierResourceRef { + barrier_index: result.len(), + resource_index: barrier_resources.len() - 1, + }, + }); } } if !barrier_resources.is_empty() { + for entry in emitted_entries { + barrier_entries.insert(entry.access, entry.resource); + } result.push(PipelineBarrier { before_command: command_index, resources: barrier_resources, @@ -462,6 +647,18 @@ impl<'a> GraphCompiler<'a> { (initial.layout, current_layout) } + fn can_fold_read( + prior: SubresourceScope, + current: SubresourceScope, + barrier: &BarrierResource, + ) -> bool { + let (_, prior_layout) = Self::layouts(prior, prior); + let (_, current_layout) = Self::layouts(prior, current); + prior_layout == current_layout + && barrier.new_layout == current_layout + && barrier.range.contains_scope(¤t) + } + fn bind_lifetimes(&self) -> (HashMap, ResourceInterference) { let mut lifetimes = HashMap::new(); for (command, command_accesses) in self.accesses.iter().enumerate() { @@ -1119,6 +1316,584 @@ mod tests { assert_eq!(compiled.barriers()[0].resources[0].id, b.id()); } + #[test] + fn write_read_write_barrier_accumulates_outstanding_write_and_read_sources() { + let resource = ResourceId(0); + let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { + size: 64, + ..Default::default() + })]; + let accesses = vec![ + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_WRITE, + )], + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::COMPUTE_SHADER, + vk::AccessFlags2::SHADER_WRITE, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + let barriers = compiler.build_barriers(&order, &batches); + + let second_write_barrier = barriers + .iter() + .find(|barrier| barrier.before_command == CommandIndex(2)) + .expect("second write has a barrier"); + assert_eq!( + second_write_barrier.resources[0].src_stage, + vk::PipelineStageFlags2::COPY | vk::PipelineStageFlags2::VERTEX_SHADER + ); + assert_eq!( + second_write_barrier.resources[0].src_access, + vk::AccessFlags2::TRANSFER_WRITE | vk::AccessFlags2::SHADER_READ + ); + } + + #[test] + fn partial_barrier_coverage_keeps_accumulating_older_write_sources() { + let resource = ResourceId(0); + let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { + size: 64, + ..Default::default() + })]; + let accesses = vec![ + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_WRITE, + )], + vec![scoped_access( + resource, + 0, + 16, + vk::PipelineStageFlags2::COMPUTE_SHADER, + vk::AccessFlags2::SHADER_WRITE, + )], + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::COLOR_ATTACHMENT_WRITE, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + let barriers = compiler.build_barriers(&order, &batches); + let third_write_barrier = barriers + .iter() + .find(|barrier| barrier.before_command == CommandIndex(2)) + .expect("third write has a barrier"); + + assert_eq!( + third_write_barrier.resources[0].src_stage, + vk::PipelineStageFlags2::COPY | vk::PipelineStageFlags2::COMPUTE_SHADER + ); + assert_eq!( + third_write_barrier.resources[0].src_access, + vk::AccessFlags2::TRANSFER_WRITE | vk::AccessFlags2::SHADER_WRITE + ); + } + + #[test] + fn disjoint_write_does_not_break_contained_read_barrier_folding() { + let resource = ResourceId(0); + let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { + size: 128, + ..Default::default() + })]; + let accesses = vec![ + vec![scoped_access( + resource, + 0, + 32, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_WRITE, + )], + vec![scoped_access( + resource, + 0, + 32, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + vec![scoped_access( + resource, + 64, + 32, + vk::PipelineStageFlags2::COMPUTE_SHADER, + vk::AccessFlags2::SHADER_WRITE, + )], + vec![scoped_access( + resource, + 8, + 8, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::UNIFORM_READ, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + let barriers = compiler.build_barriers(&order, &batches); + + assert_eq!(barriers.len(), 1); + assert_eq!(barriers[0].before_command, CommandIndex(1)); + assert_eq!( + barriers[0].resources[0].dst_stage, + vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER + ); + assert_eq!( + barriers[0].resources[0].dst_access, + vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::UNIFORM_READ + ); + } + + #[test] + fn imported_texture_read_with_different_layout_gets_initial_transition() { + let resource = ResourceId(0); + let resources = [ResourceDesc::CreatedTexture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + mip_levels: 2, + array_layers: 2, + ..Default::default() + })]; + let initial_states = [crate::resources::ResourceState { + stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, + access: vk::AccessFlags2::SHADER_READ, + layout: Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL), + }]; + // Supply the imported image's initial state directly so this stays + // device-free and does not need to construct a Vulkan Image wrapper. + let accesses = vec![ + vec![texture_scoped_access( + resource, + texture_range(vk::ImageAspectFlags::COLOR, 0, 2, 0, 2), + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + vec![texture_scoped_access( + resource, + texture_range(vk::ImageAspectFlags::COLOR, 1, 1, 1, 1), + vk::ImageLayout::GENERAL, + vk::PipelineStageFlags2::COMPUTE_SHADER, + vk::AccessFlags2::SHADER_SAMPLED_READ, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + let barriers = + compiler.build_barriers_with_initial_states(&order, &batches, &initial_states); + + assert_eq!(barriers.len(), 1); + assert_eq!(barriers[0].before_command, CommandIndex(1)); + assert_eq!( + barriers[0].resources[0].old_layout, + Some(initial_states[0].layout.unwrap()) + ); + assert_eq!( + barriers[0].resources[0].new_layout, + Some(vk::ImageLayout::GENERAL) + ); + assert_eq!( + barriers[0].resources[0].dst_access, + vk::AccessFlags2::SHADER_SAMPLED_READ + ); + } + + #[test] + fn contained_read_folds_into_prior_read_barrier_destination_masks() { + let resource = ResourceId(0); + let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { + size: 64, + ..Default::default() + })]; + let accesses = vec![ + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_WRITE, + )], + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + vec![scoped_access( + resource, + 16, + 16, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::UNIFORM_READ, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + let barriers = compiler.build_barriers(&order, &batches); + + assert_eq!(barriers.len(), 1); + assert_eq!(barriers[0].before_command, CommandIndex(1)); + assert_eq!(barriers[0].resources.len(), 1); + assert_eq!( + barriers[0].resources[0].dst_stage, + vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER + ); + assert_eq!( + barriers[0].resources[0].dst_access, + vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::UNIFORM_READ + ); + } + + #[test] + fn narrow_prior_barrier_does_not_cover_a_later_larger_read() { + let resource = ResourceId(0); + let prior = SubresourceScope { + access: vk::AccessFlags2::SHADER_READ, + stage: vk::PipelineStageFlags2::VERTEX_SHADER, + subresource: SubresourceScopeInner::Buffer(BufferRange { + offset: 0, + size: 64, + }), + }; + let current = SubresourceScope { + access: vk::AccessFlags2::UNIFORM_READ, + stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, + subresource: SubresourceScopeInner::Buffer(BufferRange { + offset: 0, + size: 32, + }), + }; + let narrow_barrier = BarrierResource { + id: resource, + range: Subresource::Buffer(BufferRange { + offset: 0, + size: 16, + }), + src_stage: vk::PipelineStageFlags2::COPY, + src_access: vk::AccessFlags2::TRANSFER_WRITE, + dst_stage: prior.stage, + dst_access: prior.access, + old_layout: None, + new_layout: None, + queue_transfer: None, + }; + + assert!(prior.contains_subresource_range(¤t)); + assert!(!GraphCompiler::can_fold_read( + prior, + current, + &narrow_barrier + )); + + // The prior scope contains the later read and the layouts match; only + // the synthetic barrier's narrower range invalidates folding. + let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { + size: 64, + ..Default::default() + })]; + let accesses = vec![ + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_WRITE, + )], + vec![scoped_access( + resource, + 0, + 16, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::UNIFORM_READ, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + let barriers = compiler.build_barriers(&order, &batches); + + let later_read_barrier = barriers + .iter() + .find(|barrier| barrier.before_command == CommandIndex(2)) + .expect("larger later read gets its own barrier"); + assert_eq!(later_read_barrier.resources[0].dst_stage, current.stage); + } + + #[test] + fn intervening_write_prevents_contained_read_barrier_folding() { + let resource = ResourceId(0); + let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { + size: 64, + ..Default::default() + })]; + let accesses = vec![ + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_WRITE, + )], + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::COMPUTE_SHADER, + vk::AccessFlags2::SHADER_WRITE, + )], + vec![scoped_access( + resource, + 16, + 16, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + let barriers = compiler.build_barriers(&order, &batches); + + assert_eq!(barriers.len(), 3); + assert_eq!(barriers[0].before_command, CommandIndex(1)); + assert_eq!(barriers[1].before_command, CommandIndex(2)); + assert_eq!(barriers[2].before_command, CommandIndex(3)); + assert_eq!( + barriers[2].resources[0].dst_stage, + vk::PipelineStageFlags2::FRAGMENT_SHADER + ); + } + + #[test] + fn contained_texture_read_folds_aspect_mip_and_layer_ranges() { + let resource = ResourceId(0); + let resources = [ResourceDesc::CreatedTexture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + mip_levels: 4, + array_layers: 8, + ..Default::default() + })]; + let accesses = vec![ + vec![texture_scoped_access( + resource, + texture_range( + vk::ImageAspectFlags::COLOR | vk::ImageAspectFlags::DEPTH, + 0, + 4, + 0, + 8, + ), + vk::ImageLayout::GENERAL, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_WRITE, + )], + vec![texture_scoped_access( + resource, + texture_range( + vk::ImageAspectFlags::COLOR | vk::ImageAspectFlags::DEPTH, + 0, + 4, + 0, + 8, + ), + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + vec![texture_scoped_access( + resource, + texture_range(vk::ImageAspectFlags::COLOR, 1, 2, 2, 3), + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::SHADER_SAMPLED_READ, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + let barriers = compiler.build_barriers(&order, &batches); + + assert_eq!(barriers.len(), 2); + assert_eq!(barriers[0].before_command, CommandIndex(0)); + assert_eq!(barriers[1].before_command, CommandIndex(1)); + let barrier = &barriers[1].resources[0]; + assert_eq!( + barrier.dst_stage, + vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER + ); + assert_eq!( + barrier.dst_access, + vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_SAMPLED_READ + ); + assert!( + matches!(barrier.range, Subresource::Texture(range) if range.aspect_mask == vk::ImageAspectFlags::COLOR | vk::ImageAspectFlags::DEPTH && range.base_mip_level == 0 && range.level_count == 4 && range.base_array_layer == 0 && range.layer_count == 8) + ); + } + + #[test] + fn overlapping_texture_reads_with_different_layouts_are_not_folded() { + let resource = ResourceId(0); + let resources = [ResourceDesc::CreatedTexture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + ..Default::default() + })]; + let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); + let accesses = vec![ + vec![texture_scoped_access( + resource, + range, + vk::ImageLayout::GENERAL, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_WRITE, + )], + vec![texture_scoped_access( + resource, + range, + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + vec![texture_scoped_access( + resource, + range, + vk::ImageLayout::GENERAL, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::SHADER_SAMPLED_READ, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + let barriers = compiler.build_barriers(&order, &batches); + + assert_eq!(barriers.len(), 3); + let current_read_barrier = &barriers[2]; + assert_eq!(current_read_barrier.before_command, CommandIndex(2)); + let barrier = ¤t_read_barrier.resources[0]; + assert_eq!(barrier.dst_stage, vk::PipelineStageFlags2::FRAGMENT_SHADER); + assert_eq!(barrier.dst_access, vk::AccessFlags2::SHADER_SAMPLED_READ); + assert_eq!( + barrier.old_layout, + Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) + ); + assert_eq!(barrier.new_layout, Some(vk::ImageLayout::GENERAL)); + } + + #[test] + fn pure_read_without_prior_barrier_entry_needs_no_barrier() { + let resource = ResourceId(0); + let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { + size: 64, + ..Default::default() + })]; + let accesses = vec![ + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + vec![scoped_access( + resource, + 16, + 16, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::UNIFORM_READ, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + + assert!(compiler.build_barriers(&order, &batches).is_empty()); + } + + #[test] + fn write_before_contained_read_emits_barrier_before_read() { + let resource = ResourceId(0); + let resources = [ResourceDesc::CreatedBuffer(GraphBufferDesc { + size: 64, + ..Default::default() + })]; + let accesses = vec![ + vec![scoped_access( + resource, + 0, + 64, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_WRITE, + )], + vec![scoped_access( + resource, + 16, + 16, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::UNIFORM_READ, + )], + ]; + let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let batches = compiler.topo_batches().expect("graph is acyclic"); + let order = batches.iter().flatten().copied().collect::>(); + let barriers = compiler.build_barriers(&order, &batches); + + assert_eq!(batches, [vec![CommandIndex(0)], vec![CommandIndex(1)]]); + assert_eq!(barriers.len(), 1); + assert_eq!(barriers[0].before_command, CommandIndex(1)); + assert_eq!( + barriers[0].resources[0].src_access, + vk::AccessFlags2::TRANSFER_WRITE + ); + assert_eq!( + barriers[0].resources[0].dst_stage, + vk::PipelineStageFlags2::FRAGMENT_SHADER + ); + assert_eq!( + barriers[0].resources[0].dst_access, + vk::AccessFlags2::UNIFORM_READ + ); + } + #[test] fn independent_commands_share_a_batch_and_dependencies_advance_to_later_batches() { let mut graph = RenderGraph::new(); @@ -1544,4 +2319,54 @@ mod tests { }, } } + + fn scoped_access( + resource: ResourceId, + offset: u64, + size: u64, + stage: vk::PipelineStageFlags2, + access: vk::AccessFlags2, + ) -> crate::recorder::ResourceAccess { + crate::recorder::ResourceAccess { + resource, + scope: SubresourceScope { + stage, + access, + subresource: SubresourceScopeInner::Buffer(BufferRange { offset, size }), + }, + } + } + + fn texture_range( + aspect_mask: vk::ImageAspectFlags, + base_mip_level: u32, + level_count: u32, + base_array_layer: u32, + layer_count: u32, + ) -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask, + base_mip_level, + level_count, + base_array_layer, + layer_count, + } + } + + fn texture_scoped_access( + resource: ResourceId, + range: vk::ImageSubresourceRange, + layout: vk::ImageLayout, + stage: vk::PipelineStageFlags2, + access: vk::AccessFlags2, + ) -> crate::recorder::ResourceAccess { + crate::recorder::ResourceAccess { + resource, + scope: SubresourceScope { + stage, + access, + subresource: SubresourceScopeInner::Texture { range, layout }, + }, + } + } } diff --git a/crates/renderer_rendergraph/src/recorder.rs b/crates/renderer_rendergraph/src/recorder.rs index 3b0e2b9..9f617a6 100644 --- a/crates/renderer_rendergraph/src/recorder.rs +++ b/crates/renderer_rendergraph/src/recorder.rs @@ -47,6 +47,11 @@ impl SideEffectMap { index.into() } + /// Record an access scope for a command and resource. + /// + /// Each scope must describe the command's complete use for that access, + /// not merely an enclosing bound. In particular, the reverse containment + /// fold is sound only when the larger scope represents the complete use. pub fn insert( &mut self, command_index: CommandIndex, @@ -58,8 +63,19 @@ impl SideEffectMap { } let accesses = &mut self.accesses[command_index.0]; for existing in accesses.iter_mut() { - if existing.resource == resource && existing.scope.try_merge(&scope) { - return; + if existing.resource == resource { + if existing.scope.try_merge(&scope) { + return; + } + if scope.has_same_access_polarity(&existing.scope) + && scope.contains_subresource(&existing.scope) + { + let mut merged = scope; + merged.stage |= existing.scope.stage; + merged.access |= existing.scope.access; + existing.scope = merged; + return; + } } } accesses.push(ResourceAccess { resource, scope }); @@ -221,3 +237,177 @@ pub(crate) fn scope_range(scope: SubresourceScope) -> Subresource { SubresourceScopeInner::Texture { range, .. } => Subresource::Texture(range), } } + +#[cfg(test)] +mod tests { + use super::*; + + fn read_scope( + offset: u64, + size: u64, + stage: vk::PipelineStageFlags2, + access: vk::AccessFlags2, + ) -> SubresourceScope { + SubresourceScope { + access, + stage, + subresource: SubresourceScopeInner::Buffer(BufferRange { offset, size }), + } + } + + #[test] + fn side_effect_map_folds_same_command_scopes_in_both_containment_directions() { + let mut map = SideEffectMap::default(); + let command = map.begin_command(); + map.insert( + command, + ResourceId(0), + read_scope( + 0, + 64, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + ), + ); + map.insert( + command, + ResourceId(0), + read_scope( + 16, + 16, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::UNIFORM_READ, + ), + ); + + let accesses = map.accesses(command); + assert_eq!(accesses.len(), 1); + assert_eq!( + accesses[0].scope.stage, + vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER + ); + assert_eq!( + accesses[0].scope.access, + vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::UNIFORM_READ + ); + assert!(matches!( + accesses[0].scope.subresource, + SubresourceScopeInner::Buffer(BufferRange { + offset: 0, + size: 64 + }) + )); + + let mut reverse = SideEffectMap::default(); + let command = reverse.begin_command(); + reverse.insert( + command, + ResourceId(1), + read_scope( + 16, + 16, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + ), + ); + reverse.insert( + command, + ResourceId(1), + read_scope( + 0, + 64, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::UNIFORM_READ, + ), + ); + + let accesses = reverse.accesses(command); + assert_eq!(accesses.len(), 1); + assert_eq!( + accesses[0].scope.stage, + vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER + ); + assert_eq!( + accesses[0].scope.access, + vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::UNIFORM_READ + ); + assert!(matches!( + accesses[0].scope.subresource, + SubresourceScopeInner::Buffer(BufferRange { + offset: 0, + size: 64 + }) + )); + } + + #[test] + fn side_effect_map_keeps_overlapping_non_subset_scopes_separate() { + let mut map = SideEffectMap::default(); + let command = map.begin_command(); + map.insert( + command, + ResourceId(0), + read_scope( + 0, + 16, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + ), + ); + map.insert( + command, + ResourceId(0), + read_scope( + 8, + 16, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::UNIFORM_READ, + ), + ); + + assert_eq!(map.accesses(command).len(), 2); + } + + #[test] + fn side_effect_map_does_not_fold_opposite_polarity_scopes() { + let mut map = SideEffectMap::default(); + let command = map.begin_command(); + map.insert( + command, + ResourceId(0), + read_scope( + 0, + 64, + vk::PipelineStageFlags2::COMPUTE_SHADER, + vk::AccessFlags2::SHADER_READ, + ), + ); + map.insert( + command, + ResourceId(0), + read_scope( + 16, + 16, + vk::PipelineStageFlags2::COMPUTE_SHADER, + vk::AccessFlags2::SHADER_WRITE, + ), + ); + + let accesses = map.accesses(command); + assert_eq!(accesses.len(), 2); + assert!(matches!( + accesses[0].scope.subresource, + SubresourceScopeInner::Buffer(BufferRange { + offset: 0, + size: 64 + }) + )); + assert!(matches!( + accesses[1].scope.subresource, + SubresourceScopeInner::Buffer(BufferRange { + offset: 16, + size: 16 + }) + )); + } +} diff --git a/crates/renderer_rendergraph/src/resources.rs b/crates/renderer_rendergraph/src/resources.rs index b5bb8db..15aff07 100644 --- a/crates/renderer_rendergraph/src/resources.rs +++ b/crates/renderer_rendergraph/src/resources.rs @@ -1,7 +1,10 @@ use std::{borrow::Cow, sync::Arc}; use ash::vk; -use renderer_resources::{buffers::Buffer as BufferResource, images::Image as ImageResource}; +use renderer_resources::{ + buffers::Buffer as BufferResource, + images::{Image as ImageResource, ImageSubresourceExt}, +}; /// Minimal graph-level buffer description. Buffer usage, queue sharing, and /// allocation strategy are inferred from recorded command accesses at resolve. @@ -121,6 +124,10 @@ impl BufferRange { pub fn overlaps(self, other: Self) -> bool { self.offset < other.end() && other.offset < self.end() } + + pub fn contains(self, other: &Self) -> bool { + self.offset <= other.offset && self.end() >= other.end() + } } #[derive(Debug, Clone, Copy)] @@ -129,6 +136,50 @@ pub enum Subresource { Texture(vk::ImageSubresourceRange), } +impl Subresource { + /// Whether this barrier range fully covers the scope's subresource range. + pub fn contains_scope(self, scope: &SubresourceScope) -> bool { + match (self, scope.subresource) { + (Self::Buffer(range), SubresourceScopeInner::Buffer(other)) => range.contains(&other), + (Self::Texture(range), SubresourceScopeInner::Texture { range: other, .. }) => { + range.aspect_mask.contains(other.aspect_mask) + && range_contains( + range.base_mip_level, + range.level_count, + other.base_mip_level, + other.level_count, + vk::REMAINING_MIP_LEVELS, + ) + && range_contains( + range.base_array_layer, + range.layer_count, + other.base_array_layer, + other.layer_count, + vk::REMAINING_ARRAY_LAYERS, + ) + } + _ => false, + } + } +} + +fn range_contains( + base: u32, + count: u32, + other_base: u32, + other_count: u32, + remaining: u32, +) -> bool { + let end = |base: u32, count: u32| { + if count == remaining { + u64::MAX + } else { + u64::from(base) + u64::from(count) + } + }; + base <= other_base && end(base, count) >= end(other_base, other_count) +} + #[derive(Debug, Clone, Copy)] pub enum SubresourceScopeInner { Buffer(BufferRange), @@ -146,7 +197,54 @@ pub struct SubresourceScope { } impl SubresourceScope { - /// Fold identical ranges with compatible access polarity into one scope. + /// Whether the scopes refer to any common buffer bytes or texture + /// subresources, regardless of access mode or image layout. + pub fn overlaps_subresource(&self, other: &Self) -> bool { + match (self.subresource, other.subresource) { + (SubresourceScopeInner::Buffer(a), SubresourceScopeInner::Buffer(b)) => a.overlaps(b), + ( + SubresourceScopeInner::Texture { range: a, .. }, + SubresourceScopeInner::Texture { range: b, .. }, + ) => a.intersects(&b), + _ => false, + } + } + + /// Whether this scope's subresource contains the other's subresource. + /// Texture ranges are compatible only when their layouts match. + pub fn contains_subresource(&self, other: &Self) -> bool { + self.contains_subresource_range(other) + && match (self.subresource, other.subresource) { + ( + SubresourceScopeInner::Texture { layout: a, .. }, + SubresourceScopeInner::Texture { layout: b, .. }, + ) => a == b, + (SubresourceScopeInner::Buffer(_), SubresourceScopeInner::Buffer(_)) => true, + _ => false, + } + } + + /// Whether this scope's range contains the other's, without considering + /// texture layout. Use this to find candidates whose layout must be checked + /// separately (for example, when deciding whether a read can fold). + pub fn contains_subresource_range(&self, other: &Self) -> bool { + match (self.subresource, other.subresource) { + (SubresourceScopeInner::Buffer(a), SubresourceScopeInner::Buffer(b)) => a.contains(&b), + ( + SubresourceScopeInner::Texture { range: a, .. }, + SubresourceScopeInner::Texture { range: b, .. }, + ) => a.contains(&b), + _ => false, + } + } + + /// Whether both scopes have the same read/write polarity. + pub fn has_same_access_polarity(&self, other: &Self) -> bool { + access_flag_is_read(self.access) == access_flag_is_read(other.access) + && access_flag_is_write(self.access) == access_flag_is_write(other.access) + } + + /// Fold sub-ranges with compatible access polarity into one scope. pub fn try_merge(&mut self, other: &Self) -> bool { if access_flag_is_read(self.access) != access_flag_is_read(other.access) || access_flag_is_write(self.access) != access_flag_is_write(other.access) @@ -154,8 +252,8 @@ impl SubresourceScope { return false; } - let same_range = match (self.subresource, other.subresource) { - (SubresourceScopeInner::Buffer(a), SubresourceScopeInner::Buffer(b)) => a == b, + let can_merge = match (self.subresource, other.subresource) { + (SubresourceScopeInner::Buffer(a), SubresourceScopeInner::Buffer(b)) => a.contains(&b), ( SubresourceScopeInner::Texture { range: a, @@ -165,71 +263,52 @@ impl SubresourceScope { range: b, layout: layout_b, }, - ) => texture_ranges_equal(a, b) && layout_a == layout_b, + ) => layout_a == layout_b && a.contains(&b), _ => false, }; - if same_range { + if can_merge { self.stage |= other.stage; self.access |= other.access; } - same_range + + can_merge } /// Whether accesses overlap and require ordering (a write or layout change). pub fn conflicts(&self, other: &Self) -> bool { - let overlaps = match (self.subresource, other.subresource) { - (SubresourceScopeInner::Buffer(a), SubresourceScopeInner::Buffer(b)) => a.overlaps(b), - ( - SubresourceScopeInner::Texture { range: a, .. }, - SubresourceScopeInner::Texture { range: b, .. }, - ) => texture_ranges_overlap(a, b), - _ => false, - }; - - if !overlaps { - return false; + // Read-after-read accesses do not conflict, even if they overlap.. + if access_flag_is_read(self.access) + && !access_flag_is_write(self.access) + && access_flag_is_read(other.access) + && !access_flag_is_write(other.access) + { + // ..unless the layouts differ for overlapping texture subresources. + return match (self.subresource, other.subresource) { + ( + SubresourceScopeInner::Texture { + layout: layout_a, + range: a, + }, + SubresourceScopeInner::Texture { + layout: layout_b, + range: b, + }, + ) => layout_a != layout_b && a.intersects(&b), + _ => false, + }; } + // for accesses involving writes: + use SubresourceScopeInner::*; match (self.subresource, other.subresource) { - ( - SubresourceScopeInner::Texture { layout: a, .. }, - SubresourceScopeInner::Texture { layout: b, .. }, - ) if a != b => true, - _ => { - !(access_flag_is_read(self.access) - && !access_flag_is_write(self.access) - && access_flag_is_read(other.access) - && !access_flag_is_write(other.access)) - } + (Buffer(a), Buffer(b)) => a.overlaps(b), + (Texture { range: a, .. }, Texture { range: b, .. }) => a.intersects(&b), + _ => false, } } } -fn texture_ranges_equal(a: vk::ImageSubresourceRange, b: vk::ImageSubresourceRange) -> bool { - a.aspect_mask == b.aspect_mask - && a.base_mip_level == b.base_mip_level - && a.level_count == b.level_count - && a.base_array_layer == b.base_array_layer - && a.layer_count == b.layer_count -} - -fn range_end(base: u32, count: u32) -> u64 { - if count == vk::REMAINING_MIP_LEVELS || count == vk::REMAINING_ARRAY_LAYERS { - u64::MAX - } else { - u64::from(base) + u64::from(count) - } -} - -fn texture_ranges_overlap(a: vk::ImageSubresourceRange, b: vk::ImageSubresourceRange) -> bool { - a.aspect_mask.intersects(b.aspect_mask) - && u64::from(a.base_mip_level) < range_end(b.base_mip_level, b.level_count) - && u64::from(b.base_mip_level) < range_end(a.base_mip_level, a.level_count) - && u64::from(a.base_array_layer) < range_end(b.base_array_layer, b.layer_count) - && u64::from(b.base_array_layer) < range_end(a.base_array_layer, a.layer_count) -} - /// True when an access mask contains a Vulkan read access. pub fn access_flag_is_read(access: vk::AccessFlags2) -> bool { access.intersects( diff --git a/crates/renderer_resources/src/images.rs b/crates/renderer_resources/src/images.rs index c8e398e..7bad7b4 100644 --- a/crates/renderer_resources/src/images.rs +++ b/crates/renderer_resources/src/images.rs @@ -1027,3 +1027,110 @@ pub const SUBRESOURCERANGE_COLOR_ALL: vk::ImageSubresourceRange = vk::ImageSubre base_array_layer: 0, layer_count: vk::REMAINING_ARRAY_LAYERS, }; + +pub trait IntoImageSubresourceLayers { + fn as_layers(&self) -> vk::ImageSubresourceLayers; +} + +pub trait IntoImageSubresourceRange { + fn as_range(&self) -> vk::ImageSubresourceRange; +} + +impl IntoImageSubresourceLayers for vk::ImageSubresource { + fn as_layers(&self) -> vk::ImageSubresourceLayers { + vk::ImageSubresourceLayers { + aspect_mask: self.aspect_mask, + mip_level: self.mip_level, + base_array_layer: self.array_layer, + layer_count: 1, + } + } +} + +impl IntoImageSubresourceRange for T +where + T: IntoImageSubresourceLayers, +{ + fn as_range(&self) -> vk::ImageSubresourceRange { + self.as_layers().as_range() + } +} + +impl IntoImageSubresourceRange for vk::ImageSubresourceLayers { + fn as_range(&self) -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: self.aspect_mask, + base_mip_level: self.mip_level, + level_count: 1, + base_array_layer: self.base_array_layer, + layer_count: self.layer_count, + } + } +} + +impl IntoImageSubresourceRange for vk::ImageSubresourceRange { + fn as_range(&self) -> vk::ImageSubresourceRange { + *self + } +} + +pub trait ImageSubresourceExt { + fn aspects(&self) -> vk::ImageAspectFlags; + fn layers(&self) -> MipRange; + fn mip_levels(&self) -> MipRange; + fn contains(&self, other: &Self) -> bool { + self.aspects().contains(other.aspects()) + && self.layers().contains(other.layers()) + && self.mip_levels().contains(other.mip_levels()) + } + fn intersects(&self, other: &Self) -> bool { + self.aspects().intersects(other.aspects()) + && self.layers().intersects(other.layers()) + && self.mip_levels().intersects(other.mip_levels()) + } +} + +impl ImageSubresourceExt for vk::ImageSubresourceLayers { + fn aspects(&self) -> vk::ImageAspectFlags { + self.aspect_mask + } + fn layers(&self) -> MipRange { + MipRange::new( + self.base_array_layer, + self.base_array_layer + self.layer_count, + ) + } + + fn mip_levels(&self) -> MipRange { + MipRange::single(self.mip_level) + } +} + +impl ImageSubresourceExt for vk::ImageSubresourceRange { + fn aspects(&self) -> vk::ImageAspectFlags { + self.aspect_mask + } + fn layers(&self) -> MipRange { + MipRange::new( + self.base_array_layer, + self.base_array_layer + self.layer_count, + ) + } + + fn mip_levels(&self) -> MipRange { + MipRange::new(self.base_mip_level, self.base_mip_level + self.level_count) + } +} + +impl ImageSubresourceExt for vk::ImageSubresource { + fn aspects(&self) -> vk::ImageAspectFlags { + self.aspect_mask + } + fn layers(&self) -> MipRange { + MipRange::single(self.mip_level) + } + + fn mip_levels(&self) -> MipRange { + MipRange::single(self.array_layer) + } +}