From afb322fa93200c6c4d90229e28279a4874eca4ff Mon Sep 17 00:00:00 2001 From: janis Date: Thu, 29 Oct 2026 00:51:52 +0100 Subject: [PATCH] rendergraph: dynamic-rendering render passes, single-pass compiler, perf - Add BeginRendering/EndRendering commands with imported/created attachment views, view plumbing, and aspect-gated depth/stencil slots - Derive and validate render-pass spans at compile time; contract spans into single DAG nodes before hazard analysis; hoist cross-boundary barriers to pass begins with in-pass policy and created-view resolution - Replace the four-pass ordering pipeline with a single command walk (build_command_orderings): span derivation, validation, lifetime-only injection, dependency edges, and Kahn batching in one pass; batches are ranges into execution_order and RenderPassSpan keeps only begin/end - Perf: node-collapsed frontier analysis, dense command/resource indexing (no SipHash on hot paths), inline barrier storage, exact preallocation, lazy Kahn ready heap, scoped barrier fast paths - Scope buffer and image alias barriers to the incoming resource per the Vulkan resource-memory-aliasing rules (availability unscoped, visibility scoped to the new owner) - Add criterion compile benchmarks (two_pass/{no_pass,pass}/per_{20,50,100}) --- Cargo.toml | 4 +- crates/renderer/src/egui.rs | 6 +- crates/renderer/src/rendering/mod.rs | 6 +- crates/renderer/src/swapchain.rs | 3 +- crates/renderer_rendergraph/Cargo.toml | 7 + .../renderer_rendergraph/benches/compile.rs | 141 + crates/renderer_rendergraph/src/commands.rs | 410 ++ crates/renderer_rendergraph/src/graph.rs | 3959 +++++++++++++++-- .../src/graph_compiler.rs | 250 ++ crates/renderer_rendergraph/src/lib.rs | 10 +- crates/renderer_rendergraph/src/recorder.rs | 71 +- .../tests/graph_validation.rs | 236 +- crates/renderer_resources/src/images.rs | 50 +- design-plans/rendergraph-render-passes.md | 575 +++ 14 files changed, 5211 insertions(+), 517 deletions(-) create mode 100644 crates/renderer_rendergraph/benches/compile.rs create mode 100644 crates/renderer_rendergraph/src/graph_compiler.rs create mode 100644 design-plans/rendergraph-render-passes.md diff --git a/Cargo.toml b/Cargo.toml index a90d647..cf45237 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ [dependencies] bevy_vulkan_render = { path = "crates/bevy_vulkan_render" } +bitflags.workspace = true [dev-dependencies] bevy = { workspace = true } @@ -28,6 +29,7 @@ debug = true [workspace.dependencies] anyhow = "1.0.89" +criterion = "0.7" thiserror = "2.0" derive_more = { version = "1.0.0", features = ["deref", "debug", "deref_mut"] } @@ -46,7 +48,7 @@ rectangle-pack = "0.4.2" # vk-sync = "0.1.6" arrayvec = "0.7.6" -tinyvec = "1.8" +tinyvec = { version = "1.8", features = ["alloc"] } indexmap = "2" petgraph = "0.7" fixedbitset = "0.5" diff --git a/crates/renderer/src/egui.rs b/crates/renderer/src/egui.rs index d431339..50444ae 100644 --- a/crates/renderer/src/egui.rs +++ b/crates/renderer/src/egui.rs @@ -571,7 +571,7 @@ pub fn egui_pass( .values() .map(|entry| { let texture = texture_handler.get_texture(entry.id).unwrap(); - let view = Arc::new(ManuallyDrop::into_inner( + let view = Arc::new( texture .create_view( ImageViewDesc::color_2d() @@ -579,7 +579,7 @@ pub fn egui_pass( .with_layer_range(0..1), ) .unwrap(), - )); + ); let sampler = samplers.get_sampler_owned(entry.as_sampler_desc()).unwrap(); (view, sampler) }) @@ -770,7 +770,7 @@ pub fn egui_pass( let color_attachment = &vk::RenderingAttachmentInfo::default() .image_layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL) - .image_view(target.create_view(ImageViewDesc::color_2d())?.raw()) + .image_view(ManuallyDrop::new(target.create_view(ImageViewDesc::color_2d())?).raw()) .load_op(vk::AttachmentLoadOp::LOAD) .store_op(vk::AttachmentStoreOp::STORE); diff --git a/crates/renderer/src/rendering/mod.rs b/crates/renderer/src/rendering/mod.rs index 68974f7..cdfd94c 100644 --- a/crates/renderer/src/rendering/mod.rs +++ b/crates/renderer/src/rendering/mod.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{mem::ManuallyDrop, sync::Arc}; pub use crate::egui_pass::{egui_pass, egui_pre_pass}; @@ -329,7 +329,9 @@ impl Wireframe { let color_attachment = &vk::RenderingAttachmentInfo::default() .image_layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL) - .image_view(target.create_view(ImageViewDesc::color_2d())?.raw()) + .image_view( + ManuallyDrop::new(target.create_view(ImageViewDesc::color_2d())?).raw(), + ) .load_op(vk::AttachmentLoadOp::LOAD) .store_op(vk::AttachmentStoreOp::STORE); diff --git a/crates/renderer/src/swapchain.rs b/crates/renderer/src/swapchain.rs index 5ca8da4..5b48759 100644 --- a/crates/renderer/src/swapchain.rs +++ b/crates/renderer/src/swapchain.rs @@ -1,5 +1,4 @@ use std::{ - mem::ManuallyDrop, num::NonZero, ops::Deref, sync::{ @@ -589,7 +588,7 @@ impl Swapchain { SwapchainImage { index: idx as u32, swapchain: self, - view: ManuallyDrop::into_inner(view), + view, acquire, release, }, diff --git a/crates/renderer_rendergraph/Cargo.toml b/crates/renderer_rendergraph/Cargo.toml index 47f92d0..0f76de9 100644 --- a/crates/renderer_rendergraph/Cargo.toml +++ b/crates/renderer_rendergraph/Cargo.toml @@ -6,10 +6,17 @@ edition = "2024" [dependencies] ash.workspace = true fixedbitset.workspace = true +tinyvec.workspace = true renderer_device = { path = "../renderer_device" } renderer_resources = { path = "../renderer_resources" } ash-formats = { git = "https://github.com/janis-bhm/ash-formats", branch = "main" } +bitflags.workspace = true [dev-dependencies] +criterion = { workspace = true } renderer_resources = { path = "../renderer_resources", features = ["testing"] } renderer_device = { path = "../renderer_device", features = ["testing"] } + +[[bench]] +name = "compile" +harness = false diff --git a/crates/renderer_rendergraph/benches/compile.rs b/crates/renderer_rendergraph/benches/compile.rs new file mode 100644 index 0000000..168ded2 --- /dev/null +++ b/crates/renderer_rendergraph/benches/compile.rs @@ -0,0 +1,141 @@ +//! These graph shapes compare command-level batching with render-pass span +//! contraction and barrier hoisting. Each has two disjoint buffer access sets +//! joined by a shared-buffer dependency across the passes. + +use std::{hint::black_box, time::Duration}; + +use ash::vk; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use renderer_rendergraph::{ + Buffer, BufferRange, GraphBufferDesc, RenderGraph, + commands::{BeginRendering, Command, EndRendering}, + recorder::{CommandIndex, CommandRecorder, RecordError, RecordedCommand, SideEffectMap}, + resources::{SubresourceScope, SubresourceScopeInner}, +}; + +struct Use { + buffer: Buffer, + write: bool, +} + +impl Command for Use { + fn side_effects(&self, command_index: CommandIndex, map: &mut SideEffectMap) { + map.insert( + command_index, + self.buffer.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COMPUTE_SHADER, + access: if self.write { + vk::AccessFlags2::SHADER_WRITE + } else { + vk::AccessFlags2::SHADER_READ + }, + subresource: SubresourceScopeInner::Buffer(BufferRange::full()), + }, + ); + } +} + +impl RecordedCommand for Use { + fn allowed_in_render_pass(&self) -> bool { + true + } + + fn apply(&mut self, _recorder: &mut CommandRecorder<'_>) -> Result<(), RecordError> { + Ok(()) + } +} + +fn build(n_groups: usize, per: usize, pass: bool) -> RenderGraph { + let mut graph = RenderGraph::new(); + let a: Vec = (0..n_groups) + .map(|_| { + graph.create_buffer(GraphBufferDesc { + size: 4096, + ..Default::default() + }) + }) + .collect(); + let b: Vec = (0..n_groups) + .map(|_| { + graph.create_buffer(GraphBufferDesc { + size: 4096, + ..Default::default() + }) + }) + .collect(); + let shared = graph.create_buffer(GraphBufferDesc { + size: 4096, + ..Default::default() + }); + + if pass { + graph.push(BeginRendering { + color: vec![], + depth: None, + }); + } + for _ in 0..per { + for &buffer in &a { + graph.push(Use { + buffer, + write: true, + }); + } + } + graph.push(Use { + buffer: shared, + write: true, + }); + if pass { + graph.push(EndRendering); + graph.push(BeginRendering { + color: vec![], + depth: None, + }); + } + for _ in 0..per { + for &buffer in &b { + graph.push(Use { + buffer, + write: true, + }); + } + } + graph.push(Use { + buffer: shared, + write: false, + }); + if pass { + graph.push(EndRendering); + } + + graph +} + +fn compile_benchmarks(criterion: &mut Criterion) { + let mut group = criterion.benchmark_group("compile"); + group + .measurement_time(Duration::from_secs(4)) + .sample_size(20); + + for (pass_name, pass) in [("no_pass", false), ("pass", true)] { + for per in [20, 50, 100] { + group.bench_function(format!("two_pass/{pass_name}/per_{per}"), |bencher| { + bencher.iter_batched( + || build(50, per, pass), + |graph| { + let compiled = graph.compile().unwrap(); + black_box(compiled.barriers().len()) + }, + BatchSize::LargeInput, + ); + }); + } + } + + group.finish(); +} + +criterion_group!(benches, compile_benchmarks); +criterion_main!(benches); diff --git a/crates/renderer_rendergraph/src/commands.rs b/crates/renderer_rendergraph/src/commands.rs index 9ee0095..a8cb27a 100644 --- a/crates/renderer_rendergraph/src/commands.rs +++ b/crates/renderer_rendergraph/src/commands.rs @@ -1,4 +1,7 @@ +use std::{cell::RefCell, sync::Arc}; + use ash::vk; +use renderer_resources::images::{ImageView, ImageViewDesc}; use crate::{ recorder::{CommandIndex, CommandRecorder, RecordError, RecordedCommand, SideEffectMap}, @@ -9,6 +12,243 @@ use crate::{ }, }; +/// Classification used by the compiler to validate render-pass spans. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CommandKind { + Begin, + End, + Graphics, + Transfer, + Compute, + Other, +} + +/// A view used by a render-pass attachment. +#[derive(Debug, Clone)] +pub enum AttachmentView { + /// A view the graph must create after resource resolution, described concretely. + Created(ImageViewDesc), + /// A caller-supplied view over an imported image. + Imported(Arc), +} + +/// A color attachment used by dynamic rendering. +#[derive(Clone)] +pub struct ColorAttachment { + pub texture: Texture, + pub view: AttachmentView, + /// Layout used while rendering (must be a legal rendering layout; validated later). + pub layout: vk::ImageLayout, + pub load_op: vk::AttachmentLoadOp, + pub store_op: vk::AttachmentStoreOp, + pub clear_value: vk::ClearValue, + /// Concrete subresource range covered by the view. Created-view sentinels are preserved for + /// compile-time validation. + pub range: vk::ImageSubresourceRange, +} + +impl std::fmt::Debug for ColorAttachment { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ColorAttachment") + .field("texture", &self.texture) + .field("view", &self.view) + .field("layout", &self.layout) + .field("load_op", &self.load_op) + .field("store_op", &self.store_op) + .field("clear_value", &"") + .field("range", &self.range) + .finish() + } +} + +impl ColorAttachment { + /// Builds an attachment around a caller-supplied view of an imported image. + pub fn imported( + texture: Texture, + view: Arc, + layout: vk::ImageLayout, + load_op: vk::AttachmentLoadOp, + store_op: vk::AttachmentStoreOp, + clear_value: vk::ClearValue, + ) -> Self { + let range = view.subresource_range(); + Self { + texture, + view: AttachmentView::Imported(view), + layout, + load_op, + store_op, + clear_value, + range, + } + } + + /// Builds an attachment whose view will be created after graph resolution. + pub fn created( + texture: Texture, + desc: ImageViewDesc, + layout: vk::ImageLayout, + load_op: vk::AttachmentLoadOp, + store_op: vk::AttachmentStoreOp, + clear_value: vk::ClearValue, + ) -> Self { + let range = image_view_desc_range(&desc); + Self { + texture, + view: AttachmentView::Created(desc), + layout, + load_op, + store_op, + clear_value, + range, + } + } +} + +/// A depth/stencil attachment used by dynamic rendering. The view's aspect mask +/// selects which depth/stencil attachment slots receive the view. +#[derive(Debug, Clone)] +pub struct DepthStencilAttachment { + pub texture: Texture, + pub view: AttachmentView, + pub layout: vk::ImageLayout, + pub depth_load_op: vk::AttachmentLoadOp, + pub depth_store_op: vk::AttachmentStoreOp, + pub stencil_load_op: vk::AttachmentLoadOp, + pub stencil_store_op: vk::AttachmentStoreOp, + pub clear_depth: f32, + pub clear_stencil: u32, + pub range: vk::ImageSubresourceRange, +} + +impl DepthStencilAttachment { + /// Builds an attachment around a caller-supplied view of an imported image. + pub fn imported( + texture: Texture, + view: Arc, + layout: vk::ImageLayout, + depth_load_op: vk::AttachmentLoadOp, + depth_store_op: vk::AttachmentStoreOp, + stencil_load_op: vk::AttachmentLoadOp, + stencil_store_op: vk::AttachmentStoreOp, + clear_depth: f32, + clear_stencil: u32, + ) -> Self { + let range = view.subresource_range(); + Self { + texture, + view: AttachmentView::Imported(view), + layout, + depth_load_op, + depth_store_op, + stencil_load_op, + stencil_store_op, + clear_depth, + clear_stencil, + range, + } + } + + /// Builds an attachment whose view will be created after graph resolution. + pub fn created( + texture: Texture, + desc: ImageViewDesc, + layout: vk::ImageLayout, + depth_load_op: vk::AttachmentLoadOp, + depth_store_op: vk::AttachmentStoreOp, + stencil_load_op: vk::AttachmentLoadOp, + stencil_store_op: vk::AttachmentStoreOp, + clear_depth: f32, + clear_stencil: u32, + ) -> Self { + let range = image_view_desc_range(&desc); + Self { + texture, + view: AttachmentView::Created(desc), + layout, + depth_load_op, + depth_store_op, + stencil_load_op, + stencil_store_op, + clear_depth, + clear_stencil, + range, + } + } +} + +fn image_view_desc_range(desc: &ImageViewDesc) -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange::default() + .aspect_mask(desc.aspect) + .base_mip_level(desc.mip_range.start) + .level_count(if desc.mip_range.end == vk::REMAINING_MIP_LEVELS { + vk::REMAINING_MIP_LEVELS + } else { + desc.mip_range.end - desc.mip_range.start + }) + .base_array_layer(desc.layer_range.start) + .layer_count(if desc.layer_range.end == vk::REMAINING_ARRAY_LAYERS { + vk::REMAINING_ARRAY_LAYERS + } else { + desc.layer_range.end - desc.layer_range.start + }) +} + +/// A borrowed summary of a render-pass attachment configuration. +#[derive(Debug, Clone, Copy)] +pub struct RenderPassDesc<'a> { + pub color: &'a [ColorAttachment], + pub depth: Option<&'a DepthStencilAttachment>, +} + +/// Begins a dynamic-rendering instance. +#[derive(Debug, Clone)] +pub struct BeginRendering { + pub color: Vec, + pub depth: Option, +} + +/// Ends a dynamic-rendering instance. +#[derive(Debug, Clone, Copy, Default)] +pub struct EndRendering; + +#[derive(Default)] +struct RenderingScratch { + color_attachments: Vec>, +} + +thread_local! { + static RENDERING_SCRATCH: RefCell = RefCell::new(RenderingScratch::default()); +} + +fn resolve_attachment_view( + texture: Texture, + view: &AttachmentView, + recorder: &CommandRecorder<'_>, +) -> Result<(vk::ImageView, vk::Extent2D), RecordError> { + recorder.image(texture.id())?; + let view = match view { + AttachmentView::Created(desc) => recorder.view(texture.id(), desc)?, + AttachmentView::Imported(view) => view.clone(), + }; + let range = view.subresource_range(); + let image_extent = view.image().extent_2d(); + let mip = range.base_mip_level.min(u32::BITS - 1); + let extent = vk::Extent2D { + width: (image_extent.width >> mip).max(1), + height: (image_extent.height >> mip).max(1), + }; + Ok((view.raw(), extent)) +} + +fn rendering_extent(current: Option, next: vk::Extent2D) -> vk::Extent2D { + current.map_or(next, |current| vk::Extent2D { + width: current.width.min(next.width), + height: current.height.min(next.height), + }) +} + /// A recorded operation declares its accesses before compilation and applies /// itself to a Vulkan command buffer after resource binding. pub trait Command: RecordedCommand { @@ -59,6 +299,10 @@ impl Command for CopyBuffer { } impl RecordedCommand for CopyBuffer { + fn kind(&self) -> CommandKind { + CommandKind::Transfer + } + fn apply(&mut self, recorder: &mut CommandRecorder<'_>) -> Result<(), RecordError> { let region = vk::BufferCopy2::default() .src_offset(self.src_range.offset) @@ -104,6 +348,10 @@ impl Command for ClearTexture { } impl RecordedCommand for ClearTexture { + fn kind(&self) -> CommandKind { + CommandKind::Transfer + } + fn apply(&mut self, recorder: &mut CommandRecorder<'_>) -> Result<(), RecordError> { let value = vk::ClearColorValue { float32: self.color, @@ -219,6 +467,14 @@ impl Command for Draw { } impl RecordedCommand for Draw { + fn kind(&self) -> CommandKind { + CommandKind::Graphics + } + + fn allowed_in_render_pass(&self) -> bool { + true + } + fn preflight(&self) -> Result<(), RecordError> { validate_managed_descriptor_sets(&self.descriptor_sets).map(|_| ()) } @@ -335,6 +591,160 @@ impl RecordedCommand for Draw { } } +impl Command for BeginRendering { + fn side_effects(&self, command_index: CommandIndex, map: &mut SideEffectMap) { + for attachment in &self.color { + let mut mask = vk::AccessFlags2::COLOR_ATTACHMENT_WRITE; + if attachment.load_op == vk::AttachmentLoadOp::LOAD { + mask |= vk::AccessFlags2::COLOR_ATTACHMENT_READ; + } + map.insert( + command_index, + attachment.texture.id(), + access( + vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT, + mask, + SubresourceScopeInner::Texture { + range: attachment.range, + layout: attachment.layout, + }, + ), + ); + } + + if let Some(attachment) = &self.depth { + let mut mask = vk::AccessFlags2::DEPTH_STENCIL_ATTACHMENT_WRITE; + if attachment.depth_load_op == vk::AttachmentLoadOp::LOAD + || attachment.stencil_load_op == vk::AttachmentLoadOp::LOAD + { + mask |= vk::AccessFlags2::DEPTH_STENCIL_ATTACHMENT_READ; + } + map.insert( + command_index, + attachment.texture.id(), + access( + vk::PipelineStageFlags2::EARLY_FRAGMENT_TESTS + | vk::PipelineStageFlags2::LATE_FRAGMENT_TESTS, + mask, + SubresourceScopeInner::Texture { + range: attachment.range, + layout: attachment.layout, + }, + ), + ); + } + } +} + +impl RecordedCommand for BeginRendering { + fn kind(&self) -> CommandKind { + CommandKind::Begin + } + + fn render_pass_desc(&self) -> Option> { + Some(RenderPassDesc { + color: &self.color, + depth: self.depth.as_ref(), + }) + } + + fn apply(&mut self, recorder: &mut CommandRecorder<'_>) -> Result<(), RecordError> { + RENDERING_SCRATCH.with(|scratch| { + let mut scratch = scratch.borrow_mut(); + let attachments = &mut scratch.color_attachments; + attachments.clear(); + + let mut extent = None; + for attachment in &self.color { + let (view, attachment_extent) = + resolve_attachment_view(attachment.texture, &attachment.view, recorder)?; + extent = Some(rendering_extent(extent, attachment_extent)); + attachments.push( + vk::RenderingAttachmentInfo::default() + .image_view(view) + .image_layout(attachment.layout) + .load_op(attachment.load_op) + .store_op(attachment.store_op) + .clear_value(attachment.clear_value), + ); + } + + let mut depth_info = None; + let mut stencil_info = None; + if let Some(attachment) = &self.depth { + let (view, attachment_extent) = + resolve_attachment_view(attachment.texture, &attachment.view, recorder)?; + extent = Some(rendering_extent(extent, attachment_extent)); + let clear_value = vk::ClearValue { + depth_stencil: vk::ClearDepthStencilValue { + depth: attachment.clear_depth, + stencil: attachment.clear_stencil, + }, + }; + if attachment.range.aspect_mask.contains(vk::ImageAspectFlags::DEPTH) { + depth_info = Some( + vk::RenderingAttachmentInfo::default() + .image_view(view) + .image_layout(attachment.layout) + .load_op(attachment.depth_load_op) + .store_op(attachment.depth_store_op) + .clear_value(clear_value), + ); + } + if attachment.range.aspect_mask.contains(vk::ImageAspectFlags::STENCIL) { + stencil_info = Some( + vk::RenderingAttachmentInfo::default() + .image_view(view) + .image_layout(attachment.layout) + .load_op(attachment.stencil_load_op) + .store_op(attachment.stencil_store_op) + .clear_value(clear_value), + ); + } + } + + let extent = extent.unwrap_or(vk::Extent2D { + width: 0, + height: 0, + }); + let mut info = vk::RenderingInfo::default() + .render_area(vk::Rect2D { + offset: vk::Offset2D { x: 0, y: 0 }, + extent, + }) + .layer_count(1) + .color_attachments(attachments); + if let Some(depth) = depth_info.as_ref() { + info = info.depth_attachment(depth); + } + if let Some(stencil) = stencil_info.as_ref() { + info = info.stencil_attachment(stencil); + } + unsafe { + recorder + .device + .cmd_begin_rendering(recorder.command_buffer, &info) + }; + Ok(()) + }) + } +} + +impl Command for EndRendering { + fn side_effects(&self, _command_index: CommandIndex, _map: &mut SideEffectMap) {} +} + +impl RecordedCommand for EndRendering { + fn kind(&self) -> CommandKind { + CommandKind::End + } + + fn apply(&mut self, recorder: &mut CommandRecorder<'_>) -> Result<(), RecordError> { + unsafe { recorder.device.cmd_end_rendering(recorder.command_buffer) }; + Ok(()) + } +} + fn validate_managed_descriptor_sets( sets: &[GraphDescriptorSet], ) -> Result, RecordError> { diff --git a/crates/renderer_rendergraph/src/graph.rs b/crates/renderer_rendergraph/src/graph.rs index 2f16613..cf81e2c 100644 --- a/crates/renderer_rendergraph/src/graph.rs +++ b/crates/renderer_rendergraph/src/graph.rs @@ -15,7 +15,7 @@ use std::{ cmp::Reverse, - collections::{BTreeSet, BinaryHeap, HashMap, VecDeque}, + collections::{BTreeSet, BinaryHeap, HashMap, HashSet, VecDeque}, sync::Arc, }; @@ -26,14 +26,18 @@ use renderer_resources::{ buffers::{Buffer as BufferResource, BufferDesc}, descriptors::DescriptorResourceBinding, device::{AllocationStrategy, MemoryLocation, QueueFlags}, - images::{Image as ImageResource, ImageDesc, ImageSubresourceExt}, + images::{Image as ImageResource, ImageDesc, ImageSubresourceExt, ImageView, ImageViewDesc}, pipeline::DescriptorSet, }; use renderer_device::QueueFamilyIndex; +use crate::graph_compiler::{Batch, CommandOrderings, build_command_orderings}; use crate::{ - recorder::{CommandIndex, CommandList, CommandRecorder, RecordError, scope_range}, + commands::{AttachmentView, CommandKind, RenderPassDesc}, + recorder::{ + CommandIndex, CommandList, CommandRecorder, RecordError, SideEffectMap, scope_range, + }, resources::{ Buffer, BufferRange, DescriptorResourceAccess, DescriptorResourceIdentity, GraphBufferDesc, GraphDescriptorSet, GraphTextureDesc, ResourceDesc, ResourceId, ResourceState, Subresource, @@ -51,6 +55,17 @@ pub enum GraphError { ConflictingDrawLayouts(ResourceId), ConflictingDescriptorInitialStates, ConflictingDescriptorDrawLayouts, + UnbalancedRenderPass, + NestedRenderPass, + DrawOutsideRenderPass(CommandIndex), + CommandNotAllowedInRenderPass(CommandIndex), + InvalidRenderPassAttachment(String), + /// An image layout transition would need to occur inside dynamic rendering. + UnsupportedIntraPassDependency, + /// An interior access overlaps a render-pass attachment subresource. + RenderPassFeedbackLoop(String), + /// A created attachment view could not be constructed or did not match its image. + ViewCreationFailed(String), PlacementFailed(ResourceId), BindFailed(ResourceId), InvalidPooledImageLayout(ResourceId), @@ -81,10 +96,20 @@ pub struct QueueTransfer { pub dst: QueueFamilyIndex, } +/// Identifies what orders a barrier before its destination command. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BarrierOrigin { + /// Transition from the resource's declared initial state (or first use of a created resource). + Initial, + /// Ordered after the command that previously accessed the resource. + Command(CommandIndex), +} + #[derive(Debug, Clone, Copy)] pub struct BarrierResource { pub id: ResourceId, pub range: Subresource, + pub origin: BarrierOrigin, pub src_stage: vk::PipelineStageFlags2, pub src_access: vk::AccessFlags2, pub dst_stage: vk::PipelineStageFlags2, @@ -94,6 +119,276 @@ pub struct BarrierResource { pub queue_transfer: Option, } +// TinyVec initializes its inline backing array with defaults; slots beyond its +// logical length remain inaccessible to barrier consumers. +impl Default for BarrierResource { + fn default() -> Self { + Self { + id: ResourceId::from_u32(0), + range: Subresource::Buffer(crate::resources::BufferRange { offset: 0, size: 0 }), + origin: BarrierOrigin::Initial, + src_stage: vk::PipelineStageFlags2::NONE, + src_access: vk::AccessFlags2::NONE, + dst_stage: vk::PipelineStageFlags2::NONE, + dst_access: vk::AccessFlags2::NONE, + old_layout: None, + new_layout: None, + queue_transfer: None, + } + } +} + +#[derive(Debug, Clone, Copy)] +struct LayoutRegion { + range: vk::ImageSubresourceRange, + layout: vk::ImageLayout, +} + +fn image_range_intersection( + a: vk::ImageSubresourceRange, + b: vk::ImageSubresourceRange, +) -> Option { + let aspect_mask = a.aspect_mask & b.aspect_mask; + if aspect_mask.is_empty() { + return None; + } + + let mip_start = u64::from(a.base_mip_level.max(b.base_mip_level)); + let mip_end = (u64::from(a.base_mip_level) + u64::from(a.level_count)) + .min(u64::from(b.base_mip_level) + u64::from(b.level_count)); + let layer_start = u64::from(a.base_array_layer.max(b.base_array_layer)); + let layer_end = (u64::from(a.base_array_layer) + u64::from(a.layer_count)) + .min(u64::from(b.base_array_layer) + u64::from(b.layer_count)); + if mip_start >= mip_end || layer_start >= layer_end { + return None; + } + + Some( + vk::ImageSubresourceRange::default() + .aspect_mask(aspect_mask) + .base_mip_level(mip_start as u32) + .level_count((mip_end - mip_start) as u32) + .base_array_layer(layer_start as u32) + .layer_count((layer_end - layer_start) as u32), + ) +} + +fn image_range_contains( + range: vk::ImageSubresourceRange, + other: vk::ImageSubresourceRange, +) -> bool { + range.aspect_mask.contains(other.aspect_mask) + && range.base_mip_level <= other.base_mip_level + && u64::from(range.base_mip_level) + u64::from(range.level_count) + >= u64::from(other.base_mip_level) + u64::from(other.level_count) + && range.base_array_layer <= other.base_array_layer + && u64::from(range.base_array_layer) + u64::from(range.layer_count) + >= u64::from(other.base_array_layer) + u64::from(other.layer_count) +} + +struct ImageRangeRemainders { + ranges: [Option; 5], + len: usize, +} + +impl ImageRangeRemainders { + fn new() -> Self { + Self { + ranges: [None; 5], + len: 0, + } + } + + fn push(&mut self, range: vk::ImageSubresourceRange) { + self.ranges[self.len] = Some(range); + self.len += 1; + } + + fn iter(self) -> impl Iterator { + self.ranges.into_iter().take(self.len).flatten() + } +} + +/// Subtracts the cut's aspect/mip/layer coverage, returning disjoint remainder ranges. +fn subtract_image_range( + range: vk::ImageSubresourceRange, + cut: vk::ImageSubresourceRange, +) -> ImageRangeRemainders { + let mut remainder = ImageRangeRemainders::new(); + let Some(intersection) = image_range_intersection(range, cut) else { + remainder.push(range); + return remainder; + }; + + let other_aspects = range.aspect_mask & !intersection.aspect_mask; + if !other_aspects.is_empty() { + remainder.push(vk::ImageSubresourceRange { + aspect_mask: other_aspects, + ..range + }); + } + + let aspects = intersection.aspect_mask; + let range_mip_start = u64::from(range.base_mip_level); + let range_mip_end = range_mip_start + u64::from(range.level_count); + let range_layer_start = u64::from(range.base_array_layer); + let range_layer_end = range_layer_start + u64::from(range.layer_count); + let cut_mip_start = u64::from(intersection.base_mip_level); + let cut_mip_end = cut_mip_start + u64::from(intersection.level_count); + let cut_layer_start = u64::from(intersection.base_array_layer); + let cut_layer_end = cut_layer_start + u64::from(intersection.layer_count); + let rect = |mip_start: u64, mip_end: u64, layer_start: u64, layer_end: u64| { + (mip_start < mip_end && layer_start < layer_end).then(|| { + vk::ImageSubresourceRange::default() + .aspect_mask(aspects) + .base_mip_level(mip_start as u32) + .level_count((mip_end - mip_start) as u32) + .base_array_layer(layer_start as u32) + .layer_count((layer_end - layer_start) as u32) + }) + }; + + // Four non-overlapping strips cover the rectangle around the intersection. + for piece in [ + rect( + range_mip_start, + cut_mip_start, + range_layer_start, + range_layer_end, + ), + rect( + cut_mip_end, + range_mip_end, + range_layer_start, + range_layer_end, + ), + rect( + cut_mip_start, + cut_mip_end, + range_layer_start, + cut_layer_start, + ), + rect(cut_mip_start, cut_mip_end, cut_layer_end, range_layer_end), + ] + .into_iter() + .flatten() + { + remainder.push(piece); + } + remainder +} + +fn update_layout_regions( + regions: &mut Vec, + range: vk::ImageSubresourceRange, + layout: vk::ImageLayout, +) { + let intersects = regions + .iter() + .filter(|region| image_range_intersection(region.range, range).is_some()) + .count(); + if intersects == 0 { + regions.push(LayoutRegion { range, layout }); + return; + } + + if regions + .iter() + .any(|region| same_image_subresource_range(region.range, range) && region.layout == layout) + { + return; + } + + if let Some(region) = regions + .iter_mut() + .find(|region| image_range_contains(region.range, range)) + { + if region.layout == layout { + return; + } + if same_image_subresource_range(region.range, range) { + region.layout = layout; + return; + } + } + + if regions + .iter() + .any(|region| image_range_contains(region.range, range) && region.layout == layout) + { + return; + } + + // Replacing fully covered regions in place avoids draining and rebuilding the + // common case where a transition supersedes existing layout coverage. + if regions + .iter() + .filter(|region| image_range_intersection(region.range, range).is_some()) + .all(|region| image_range_contains(range, region.range)) + { + regions.retain(|region| image_range_intersection(region.range, range).is_none()); + regions.push(LayoutRegion { range, layout }); + return; + } + + let mut updated = Vec::with_capacity(regions.len() + 4); + for region in regions.drain(..) { + for remainder in subtract_image_range(region.range, range).iter() { + updated.push(LayoutRegion { + range: remainder, + layout: region.layout, + }); + } + } + updated.push(LayoutRegion { range, layout }); + *regions = updated; +} + +fn layout_pieces( + range: vk::ImageSubresourceRange, + fallback: vk::ImageLayout, + regions: &[LayoutRegion], +) -> Vec<(vk::ImageSubresourceRange, vk::ImageLayout)> { + if !regions + .iter() + .any(|region| image_range_intersection(range, region.range).is_some()) + { + return vec![(range, fallback)]; + } + if let Some(region) = regions + .iter() + .find(|region| image_range_contains(region.range, range)) + { + return vec![(range, region.layout)]; + } + + let mut uncovered = Vec::with_capacity(5); + uncovered.push(range); + let mut next_uncovered = Vec::with_capacity(5); + let mut pieces = Vec::with_capacity(regions.len() + 1); + for region in regions { + let Some(intersection) = image_range_intersection(range, region.range) else { + continue; + }; + pieces.push((intersection, region.layout)); + next_uncovered.clear(); + for piece in uncovered.drain(..) { + next_uncovered.extend(subtract_image_range(piece, intersection).iter()); + } + std::mem::swap(&mut uncovered, &mut next_uncovered); + } + + if uncovered.is_empty() + && let Some((_, uniform_layout)) = pieces.first().copied() + && pieces.iter().all(|(_, layout)| *layout == uniform_layout) + { + return vec![(range, uniform_layout)]; + } + + pieces.extend(uncovered.into_iter().map(|piece| (piece, fallback))); + pieces +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct MemoryBarrier { pub src_stage: vk::PipelineStageFlags2, @@ -106,7 +401,26 @@ pub struct MemoryBarrier { pub struct PipelineBarrier { pub before_command: CommandIndex, pub memory_barriers: Vec, - pub resources: Vec, + /// Inline storage avoids heap allocation for the common single-resource barrier; + /// capacity two still lets draw-input barriers spill only when they have 3+ resources. + pub resources: tinyvec::TinyVec<[BarrierResource; 2]>, +} + +/// Command ids are dense `0..command_count`, so barrier-index groups use direct vector indexing. +struct BarriersByCommand { + entries: Vec>, +} + +impl BarriersByCommand { + fn new(command_count: usize) -> Self { + Self { + entries: vec![Vec::new(); command_count], + } + } + + fn get(&self, command: CommandIndex) -> &[usize] { + &self.entries[command.index()] + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -136,6 +450,393 @@ pub struct ResourceUse { pub access: vk::AccessFlags2, } +/// Resource scope collapsed to one entry per contracted DAG node. +/// +/// The representative commands are both real accesses: the first is used to key +/// initial-use barriers, while the last keys terminal-use ordering checks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct CollapsedUse { + node: usize, + first_command: CommandIndex, + last_command: CommandIndex, + stage: vk::PipelineStageFlags2, + access: vk::AccessFlags2, +} + +/// Per-resource terminal and initial uses cached for placement and alias barriers. +#[derive(Debug, Default, Clone)] +pub struct ResourceFrontiers { + uses: Vec, Vec)>>, + collapsed: Vec, Vec)>>, + command_level_alias_frontiers: bool, +} + +#[derive(Clone, Copy)] +enum AliasUse<'a> { + Command(&'a ResourceUse), + Collapsed(&'a CollapsedUse), +} + +impl AliasUse<'_> { + fn first_command(self) -> CommandIndex { + match self { + Self::Command(use_) => use_.command, + Self::Collapsed(use_) => use_.first_command, + } + } + + fn last_command(self) -> CommandIndex { + match self { + Self::Command(use_) => use_.command, + Self::Collapsed(use_) => use_.last_command, + } + } + + fn stage(self) -> vk::PipelineStageFlags2 { + match self { + Self::Command(use_) => use_.stage, + Self::Collapsed(use_) => use_.stage, + } + } + + fn access(self) -> vk::AccessFlags2 { + match self { + Self::Command(use_) => use_.access, + Self::Collapsed(use_) => use_.access, + } + } +} + +enum AliasUses<'a> { + Command(&'a [ResourceUse]), + Collapsed(&'a [CollapsedUse]), +} + +enum AliasUseIter<'a> { + Command(std::slice::Iter<'a, ResourceUse>), + Collapsed(std::slice::Iter<'a, CollapsedUse>), +} + +impl<'a> Iterator for AliasUseIter<'a> { + type Item = AliasUse<'a>; + + fn next(&mut self) -> Option { + match self { + Self::Command(iter) => iter.next().map(AliasUse::Command), + Self::Collapsed(iter) => iter.next().map(AliasUse::Collapsed), + } + } +} + +impl<'a> AliasUses<'a> { + fn iter(&self) -> AliasUseIter<'a> { + match self { + Self::Command(uses) => AliasUseIter::Command(uses.iter()), + Self::Collapsed(uses) => AliasUseIter::Collapsed(uses.iter()), + } + } + + fn is_empty(&self) -> bool { + match self { + Self::Command(uses) => uses.is_empty(), + Self::Collapsed(uses) => uses.is_empty(), + } + } +} + +impl ResourceFrontiers { + /// Computes frontiers directly from command-level uses when there are no + /// render-pass spans. With identity command-to-node mapping, each command + /// use is one collapsed use, so this produces the same frontiers as the + /// contracted-node path without constructing a copy of every use. + fn from_command_uses( + uses_by_resource: &HashMap>, + happens_before: &HappensBefore, + ) -> Self { + let mut uses_frontiers: Vec, Vec)>> = + vec![None; resource_frontier_len(uses_by_resource)]; + for (&id, uses) in uses_by_resource { + let terminal_uses = terminal_uses(uses, happens_before); + let initial_uses = initial_uses(uses, happens_before); + uses_frontiers[id.index()] = Some(( + terminal_uses.iter().map(|use_| **use_).collect(), + initial_uses.iter().map(|use_| **use_).collect(), + )); + } + + Self { + uses: uses_frontiers, + collapsed: Vec::new(), + command_level_alias_frontiers: true, + } + } + + /// Computes owned terminal/initial use sets once for each resource. + pub fn from_uses( + uses_by_resource: &HashMap>, + happens_before: &HappensBefore, + ) -> Self { + let collapsed_uses = collapse_uses(uses_by_resource, &happens_before.node_of_command); + let collapsed = Self::frontier_map(&collapsed_uses, happens_before); + let mut uses: Vec, Vec)>> = + vec![None; resource_frontier_len(uses_by_resource)]; + for (&id, resource_uses) in uses_by_resource { + let terminal = terminal_uses(resource_uses, happens_before) + .into_iter() + .copied() + .collect(); + let initial = initial_uses(resource_uses, happens_before) + .into_iter() + .copied() + .collect(); + uses[id.index()] = Some((terminal, initial)); + } + Self { + uses, + collapsed, + command_level_alias_frontiers: false, + } + } + + /// Computes frontiers from uses already collapsed by contracted DAG node. + fn from_collapsed_uses( + uses_by_resource: &HashMap>, + happens_before: &HappensBefore, + ) -> Self { + let collapsed = Self::frontier_map(uses_by_resource, happens_before); + let mut uses: Vec, Vec)>> = + vec![None; resource_frontier_len(uses_by_resource)]; + for (index, frontier) in collapsed.iter().enumerate() { + if let Some((terminal, initial)) = frontier { + let terminal = terminal.iter().map(collapsed_use_as_resource_use).collect(); + let initial = initial.iter().map(collapsed_use_as_resource_use).collect(); + uses[index] = Some((terminal, initial)); + } + } + Self { + uses, + collapsed, + command_level_alias_frontiers: false, + } + } + + fn frontier_map( + uses_by_resource: &HashMap>, + happens_before: &HappensBefore, + ) -> Vec, Vec)>> { + let mut frontiers: Vec, Vec)>> = + vec![None; resource_frontier_len(uses_by_resource)]; + for (&id, uses) in uses_by_resource { + frontiers[id.index()] = Some(( + terminal_collapsed_uses(uses, happens_before), + initial_collapsed_uses(uses, happens_before), + )); + } + frontiers + } + + /// Terminal uses not ordered before another use of this resource. + pub fn terminal(&self, id: ResourceId) -> &[ResourceUse] { + self.uses + .get(id.index()) + .and_then(Option::as_ref) + .map(|(terminal, _)| terminal.as_slice()) + .unwrap_or_default() + } + + /// Initial uses not ordered after another use of this resource. + pub fn initial(&self, id: ResourceId) -> &[ResourceUse] { + self.uses + .get(id.index()) + .and_then(Option::as_ref) + .map(|(_, initial)| initial.as_slice()) + .unwrap_or_default() + } + + fn collapsed_terminal(&self, id: ResourceId) -> &[CollapsedUse] { + self.collapsed + .get(id.index()) + .and_then(Option::as_ref) + .map(|(terminal, _)| terminal.as_slice()) + .unwrap_or_default() + } + + fn collapsed_initial(&self, id: ResourceId) -> &[CollapsedUse] { + self.collapsed + .get(id.index()) + .and_then(Option::as_ref) + .map(|(_, initial)| initial.as_slice()) + .unwrap_or_default() + } + + fn alias_terminal(&self, id: ResourceId) -> AliasUses<'_> { + if self.command_level_alias_frontiers { + AliasUses::Command(self.terminal(id)) + } else { + AliasUses::Collapsed(self.collapsed_terminal(id)) + } + } + + fn alias_initial(&self, id: ResourceId) -> AliasUses<'_> { + if self.command_level_alias_frontiers { + AliasUses::Command(self.initial(id)) + } else { + AliasUses::Collapsed(self.collapsed_initial(id)) + } + } + + /// Whether every terminal use of `a` precedes every initial use of `b`. + /// Empty frontier sets are rejected rather than treated as vacuously safe. + pub fn alias_in_order( + &self, + a: ResourceId, + b: ResourceId, + happens_before: &HappensBefore, + ) -> bool { + if self.command_level_alias_frontiers { + let terminal = self.terminal(a); + let initial = self.initial(b); + return !terminal.is_empty() + && !initial.is_empty() + && terminal.iter().all(|terminal| { + initial.iter().all(|initial| { + happens_before.happens_before(terminal.command, initial.command) + }) + }); + } + + let terminal = self.alias_terminal(a); + let initial = self.alias_initial(b); + !terminal.is_empty() + && !initial.is_empty() + && terminal.iter().all(|terminal| { + initial.iter().all(|initial| { + happens_before.happens_before(terminal.last_command(), initial.first_command()) + }) + }) + } + + /// Whether either resource order permits safe placement reuse. + pub fn alias_in_order_either( + &self, + a: ResourceId, + b: ResourceId, + happens_before: &HappensBefore, + ) -> bool { + self.alias_in_order(a, b, happens_before) || self.alias_in_order(b, a, happens_before) + } +} + +/// Resource IDs are dense graph indices, so frontier slots can be queried +/// directly without hashing on the placement/interference hot path. +fn resource_frontier_len(uses_by_resource: &HashMap) -> usize { + uses_by_resource + .keys() + .map(|id| id.index()) + .max() + .map_or(0, |max_index| max_index + 1) +} + +fn collapsed_use_as_resource_use(use_: &CollapsedUse) -> ResourceUse { + ResourceUse { + command: use_.first_command, + stage: use_.stage, + access: use_.access, + } +} + +fn collapse_uses( + uses_by_resource: &HashMap>, + node_of_command: &[usize], +) -> HashMap> { + uses_by_resource + .iter() + .map(|(&id, uses)| { + let mut collapsed: Vec = Vec::new(); + for use_ in uses { + let node = node_of_command + .get(use_.command.index()) + .copied() + .unwrap_or(use_.command.index()); + if let Some(last) = collapsed.last_mut() + && last.node == node + { + last.last_command = use_.command; + last.stage |= use_.stage; + last.access |= use_.access; + } else { + collapsed.push(CollapsedUse { + node, + first_command: use_.command, + last_command: use_.command, + stage: use_.stage, + access: use_.access, + }); + } + } + (id, collapsed) + }) + .collect() +} + +/* + * Frontier analysis operates on the contracted DAG nodes. For uses in one node, + * same-node ordering remains deliberately false and the single collapsed entry + * therefore remains both an initial and terminal use. + */ +fn terminal_collapsed_uses( + uses: &[CollapsedUse], + happens_before: &HappensBefore, +) -> Vec { + uses.iter() + .filter(|candidate| { + !uses.iter().any(|other| { + candidate.node != other.node + && happens_before.happens_before(candidate.last_command, other.first_command) + }) + }) + .copied() + .collect() +} + +fn initial_collapsed_uses( + uses: &[CollapsedUse], + happens_before: &HappensBefore, +) -> Vec { + uses.iter() + .filter(|candidate| { + !uses.iter().any(|other| { + candidate.node != other.node + && happens_before.happens_before(other.last_command, candidate.first_command) + }) + }) + .copied() + .collect() +} + +/// A render-pass span in original command-recording order. +#[derive(Debug, Clone)] +pub struct RenderPassSpan { + /// Begin-rendering command index. + pub begin: CommandIndex, + /// End-rendering command index. + pub end: CommandIndex, +} + +impl RenderPassSpan { + fn interior_range(&self) -> std::ops::Range { + debug_assert!( + self.end >= self.begin, + "RenderPassSpan end must not precede begin" + ); + self.begin.index() + 1..self.end.index() + } + + fn commands(&self) -> impl Iterator + '_ { + (self.begin.index()..=self.end.index()).map(CommandIndex::from) + } +} + /// The resource-level interference graph used by transient-resource aliasing. /// An edge means no safe ordering between the resources was proven. #[derive(Debug, Clone, Default)] @@ -201,48 +902,80 @@ impl ResourceInterference { } } -/// Compact transitive-closure rows, with one bit for every command index. -/// Built once during `compile()` by walking the Kahn batches in reverse order. +/// Compact transitive-closure rows over contracted nodes, queried by command index. +/// Commands within one render-pass span map to the same node and are deliberately +/// unordered with respect to one another for aliasing and interference analysis. #[derive(Debug, Clone, Default)] pub struct HappensBefore { rows: Vec, + node_of_command: Vec, } impl HappensBefore { - fn from_dag(edges: &[Vec], batches: &[Vec]) -> Self { + fn from_dag( + edges: &[Vec], + execution_order: &[CommandIndex], + node_of_command: &[usize], + node_representatives: &[CommandIndex], + ) -> Self { let mut closure = Self { rows: (0..edges.len()) .map(|_| FixedBitSet::with_capacity(edges.len())) .collect(), + node_of_command: node_of_command.to_vec(), }; - // Process nodes in reverse topological order so each successor row is complete. - for command in batches - .iter() - .flatten() - .rev() - .map(|command| command.index()) - { - for &successor in &edges[command] { - closure.rows[command].insert(successor); - if command != successor { - let (command_row, successor_row) = if command < successor { + // The execution order restricted to representatives is a valid DAG topological + // order; process it backwards so each successor row is complete. + for &representative in execution_order.iter().rev() { + let node = closure.node_of_command[representative.index()]; + if node_representatives[node] != representative { + continue; + } + for &successor in &edges[node] { + closure.rows[node].insert(successor); + if node != successor { + let (node_row, successor_row) = if node < successor { let (before_successor, from_successor) = closure.rows.split_at_mut(successor); - (&mut before_successor[command], &from_successor[0]) + (&mut before_successor[node], &from_successor[0]) } else { - let (before_command, from_command) = closure.rows.split_at_mut(command); - (&mut from_command[0], &before_command[successor]) + let (before_node, from_node) = closure.rows.split_at_mut(node); + (&mut from_node[0], &before_node[successor]) }; - command_row.union_with(successor_row); + node_row.union_with(successor_row); } } } closure } + fn from_command_dag(edges: &[Vec], batches: &[Vec]) -> Self { + let node_of_command = (0..edges.len()).collect::>(); + let execution_order = batches.iter().flatten().copied().collect::>(); + let node_representatives = node_of_command + .iter() + .map(|&command| CommandIndex(command)) + .collect::>(); + Self::from_dag( + edges, + &execution_order, + &node_of_command, + &node_representatives, + ) + } + fn contains(&self, from: usize, to: usize) -> bool { - self.rows.get(from).is_some_and(|row| row.contains(to)) + let (Some(&from_node), Some(&to_node)) = + (self.node_of_command.get(from), self.node_of_command.get(to)) + else { + return false; + }; + from_node != to_node + && self + .rows + .get(from_node) + .is_some_and(|row| row.contains(to_node)) } /// Whether command `from` transitively precedes command `to`. @@ -251,63 +984,156 @@ impl HappensBefore { } } +bitflags::bitflags! { + #[derive(Debug, Default, Copy, Clone)] + struct KahnVertex: u8 { + const READY = 0b0000_0001; + const OCCUPIED = 0b0000_0010; + } +} + +impl KahnVertex { + fn is_ready(self) -> bool { + self.contains(KahnVertex::READY) + } + fn is_occupied(self) -> bool { + self.contains(KahnVertex::OCCUPIED) + } + + fn make_unready(&mut self) -> bool { + let was_ready = self.contains(KahnVertex::READY); + self.remove(KahnVertex::READY); + was_ready + } + + fn make_ready(&mut self) -> bool { + let was_ready = self.contains(KahnVertex::READY); + self.insert(KahnVertex::READY); + was_ready + } + fn set_occupied(&mut self) -> bool { + let was_unoccupied = !self.contains(KahnVertex::OCCUPIED); + self.insert(KahnVertex::OCCUPIED); + was_unoccupied + } +} + /// Deterministic Kahn topological-sort state for command dependencies. -struct Kahn { - edges: Vec>, +/// Ready nodes are added lazily when an occupied node has no remaining dependencies. +#[derive(Debug)] +pub(crate) struct Kahn { + pub(crate) edges: Vec>, indegree: Vec, ready: BinaryHeap>, // TODO: bitvec - is_ready: Vec, + nodes: Vec, emitted: usize, total: usize, } +/// An iterator over the next batch of ready nodes in a Kahn topological sort. +/// It drains the ready heap, skipping stale or unoccupied entries, and the caller +/// must call `complete` on each node in the batch before querying for another batch. +pub(crate) struct KahnBatchIter<'a> { + kahn: &'a mut Kahn, +} + +impl Iterator for KahnBatchIter<'_> { + type Item = usize; + + fn next(&mut self) -> Option { + while let Some(Reverse(node)) = self.kahn.ready.pop() { + if let Some(vert) = self.kahn.nodes.get_mut(node) + && vert.make_unready() + && vert.is_occupied() + { + return Some(node); + } + } + + None + } +} + impl Kahn { - fn new(node_count: usize) -> Self { + pub(crate) fn new(node_count: usize) -> Self { Self { edges: vec![Vec::new(); node_count], indegree: vec![0; node_count], - ready: (0..node_count).map(Reverse).collect(), - is_ready: vec![true; node_count], + ready: BinaryHeap::new(), + nodes: vec![KahnVertex::default(); node_count], emitted: 0, - total: node_count, + total: 0, } } - fn add_edge(&mut self, from: usize, to: usize) { + pub(crate) fn add_node(&mut self, node: usize) { + if self.nodes[node].set_occupied() { + self.total += 1; + if self.indegree[node] == 0 { + self.nodes[node].make_ready(); + self.ready.push(Reverse(node)); + } + } + } + + pub(crate) fn add_edge(&mut self, from: usize, to: usize) { let edges = &mut self.edges[from]; + + let mut newly_occupied = [false; 2]; + for (index, node) in self + .nodes + .get_disjoint_mut([from, to]) + .expect("Kahn nodes must be distinct") + .into_iter() + .enumerate() + { + if node.set_occupied() { + self.total += 1; + newly_occupied[index] = true; + } + } + + for (node, newly_occupied) in [from, to].into_iter().zip(newly_occupied) { + if newly_occupied && self.indegree[node] == 0 { + self.nodes[node].make_ready(); + self.ready.push(Reverse(node)); + } + } + if let Err(index) = edges.binary_search(&to) { edges.insert(index, to); self.indegree[to] += 1; - if self.is_ready[to] { - self.is_ready[to] = false; - } + self.nodes[to].remove(KahnVertex::READY); } } - fn pop_batch(&mut self) -> Vec { - let mut batch = Vec::new(); - while let Some(Reverse(node)) = self.ready.pop() { - if self.is_ready[node] { - self.is_ready[node] = false; - batch.push(node); - } - } - batch + pub(crate) fn iter_batch(&mut self) -> KahnBatchIter<'_> { + KahnBatchIter { kahn: self } } - fn complete(&mut self, node: usize) { + pub(crate) fn pop_batch(&mut self) -> Vec { + Vec::from_iter(self.iter_batch()) + } + + pub(crate) fn complete(&mut self, node: usize) { + if !self.nodes[node].is_occupied() { + return; + } + self.emitted += 1; for &dependent in &self.edges[node] { - self.indegree[dependent] -= 1; - if self.indegree[dependent] == 0 { - self.is_ready[dependent] = true; - self.ready.push(Reverse(dependent)); + if self.nodes[dependent].is_occupied() { + self.indegree[dependent] -= 1; + if self.indegree[dependent] == 0 { + self.nodes[dependent].make_ready(); + self.ready.push(Reverse(dependent)); + } } } } - fn is_complete(&self) -> bool { + pub(crate) fn is_complete(&self) -> bool { self.emitted == self.total } } @@ -732,7 +1558,30 @@ impl RenderGraph { /// are added by `CompiledGraph::resolve`. The closure is derived from /// resource-hazard edges; batch membership alone is not proof that uses are /// ordered. - pub fn compile(self) -> Result { + pub fn compile(mut self) -> Result { + // Keep dependency construction on the recorded accesses; the walk adds + // lifetime-only attachment accesses to `side_effects` as it proceeds. + let orderings = build_command_orderings( + &self.resources, + &self.commands.commands, + &mut self.commands.side_effects, + )?; + + let CommandOrderings { + execution_order, + batches, + spans: render_pass_spans, + node_of_command, + node_representatives, + dependency_edges, + } = orderings; + + validate_feedback_loops( + &self.commands.commands, + &self.commands.side_effects, + &render_pass_spans, + )?; + // Check for conflicting draw layouts before building the dependency graph. for accesses in self.commands.side_effects.command_accesses() { for (index, first) in accesses.iter().enumerate() { @@ -758,134 +1607,703 @@ impl RenderGraph { let command_count = self.commands.len(); let accesses = self.commands.side_effects.command_accesses(); - let compiler = GraphCompiler::new(&self.resources, accesses, command_count); - let (batches, dependency_edges) = compiler.topo_batches_and_dependencies()?; + let compiler = GraphCompiler::new_with_nodes( + &self.resources, + accesses, + command_count, + &self.commands.side_effects, + node_of_command, + node_representatives, + &render_pass_spans, + ); compiler.validate_resources()?; - let barriers = compiler.build_barriers(&batches); - let happens_before = HappensBefore::from_dag(&dependency_edges, &batches); - let uses_by_resource = compiler.resource_uses(); - let (lifetimes, interference) = - compiler.bind_lifetimes(&batches, &happens_before, &uses_by_resource); - - let ordinary_barrier_count = barriers.len(); - let mut barriers = barriers; - - compiler.build_alias_barriers( - &mut barriers, - &lifetimes, - &uses_by_resource, - &happens_before, + let mut barriers = compiler.build_barriers(&execution_order); + hoist_span_barriers(&mut barriers, &render_pass_spans, command_count); + validate_intra_pass_barriers(&barriers, &render_pass_spans)?; + let happens_before = HappensBefore::from_dag( + &dependency_edges, + &execution_order, + &compiler.node_of_command, + &compiler.node_representatives, ); + let uses_by_resource = compiler.resource_uses(); + // Span graphs need contracted uses so accesses within one pass become + // a single node. Without spans, command nodes are the identity mapping: + // command-level frontiers are therefore exactly the contracted + // frontiers, and avoid the otherwise redundant collapsed-use map. + let frontiers = if render_pass_spans.is_empty() { + let has_created_resource_uses = uses_by_resource + .keys() + .any(|id| self.resources[id.index()].is_created()); + if has_created_resource_uses { + ResourceFrontiers::from_command_uses(&uses_by_resource, &happens_before) + } else { + ResourceFrontiers::default() + } + } else { + let collapsed_uses_by_resource = compiler.collapsed_resource_uses(&uses_by_resource); + // Imported resources never enter transient packing; avoid frontier + // construction when there is no created resource that could be aliased. + let has_created_resource_uses = collapsed_uses_by_resource + .keys() + .any(|id| self.resources[id.index()].is_created()); + if !has_created_resource_uses { + ResourceFrontiers::default() + } else { + ResourceFrontiers::from_collapsed_uses(&collapsed_uses_by_resource, &happens_before) + } + }; + let (lifetimes, interference) = compiler.bind_lifetimes( + &execution_order, + &batches, + &frontiers, + &happens_before, + &uses_by_resource, + &render_pass_spans, + ); + let ordinary_barrier_count = barriers.len(); + let has_created_views = render_pass_spans.iter().any(|span| { + self.commands.commands[span.begin.index()] + .render_pass_desc() + .is_some_and(|desc| { + desc.color + .iter() + .any(|attachment| matches!(attachment.view, AttachmentView::Created(_))) + || desc.depth.is_some_and(|attachment| { + matches!(attachment.view, AttachmentView::Created(_)) + }) + }) + }); + + compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); + hoist_span_barriers(&mut barriers, &render_pass_spans, command_count); + validate_intra_pass_barriers(&barriers, &render_pass_spans)?; + + let barrier_indices_by_command = grouped_barrier_indices(&barriers, command_count); + let GraphCompiler { + node_of_command: compiled_node_of_command, + node_representatives: compiled_node_representatives, + .. + } = compiler; - let barrier_indices_by_command = grouped_barrier_indices(&barriers); Ok(CompiledGraph { resources: self.resources, resolved: HashMap::new(), + views: HashMap::new(), + created_view_bindings: Vec::new(), + has_created_views, placements_resolved: false, commands: self.commands, batches, + execution_order, barriers, barrier_indices_by_command, ordinary_barrier_count, lifetimes, interference, dependency_edges, + node_of_command: compiled_node_of_command, + node_representatives: compiled_node_representatives, happens_before, uses_by_resource, + frontiers, + render_pass_spans, }) } } +fn validate_feedback_loops( + commands: &[Box], + side_effects: &SideEffectMap, + spans: &[RenderPassSpan], +) -> Result<(), GraphError> { + for span in spans { + let Some(desc) = commands[span.begin.index()].render_pass_desc() else { + continue; + }; + let attachments = desc + .color + .iter() + .map(|attachment| (attachment.texture.id(), attachment.range)) + .chain( + desc.depth + .into_iter() + .map(|attachment| (attachment.texture.id(), attachment.range)), + ) + .collect::>(); + + for command in span.interior_range().map(CommandIndex::from) { + for access in side_effects.accesses(command) { + let SubresourceScopeInner::Texture { range, .. } = access.scope.subresource else { + continue; + }; + if attachments.iter().any(|(id, attachment_range)| { + *id == access.resource + && image_range_intersection(range, *attachment_range).is_some() + }) { + return Err(GraphError::RenderPassFeedbackLoop(format!( + "texture {:?} is accessed by command {:?} inside its render pass", + access.resource, command + ))); + } + } + } + } + Ok(()) +} + +/// Move outside-origin interior transitions to their span begin, retaining the +/// sequence of transitions so Vulkan observes chained layouts in order. +fn hoist_span_barriers( + barriers: &mut Vec, + spans: &[RenderPassSpan], + command_count: usize, +) { + if spans.is_empty() { + return; + } + + let mut hoisted = (0..command_count) + .map(|_| None::>) + .collect::>(); + + let mut hoisted_memory = (0..command_count) + .map(|_| None::>) + .collect::>(); + + let mut interior_owner = vec![None; command_count]; + for span in spans { + hoisted[span.begin.index()] = Some(Vec::new()); + hoisted_memory[span.begin.index()] = Some(Vec::new()); + for command in span.interior_range().map(CommandIndex::from) { + interior_owner[command.index()] = Some(span.begin); + } + } + + for barrier in barriers.iter_mut() { + let Some(begin) = interior_owner[barrier.before_command.index()] else { + continue; + }; + + // Legacy global memory barriers carry no per-resource origin, so retain + // their conservative pass-boundary hoisting behavior. + hoisted_memory[begin.index()] + .as_mut() + .expect("span begin has a memory-barrier entry") + .append(&mut barrier.memory_barriers); + let mut retained = tinyvec::TinyVec::<[BarrierResource; 2]>::new(); + for resource in barrier.resources.drain(..) { + let outside_origin = match resource.origin { + BarrierOrigin::Initial => true, + BarrierOrigin::Command(origin) => interior_owner[origin.index()] != Some(begin), + }; + if outside_origin { + let resources = hoisted[begin.index()] + .as_mut() + .expect("span has hoisted resources"); + resources.push(resource); + } else { + retained.push(resource); + } + } + barrier.resources = retained; + } + + barriers.retain(|barrier| !barrier.resources.is_empty() || !barrier.memory_barriers.is_empty()); + + let mut barrier_index_by_command = vec![None; command_count]; + for (index, barrier) in barriers.iter().enumerate() { + barrier_index_by_command[barrier.before_command.index()].get_or_insert(index); + } + for span in spans { + let begin = span.begin; + let resources = hoisted[begin.index()].take().unwrap_or_default(); + let memory_barriers = + coalesce_hoisted_memory(hoisted_memory[begin.index()].take().unwrap_or_default()); + let resources = coalesce_hoisted_resources(resources); + + if resources.is_empty() && memory_barriers.is_empty() { + continue; + } + + if let Some(index) = barrier_index_by_command[begin.index()] { + barriers[index].memory_barriers.extend(memory_barriers); + barriers[index].resources.extend(resources); + } else { + barriers.push(PipelineBarrier { + before_command: begin, + memory_barriers, + resources: resources.into_iter().collect(), + }); + barrier_index_by_command[begin.index()] = Some(barriers.len() - 1); + } + } +} + +fn coalesce_hoisted_resources(incoming: Vec) -> Vec { + let mut resources = Vec::new(); + let mut exact_image_indices = HashMap::new(); + let mut target_image_indices = HashMap::new(); + let mut target_range_indices: HashMap<(ResourceId, vk::ImageLayout), Vec> = + HashMap::new(); + let mut buffer_indices = HashMap::new(); + for resource in incoming { + coalesce_hoisted_image_barrier( + &mut resources, + &mut exact_image_indices, + &mut target_image_indices, + &mut target_range_indices, + &mut buffer_indices, + resource, + ); + } + resources +} + +fn coalesce_hoisted_image_barrier( + resources: &mut Vec, + exact_image_indices: &mut HashMap< + ( + ResourceId, + (u32, u32, u32, u32, u32), + vk::ImageLayout, + vk::ImageLayout, + BarrierOrigin, + ), + usize, + >, + target_image_indices: &mut HashMap< + ( + ResourceId, + (u32, u32, u32, u32, u32), + vk::ImageLayout, + BarrierOrigin, + ), + Vec, + >, + target_range_indices: &mut HashMap<(ResourceId, vk::ImageLayout), Vec>, + buffer_indices: &mut HashMap<(ResourceId, u64, u64, BarrierOrigin, u64, u64), usize>, + mut incoming: BarrierResource, +) { + let Subresource::Texture(incoming_range) = incoming.range else { + let Subresource::Buffer(incoming_range) = incoming.range else { + unreachable!() + }; + let key = ( + incoming.id, + incoming_range.offset, + incoming_range.size, + incoming.origin, + incoming.src_stage.as_raw(), + incoming.src_access.as_raw(), + ); + if let Some(&index) = buffer_indices.get(&key) { + let existing = &mut resources[index]; + existing.dst_stage |= incoming.dst_stage; + existing.dst_access |= incoming.dst_access; + } else { + buffer_indices.insert(key, resources.len()); + resources.push(incoming); + } + return; + }; + let (Some(old_layout), Some(new_layout)) = (incoming.old_layout, incoming.new_layout) else { + resources.push(incoming); + return; + }; + + let range_key = image_range_key(incoming_range); + let exact_key = ( + incoming.id, + range_key, + old_layout, + new_layout, + incoming.origin, + ); + let existing_index = exact_image_indices.get(&exact_key).copied().or_else(|| { + [new_layout, old_layout].into_iter().find_map(|layout| { + target_image_indices + .get(&(incoming.id, range_key, layout, incoming.origin)) + .and_then(|indices| { + indices.iter().copied().find(|&index| { + resources[index].new_layout == Some(new_layout) + && (resources[index].old_layout == Some(old_layout) + || resources[index].new_layout == Some(old_layout)) + }) + }) + }) + }); + if let Some(index) = existing_index { + let existing = &mut resources[index]; + existing.dst_stage |= incoming.dst_stage; + existing.dst_access |= incoming.dst_access; + return; + } + + // Keep only subresources not already transitioned to this target layout. + // Layout-region normalization normally makes these residual pieces empty or + // disjoint, but retaining subtraction here protects partially overlapping ranges. + let mut residual = vec![incoming_range]; + for &index in target_range_indices + .get(&(incoming.id, new_layout)) + .into_iter() + .flatten() + { + let existing = &resources[index]; + let Subresource::Texture(covered) = existing.range else { + continue; + }; + if same_image_subresource_range(covered, incoming_range) { + continue; + } + residual = residual + .into_iter() + .flat_map(|piece| subtract_image_range(piece, covered).iter()) + .collect(); + if residual.is_empty() { + break; + } + } + for range in residual { + incoming.range = Subresource::Texture(range); + let key = ( + incoming.id, + image_range_key(range), + old_layout, + new_layout, + incoming.origin, + ); + exact_image_indices.insert(key, resources.len()); + target_image_indices + .entry(( + incoming.id, + image_range_key(range), + old_layout, + incoming.origin, + )) + .or_default() + .push(resources.len()); + target_image_indices + .entry(( + incoming.id, + image_range_key(range), + new_layout, + incoming.origin, + )) + .or_default() + .push(resources.len()); + target_range_indices + .entry((incoming.id, new_layout)) + .or_default() + .push(resources.len()); + resources.push(incoming); + } +} + +fn image_range_key(range: vk::ImageSubresourceRange) -> (u32, u32, u32, u32, u32) { + ( + range.aspect_mask.as_raw(), + range.base_mip_level, + range.level_count, + range.base_array_layer, + range.layer_count, + ) +} + +fn coalesce_hoisted_memory(incoming: Vec) -> Vec { + let mut indices = HashMap::new(); + let mut barriers = Vec::new(); + for barrier in incoming { + let key = ( + barrier.src_stage, + barrier.src_access, + barrier.dst_stage, + barrier.dst_access, + ); + if indices.contains_key(&key) { + continue; + } + indices.insert(key, barriers.len()); + barriers.push(barrier); + } + barriers +} + +fn validate_intra_pass_barriers( + barriers: &[PipelineBarrier], + spans: &[RenderPassSpan], +) -> Result<(), GraphError> { + if spans.is_empty() { + return Ok(()); + } + + for barrier in barriers { + if is_render_pass_interior(barrier.before_command, spans) + && barrier.resources.iter().any(|resource| { + matches!((resource.old_layout, resource.new_layout), (Some(old), Some(new)) if old != new) + }) + { + return Err(GraphError::UnsupportedIntraPassDependency); + } + } + Ok(()) +} + +fn is_render_pass_interior(command: CommandIndex, spans: &[RenderPassSpan]) -> bool { + let span_index = spans.partition_point(|span| span.end < command); + spans + .get(span_index) + .is_some_and(|span| span.interior_range().contains(&command.index())) +} + +fn flatten_batches(batches: &[Vec]) -> Vec { + batches.iter().flatten().copied().collect() +} + +pub(crate) fn validate_render_pass_attachments( + resources: &[ResourceDesc], + desc: RenderPassDesc<'_>, +) -> Result<(), GraphError> { + let mut seen_subresources = HashSet::new(); + let attachments = desc + .color + .iter() + .map(|attachment| { + ( + attachment.texture, + &attachment.view, + attachment.range, + attachment.layout, + true, + ) + }) + .chain(desc.depth.into_iter().map(|attachment| { + ( + attachment.texture, + &attachment.view, + attachment.range, + attachment.layout, + false, + ) + })); + + for (texture, view, range, layout, is_color) in attachments { + let id = texture.id(); + let Some(resource) = resources.get(id.index()) else { + return Err(GraphError::UnknownResource(id)); + }; + let reason = |message: &str| { + GraphError::InvalidRenderPassAttachment(format!("texture {id:?}: {message}")) + }; + let counts = match resource { + ResourceDesc::CreatedTexture(desc) => Some((desc.mip_levels, desc.array_layers)), + ResourceDesc::ImportedTexture { .. } => None, + ResourceDesc::CreatedBuffer(_) | ResourceDesc::ImportedBuffer { .. } => { + return Err(reason("attachment resource is not a texture")); + } + }; + + if let AttachmentView::Imported(view) = view + && let ResourceDesc::ImportedTexture { resource, .. } = resource + && view.image().as_ref() != resource.as_ref() + { + return Err(reason("imported view refers to a different image")); + } + if let AttachmentView::Created(view) = view + && (view.mip_range.end == vk::REMAINING_MIP_LEVELS + || view.layer_range.end == vk::REMAINING_ARRAY_LAYERS) + { + return Err(reason("created view uses a remaining-range sentinel")); + } + if let AttachmentView::Created(view) = view + && (view.mip_range.end < view.mip_range.start + || view.layer_range.end < view.layer_range.start) + { + return Err(reason("created view range is reversed")); + } + + let view_range = match view { + AttachmentView::Imported(view) => view.subresource_range(), + AttachmentView::Created(view) => vk::ImageSubresourceRange::default() + .aspect_mask(view.aspect) + .base_mip_level(view.mip_range.start) + .level_count(view.mip_range.end - view.mip_range.start) + .base_array_layer(view.layer_range.start) + .layer_count(view.layer_range.end - view.layer_range.start), + }; + if !same_image_subresource_range(range, view_range) { + return Err(reason("declared range does not match the attachment view")); + } + + if !is_legal_attachment_layout(layout, is_color) { + return Err(reason("layout is not legal for this attachment type")); + } + if range.level_count == vk::REMAINING_MIP_LEVELS + || range.layer_count == vk::REMAINING_ARRAY_LAYERS + || range.level_count == 0 + || range.layer_count == 0 + { + return Err(reason("attachment range must be concrete and non-empty")); + } + let Some(mip_end) = range.base_mip_level.checked_add(range.level_count) else { + return Err(reason("mip range overflows")); + }; + let Some(layer_end) = range.base_array_layer.checked_add(range.layer_count) else { + return Err(reason("layer range overflows")); + }; + if let Some((mip_count, layer_count)) = counts + && (mip_end > mip_count || layer_end > layer_count) + { + return Err(reason("attachment range exceeds texture mip/layer counts")); + } + + let aspects = [ + vk::ImageAspectFlags::COLOR, + vk::ImageAspectFlags::DEPTH, + vk::ImageAspectFlags::STENCIL, + ] + .into_iter() + .filter(|aspect| range.aspect_mask.intersects(*aspect)) + .collect::>(); + let known_aspects = aspects + .iter() + .copied() + .fold(vk::ImageAspectFlags::empty(), |mask, aspect| mask | aspect); + if aspects.is_empty() || known_aspects != range.aspect_mask { + return Err(reason("attachment range has unsupported aspect bits")); + } + + for aspect in aspects { + for mip in range.base_mip_level..mip_end { + for layer in range.base_array_layer..layer_end { + if !seen_subresources.insert((id, aspect.as_raw(), mip, layer)) { + return Err(reason("duplicate attachment subresource")); + } + } + } + } + } + + Ok(()) +} + +fn same_image_subresource_range( + a: vk::ImageSubresourceRange, + b: vk::ImageSubresourceRange, +) -> bool { + a.aspect_mask == b.aspect_mask + && a.base_mip_level == b.base_mip_level + && a.level_count == b.level_count + && a.base_array_layer == b.base_array_layer + && a.layer_count == b.layer_count +} + +fn is_legal_attachment_layout(layout: vk::ImageLayout, is_color: bool) -> bool { + if is_color { + matches!( + layout, + vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL | vk::ImageLayout::GENERAL + ) + } else { + matches!( + layout, + vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL + | vk::ImageLayout::DEPTH_READ_ONLY_OPTIMAL + | vk::ImageLayout::DEPTH_STENCIL_READ_ONLY_OPTIMAL + | vk::ImageLayout::DEPTH_ATTACHMENT_STENCIL_READ_ONLY_OPTIMAL + | vk::ImageLayout::GENERAL + ) + } +} + struct GraphCompiler<'a> { resources: &'a [ResourceDesc], accesses: &'a [Vec], command_count: usize, + node_of_command: Vec, + node_representatives: Vec, + node_count: usize, + lifetime_only: BTreeSet<(usize, ResourceId)>, } impl<'a> GraphCompiler<'a> { - fn new( + fn new_with_nodes( resources: &'a [ResourceDesc], accesses: &'a [Vec], command_count: usize, + side_effects: &SideEffectMap, + node_of_command: Vec, + node_representatives: Vec, + spans: &[RenderPassSpan], ) -> Self { + let node_count = node_representatives.len(); Self { resources, accesses, command_count, + node_of_command, + node_representatives, + node_count, + lifetime_only: if spans.is_empty() { + BTreeSet::new() + } else { + accesses + .iter() + .enumerate() + .flat_map(|(command, accesses)| { + accesses.iter().filter_map(move |access| { + side_effects + .is_lifetime_only(command.into(), access.resource) + .then_some((command, access.resource)) + }) + }) + .collect() + }, } } - fn dependencies(&self) -> Kahn { - let mut kahn = Kahn::new(self.command_count); - let mut accesses_by_resource: HashMap)>> = - HashMap::new(); - - for (command, accesses) in self.accesses.iter().enumerate() { - for access in accesses { - let resource_accesses = accesses_by_resource.entry(access.resource).or_default(); - if let Some((last_command, scopes)) = resource_accesses.last_mut() - && *last_command == command - { - scopes.push(access.scope); - } else { - resource_accesses.push((command, vec![access.scope])); - } - } - } - - let mut priors_by_command = vec![Vec::new(); self.command_count]; - for resource_accesses in accesses_by_resource.values() { - for (current_position, (current, current_scopes)) in - resource_accesses.iter().enumerate() + #[cfg(test)] + fn new( + resources: &'a [ResourceDesc], + accesses: &'a [Vec], + command_count: usize, + side_effects: &SideEffectMap, + spans: &[RenderPassSpan], + ) -> Self { + let mut node_of_command = vec![usize::MAX; command_count]; + let mut node_representatives = Vec::new(); + let mut span_index = 0; + let mut command = 0; + while command < command_count { + let node = node_representatives.len(); + if let Some(span) = spans.get(span_index) + && span.begin.index() == command { - for (prior, prior_scopes) in &resource_accesses[..current_position] { - if prior_scopes.iter().any(|prior_scope| { - current_scopes - .iter() - .any(|current_scope| prior_scope.conflicts(current_scope)) - }) { - priors_by_command[*current].push(*prior); - } - } + node_representatives.push(span.begin); + node_of_command[command..=span.end.index()].fill(node); + command = span.end.index() + 1; + span_index += 1; + } else { + node_representatives.push(CommandIndex(command)); + node_of_command[command] = node; + command += 1; } } - - for current in 0..self.command_count { - priors_by_command[current].sort_unstable(); - priors_by_command[current].dedup(); - for &prior in &priors_by_command[current] { - kahn.add_edge(prior, current); - } - } - kahn + Self::new_with_nodes( + resources, + accesses, + command_count, + side_effects, + node_of_command, + node_representatives, + spans, + ) } - fn topo_batches(&self) -> Result>, GraphError> { - self.topo_batches_and_dependencies() - .map(|(batches, _)| batches) - } - - fn topo_batches_and_dependencies( - &self, - ) -> Result<(Vec>, Vec>), GraphError> { - let mut kahn = self.dependencies(); - let dependency_edges = kahn.edges.clone(); - let mut batches = Vec::new(); - while !kahn.is_complete() { - let batch = kahn.pop_batch(); - if batch.is_empty() { - return Err(GraphError::DependencyCycle); - } - let batch = batch - .into_iter() - .inspect(|&index| kahn.complete(index)) - .map(CommandIndex::from) - .collect(); - batches.push(batch); - } - Ok((batches, dependency_edges)) + fn is_lifetime_only(&self, command: CommandIndex, resource: ResourceId) -> bool { + self.lifetime_only.contains(&(command.index(), resource)) } fn validate_resources(&self) -> Result<(), GraphError> { @@ -899,45 +2317,49 @@ impl<'a> GraphCompiler<'a> { Ok(()) } - fn build_barriers(&self, batches: &[Vec]) -> Vec { + fn build_barriers(&self, execution_order: &[CommandIndex]) -> Vec { let initial_states = self .resources .iter() .map(ResourceDesc::initial_state) .collect::>(); - self.build_barriers_with_initial_states(batches, &initial_states) + self.build_barriers_with_initial_states(execution_order, &initial_states) } fn build_barriers_with_initial_states( &self, - batches: &[Vec], + execution_order: &[CommandIndex], initial_states: &[crate::resources::ResourceState], ) -> Vec { - let mut batch_indices = vec![usize::MAX; self.command_count]; - for (batch_index, batch) in batches.iter().enumerate() { - for command in batch { - batch_indices[command.index()] = batch_index; - } + let mut execution_positions = vec![usize::MAX; self.command_count]; + for (position, command) in execution_order.iter().enumerate() { + execution_positions[command.index()] = position; } - let mut result: Vec = Vec::new(); - let mut prior_accesses_by_resource: HashMap< - ResourceId, + let mut result: Vec = Vec::with_capacity(self.command_count); + let mut prior_accesses_by_resource: Vec< Vec<(CommandIndex, crate::recorder::ResourceAccess)>, - > = HashMap::new(); - let mut prior_access_summaries: HashMap = HashMap::new(); + > = (0..self.resources.len()).map(|_| Vec::new()).collect(); + let mut prior_access_summaries = (0..self.resources.len()) + .map(|_| None::) + .collect::>(); + let mut current_layouts: Vec> = + (0..self.resources.len()).map(|_| Vec::new()).collect(); - for &command_index in batches.iter().flatten() { - let mut barrier_resources = Vec::new(); + for &command_index in execution_order { + let mut barrier_resources = tinyvec::TinyVec::new(); for current in &self.accesses[command_index.index()] { + if self.is_lifetime_only(command_index, current.resource) { + continue; + } // The most recent conflicting access is the complete source scope // for this barrier; prior barrier masks are not carried forward. let read_only = crate::resources::access_flag_is_read(current.scope.access) && !access_flag_is_write(current.scope.access); let can_skip_history_scan = read_only - && prior_access_summaries - .get(¤t.resource) + && prior_access_summaries[current.resource.index()] + .as_ref() .is_some_and(|summary| { !summary.has_write && !summary.has_non_read_only @@ -949,8 +2371,7 @@ impl<'a> GraphCompiler<'a> { } }); let previous = (!can_skip_history_scan) - .then(|| prior_accesses_by_resource.get(¤t.resource)) - .flatten() + .then(|| &prior_accesses_by_resource[current.resource.index()]) .and_then(|prior_accesses| { prior_accesses .iter() @@ -961,25 +2382,48 @@ impl<'a> GraphCompiler<'a> { if let Some((previous_command, previous)) = previous { debug_assert!( - batch_indices[previous_command.index()] - < batch_indices[command_index.index()] + execution_positions[previous_command.index()] + < execution_positions[command_index.index()] ); - // Layouts follow the latest conflicting access, which is - // the state the resource actually reached most recently. - let (old_layout, new_layout) = Self::layouts(previous.scope, current.scope); + let (fallback_layout, new_layout) = + Self::layouts(previous.scope, current.scope); + let pieces = match (scope_range(current.scope), fallback_layout) { + (Subresource::Texture(range), Some(fallback)) => layout_pieces( + range, + fallback, + ¤t_layouts[current.resource.index()], + ) + .into_iter() + .map(|(range, layout)| (Subresource::Texture(range), Some(layout))) + .collect::>(), + (range, old_layout) => vec![(range, old_layout)], + }; - barrier_resources.push(BarrierResource { - id: current.resource, - range: scope_range(current.scope), - src_stage: previous.scope.stage, - src_access: previous.scope.access, - dst_stage: current.scope.stage, - dst_access: current.scope.access, - old_layout, - new_layout, - queue_transfer: None, - }); + for (range, old_layout) in pieces { + barrier_resources.push(BarrierResource { + id: current.resource, + range, + origin: BarrierOrigin::Command(previous_command), + src_stage: previous.scope.stage, + src_access: previous.scope.access, + dst_stage: current.scope.stage, + dst_access: current.scope.access, + old_layout, + new_layout, + queue_transfer: None, + }); + if let (Subresource::Texture(range), Some(old), Some(new)) = + (range, old_layout, new_layout) + && old != new + { + update_layout_regions( + &mut current_layouts[current.resource.index()], + range, + new, + ); + } + } continue; } @@ -999,17 +2443,41 @@ impl<'a> GraphCompiler<'a> { || access_flag_is_write(current.scope.access)); if layout_transition || initial_hazard { - barrier_resources.push(BarrierResource { - id: current.resource, - range: scope_range(current.scope), - src_stage: initial.stage, - src_access: initial.access, - dst_stage: current.scope.stage, - dst_access: current.scope.access, - old_layout, - new_layout, - queue_transfer: None, - }); + let pieces = match (scope_range(current.scope), old_layout) { + (Subresource::Texture(range), Some(fallback)) => layout_pieces( + range, + fallback, + ¤t_layouts[current.resource.index()], + ) + .into_iter() + .map(|(range, layout)| (Subresource::Texture(range), Some(layout))) + .collect::>(), + (range, old_layout) => vec![(range, old_layout)], + }; + for (range, old_layout) in pieces { + barrier_resources.push(BarrierResource { + id: current.resource, + range, + origin: BarrierOrigin::Initial, + src_stage: initial.stage, + src_access: initial.access, + dst_stage: current.scope.stage, + dst_access: current.scope.access, + old_layout, + new_layout, + queue_transfer: None, + }); + if let (Subresource::Texture(range), Some(old), Some(new)) = + (range, old_layout, new_layout) + && old != new + { + update_layout_regions( + &mut current_layouts[current.resource.index()], + range, + new, + ); + } + } } } @@ -1022,12 +2490,13 @@ impl<'a> GraphCompiler<'a> { } for access in &self.accesses[command_index.index()] { - prior_accesses_by_resource - .entry(access.resource) - .or_default() - .push((command_index, *access)); + if self.is_lifetime_only(command_index, access.resource) { + continue; + } + prior_accesses_by_resource[access.resource.index()].push((command_index, *access)); - let summary = prior_access_summaries.entry(access.resource).or_default(); + let summary = prior_access_summaries[access.resource.index()] + .get_or_insert_with(PriorAccessSummary::default); let is_read = crate::resources::access_flag_is_read(access.scope.access); let is_write = access_flag_is_write(access.scope.access); summary.has_write |= is_write; @@ -1053,14 +2522,12 @@ impl<'a> GraphCompiler<'a> { &self, barriers: &mut Vec, lifetimes: &HashMap, - uses_by_resource: &HashMap>, + frontiers: &ResourceFrontiers, happens_before: &HappensBefore, ) { - let mut barrier_index_by_command = HashMap::new(); + let mut barrier_index_by_command = vec![None; self.command_count]; for (index, barrier) in barriers.iter().enumerate() { - barrier_index_by_command - .entry(barrier.before_command) - .or_insert(index); + barrier_index_by_command[barrier.before_command.index()].get_or_insert(index); } let mut resources_by_pool: HashMap> = HashMap::new(); @@ -1082,10 +2549,7 @@ impl<'a> GraphCompiler<'a> { let Some(new_lifetime) = lifetimes.get(&new_id) else { continue; }; - let Some(new_uses) = uses_by_resource.get(&new_id) else { - continue; - }; - let new_first_uses = initial_uses(new_uses, happens_before); + let new_first_uses = frontiers.alias_initial(new_id); if new_first_uses.is_empty() { continue; } @@ -1108,15 +2572,15 @@ impl<'a> GraphCompiler<'a> { let old_terminal_uses = old_resources .iter() - .map(|old_id| { - uses_by_resource - .get(old_id) - .map(|uses| terminal_uses(uses, happens_before)) - .unwrap_or_default() - }) + .map(|&old_id| frontiers.alias_terminal(old_id)) .collect::>(); - for &first_use in &new_first_uses { + for first_use in new_first_uses.iter() { + let first_use_command = ResourceUse { + command: first_use.first_command(), + stage: first_use.stage(), + access: first_use.access(), + }; let mut old_stage = vk::PipelineStageFlags2::NONE; let mut old_access = vk::AccessFlags2::NONE; let mut old_image_writes = false; @@ -1128,20 +2592,21 @@ impl<'a> GraphCompiler<'a> { for old_terminal_uses in &old_terminal_uses { if old_terminal_uses.is_empty() || !old_terminal_uses.iter().all(|terminal| { - happens_before.happens_before(terminal.command, first_use.command) + happens_before + .happens_before(terminal.last_command(), first_use.first_command()) }) { continue; } - let (src_stage, src_access) = aggregate_resource_uses(old_terminal_uses); + let (src_stage, src_access) = aggregate_alias_uses(old_terminal_uses.iter()); match &self.resources[new_id.index()] { ResourceDesc::CreatedBuffer(_) => { has_old_buffer = true; merged_src_stage |= src_stage; merged_src_access |= if old_terminal_uses .iter() - .any(|use_| access_flag_is_write(use_.access)) + .any(|use_| access_flag_is_write(use_.access())) { src_access } else { @@ -1154,7 +2619,7 @@ impl<'a> GraphCompiler<'a> { old_access |= src_access; old_image_writes |= old_terminal_uses .iter() - .any(|use_| access_flag_is_write(use_.access)); + .any(|use_| access_flag_is_write(use_.access())); } ResourceDesc::ImportedBuffer { .. } | ResourceDesc::ImportedTexture { .. } => {} @@ -1165,7 +2630,7 @@ impl<'a> GraphCompiler<'a> { let barrier = pipeline_barrier_at_command_indexed( barriers, &mut barrier_index_by_command, - first_use.command, + first_use.first_command(), ); let mut found_image_barrier = false; for image_barrier in barrier @@ -1180,26 +2645,28 @@ impl<'a> GraphCompiler<'a> { } else { vk::AccessFlags2::NONE }; - image_barrier.dst_stage = first_use.stage; - image_barrier.dst_access = first_use.access; + image_barrier.dst_stage = first_use.stage(); + image_barrier.dst_access = first_use.access(); image_barrier.old_layout = Some(vk::ImageLayout::UNDEFINED); + image_barrier.origin = BarrierOrigin::Initial; } if !found_image_barrier { + let first_scope = + first_use_scope(&first_use_command, self.accesses, new_id); barrier.resources.push(BarrierResource { id: new_id, - range: scope_range(first_use_scope(first_use, self.accesses, new_id)), + range: scope_range(first_scope), + origin: BarrierOrigin::Initial, src_stage: old_stage, src_access: if old_image_writes { old_access } else { vk::AccessFlags2::NONE }, - dst_stage: first_use.stage, - dst_access: first_use.access, + dst_stage: first_use.stage(), + dst_access: first_use.access(), old_layout: Some(vk::ImageLayout::UNDEFINED), - new_layout: match first_use_scope(first_use, self.accesses, new_id) - .subresource - { + new_layout: match first_scope.subresource { SubresourceScopeInner::Texture { layout, .. } => Some(layout), SubresourceScopeInner::Buffer(_) => None, }, @@ -1212,14 +2679,46 @@ impl<'a> GraphCompiler<'a> { let barrier = pipeline_barrier_at_command_indexed( barriers, &mut barrier_index_by_command, - first_use.command, + first_use.first_command(), ); - barrier.memory_barriers.push(MemoryBarrier { - src_stage: merged_src_stage, - src_access: merged_src_access, - dst_stage: first_use.stage, - dst_access: first_use.access, - }); + // Vulkan aliasing needs one dependency from all outgoing terminal uses + // to this incoming first use. Availability (src masks) is unscoped for + // a buffer barrier, while visibility (dst masks) is scoped to its range; + // targeting the new owner is therefore sufficient. A global barrier is + // valid but strictly coarser. Keep the union from every overlapping + // occupant, including execution-only src masks for read-only terminals. + let mut found_buffer_barrier = false; + for buffer_barrier in barrier + .resources + .iter_mut() + .filter(|resource| resource.id == new_id) + { + found_buffer_barrier = true; + buffer_barrier.src_stage |= merged_src_stage; + buffer_barrier.src_access |= merged_src_access; + // Preserve the entry's destination scope and origin: a command-origin + // entry inside a pass remains there, unlike a newly added Initial entry. + } + if !found_buffer_barrier { + // Initial origin gives newly added alias dependencies the same + // span-begin hoisting behavior as the former global barriers. + // Each graph buffer owns a distinct VkBuffer handle, even when its + // allocation is pooled at Placement::offset. Barrier ranges are + // relative to that handle, so cover it fully rather than using the + // backing allocation's placement offset/size. + barrier.resources.push(BarrierResource { + id: new_id, + range: Subresource::Buffer(BufferRange::full()), + origin: BarrierOrigin::Initial, + src_stage: merged_src_stage, + src_access: merged_src_access, + dst_stage: first_use.stage(), + dst_access: first_use.access(), + old_layout: None, + new_layout: None, + queue_transfer: None, + }); + } } } } @@ -1246,19 +2745,32 @@ impl<'a> GraphCompiler<'a> { SubresourceScopeInner::Buffer(_) => None, }; - (initial.layout, current_layout) + let initial_layout = match current.subresource { + SubresourceScopeInner::Texture { .. } => { + Some(initial.layout.unwrap_or(vk::ImageLayout::UNDEFINED)) + } + SubresourceScopeInner::Buffer(_) => None, + }; + + (initial_layout, current_layout) } fn bind_lifetimes( &self, - batches: &[Vec], + execution_order: &[CommandIndex], + batches: &[Batch], + frontiers: &ResourceFrontiers, happens_before: &HappensBefore, uses_by_resource: &HashMap>, + spans: &[RenderPassSpan], ) -> (HashMap, ResourceInterference) { - let mut batch_for_command = vec![usize::MAX; self.command_count]; + let mut batch_for_node = vec![usize::MAX; self.node_count]; for (batch_index, batch) in batches.iter().enumerate() { - for command in batch { - batch_for_command[command.index()] = batch_index; + // Span interiors map back to their begin node and are always in the + // same execution batch as that begin command. + for &command in &execution_order[batch.start..batch.start + batch.count] { + let node = self.node_of_command[command.index()]; + batch_for_node[node] = batch_index; } } @@ -1268,12 +2780,13 @@ impl<'a> GraphCompiler<'a> { lifetimes .entry(access.resource) .and_modify(|lifetime: &mut ResourceLifetime| { - lifetime.first_batch = lifetime.first_batch.min(batch_for_command[command]); - lifetime.last_batch = lifetime.last_batch.max(batch_for_command[command]); + let batch = batch_for_node[self.node_of_command[command]]; + lifetime.first_batch = lifetime.first_batch.min(batch); + lifetime.last_batch = lifetime.last_batch.max(batch); }) .or_insert(ResourceLifetime { - first_batch: batch_for_command[command], - last_batch: batch_for_command[command], + first_batch: batch_for_node[self.node_of_command[command]], + last_batch: batch_for_node[self.node_of_command[command]], placement: Placement { offset: 0, size: match &self.resources[access.resource.index()] { @@ -1291,12 +2804,68 @@ impl<'a> GraphCompiler<'a> { } } - let interference = - ResourceInterference::from_resource_uses(uses_by_resource, happens_before); + let mut interference = ResourceInterference::from_frontiers(frontiers, happens_before); + + let mut pass_commands = vec![false; self.command_count]; + for span in spans { + pass_commands[span.begin.index()..=span.end.index()].fill(true); + } + + let pass_resources = uses_by_resource + .iter() + .filter_map(|(&id, uses)| { + (self.resources[id.index()].is_created() + && uses.iter().any(|use_| pass_commands[use_.command.index()])) + .then_some(id) + }) + .collect::>(); + + for (index, &a) in pass_resources.iter().enumerate() { + for &b in &pass_resources[index + 1..] { + // Different pool classes cannot share memory, so pass-boundary + // alias transitions only constrain resources of the same class. + if pool_class_for_resource(&self.resources[a.index()]) + != pool_class_for_resource(&self.resources[b.index()]) + { + continue; + } + interference.edges[a.index()].insert(b); + interference.edges[b.index()].insert(a); + } + } (lifetimes, interference) } + #[cfg(test)] + fn bind_lifetimes_for_test( + &self, + batches: &[Vec], + frontiers: &ResourceFrontiers, + happens_before: &HappensBefore, + uses_by_resource: &HashMap>, + spans: &[RenderPassSpan], + ) -> (HashMap, ResourceInterference) { + let execution_order = flatten_batches(batches); + let mut range_batches = Vec::new(); + let mut start = 0; + for batch in batches { + range_batches.push(Batch { + start, + count: batch.len(), + }); + start += batch.len(); + } + self.bind_lifetimes( + &execution_order, + &range_batches, + frontiers, + happens_before, + uses_by_resource, + spans, + ) + } + fn resource_uses(&self) -> HashMap> { let mut uses_by_resource: HashMap> = HashMap::new(); for (command, command_accesses) in self.accesses.iter().enumerate() { @@ -1318,6 +2887,14 @@ impl<'a> GraphCompiler<'a> { } uses_by_resource } + + /// Collapse each resource's command-level scopes to one first/last representative per DAG node. + fn collapsed_resource_uses( + &self, + uses_by_resource: &HashMap>, + ) -> HashMap> { + collapse_uses(uses_by_resource, &self.node_of_command) + } } #[derive(Default)] @@ -1329,43 +2906,39 @@ struct PriorAccessSummary { } impl ResourceInterference { - fn from_resource_uses( - uses_by_resource: &HashMap>, - happens_before: &HappensBefore, - ) -> Self { + fn from_frontiers(frontiers: &ResourceFrontiers, happens_before: &HappensBefore) -> Self { let mut result = Self { edges: vec![ BTreeSet::new(); - uses_by_resource - .keys() - .map(|id| id.index()) - .max() - .unwrap_or(0) - + 1 + frontiers.uses.len().max(frontiers.collapsed.len()).max(1) ], }; - let resource_ids = uses_by_resource.keys().copied().collect::>(); - let mut frontiers: Vec, Vec<&ResourceUse>)>> = - vec![None; result.edges.len()]; - for &resource in &resource_ids { - let uses = &uses_by_resource[&resource]; - frontiers[resource.index()] = Some(( - terminal_uses(uses, happens_before), - initial_uses(uses, happens_before), - )); - } - + let resource_ids = if frontiers.command_level_alias_frontiers { + frontiers + .uses + .iter() + .enumerate() + .filter_map(|(index, frontier)| { + frontier + .as_ref() + .map(|_| ResourceId::from_u32(index as u32)) + }) + .collect::>() + } else { + frontiers + .collapsed + .iter() + .enumerate() + .filter_map(|(index, frontier)| { + frontier + .as_ref() + .map(|_| ResourceId::from_u32(index as u32)) + }) + .collect::>() + }; for (index, &a) in resource_ids.iter().enumerate() { for &b in &resource_ids[index + 1..] { - let (terminal_a, initial_a) = frontiers[a.index()] - .as_ref() - .expect("resource frontier was computed"); - let (terminal_b, initial_b) = frontiers[b.index()] - .as_ref() - .expect("resource frontier was computed"); - if !resource_use_sets_alias_in_order(&terminal_a, &initial_b, happens_before) - && !resource_use_sets_alias_in_order(&terminal_b, &initial_a, happens_before) - { + if !frontiers.alias_in_order_either(a, b, happens_before) { result.edges[a.index()].insert(b); result.edges[b.index()].insert(a); } @@ -1427,10 +3000,10 @@ pub fn aliasing_permitted( ) -> bool { let terminal_a = terminal_uses(uses_a, happens_before); let initial_b = initial_uses(uses_b, happens_before); - resource_use_sets_alias_in_order(&terminal_a, &initial_b, happens_before) + resource_use_ref_slices_alias_in_order(&terminal_a, &initial_b, happens_before) } -fn resource_use_sets_alias_in_order( +fn resource_use_ref_slices_alias_in_order( terminal_a: &[&ResourceUse], initial_b: &[&ResourceUse], happens_before: &HappensBefore, @@ -1444,13 +3017,24 @@ fn resource_use_sets_alias_in_order( }) } -fn aggregate_resource_uses(uses: &[&ResourceUse]) -> (vk::PipelineStageFlags2, vk::AccessFlags2) { +fn aggregate_resource_uses(uses: &[ResourceUse]) -> (vk::PipelineStageFlags2, vk::AccessFlags2) { uses.iter().fold( (vk::PipelineStageFlags2::NONE, vk::AccessFlags2::NONE), |(stage, access), resource_use| (stage | resource_use.stage, access | resource_use.access), ) } +fn aggregate_alias_uses<'a>( + uses: impl Iterator>, +) -> (vk::PipelineStageFlags2, vk::AccessFlags2) { + uses.fold( + (vk::PipelineStageFlags2::NONE, vk::AccessFlags2::NONE), + |(stage, access), resource_use| { + (stage | resource_use.stage(), access | resource_use.access()) + }, + ) +} + fn placement_ranges_overlap(a: Placement, b: Placement) -> bool { if a.pool.is_none() || a.pool != b.pool || a.size == 0 || b.size == 0 { return false; @@ -1510,6 +3094,16 @@ fn pool_class_for_image(tiling: vk::ImageTiling) -> PoolClass { } } +fn pool_class_for_resource(resource: &ResourceDesc) -> PoolClass { + match resource { + ResourceDesc::CreatedBuffer(_) => PoolClass::Linear, + ResourceDesc::CreatedTexture(desc) => pool_class_for_image(desc.tiling), + ResourceDesc::ImportedBuffer { .. } | ResourceDesc::ImportedTexture { .. } => { + unreachable!("imported resources are not transient placement candidates") + } + } +} + fn validate_pooled_image_layout( id: ResourceId, desc: &GraphTextureDesc, @@ -1603,7 +3197,7 @@ fn first_fit_placement( occupied: &[(ResourceId, u64)], lifetimes: &HashMap, interference: &ResourceInterference, - uses_by_resource: &HashMap>, + frontiers: &ResourceFrontiers, happens_before: &HappensBefore, ) -> Option { if alignment == 0 || size == 0 { @@ -1639,14 +3233,8 @@ fn first_fit_placement( continue; } - let ordered = uses_by_resource - .get(&id) - .zip(uses_by_resource.get(&other_id)) - .is_some_and(|(new_uses, old_uses)| { - !interference.interferes(id, other_id) - && (resources_alias_in_order(new_uses, old_uses, happens_before) - || resources_alias_in_order(old_uses, new_uses, happens_before)) - }); + let ordered = !interference.interferes(id, other_id) + && frontiers.alias_in_order_either(id, other_id, happens_before); if ordered { range_index += 1; @@ -1716,7 +3304,7 @@ fn pipeline_barrier_at_command( barriers.push(PipelineBarrier { before_command: command, memory_barriers: Vec::new(), - resources: Vec::new(), + resources: tinyvec::TinyVec::new(), }); barriers.len() - 1 }); @@ -1725,14 +3313,14 @@ fn pipeline_barrier_at_command( fn pipeline_barrier_at_command_indexed<'a>( barriers: &'a mut Vec, - index_by_command: &mut HashMap, + index_by_command: &mut [Option], command: CommandIndex, ) -> &'a mut PipelineBarrier { - let index = *index_by_command.entry(command).or_insert_with(|| { + let index = *index_by_command[command.index()].get_or_insert_with(|| { barriers.push(PipelineBarrier { before_command: command, memory_barriers: Vec::new(), - resources: Vec::new(), + resources: tinyvec::TinyVec::new(), }); barriers.len() - 1 }); @@ -2007,17 +3595,25 @@ fn shader_stages_to_pipeline(stages: vk::ShaderStageFlags) -> Option, resolved: HashMap, + views: HashMap<(ResourceId, ImageViewDesc), Arc>, + created_view_bindings: Vec<(ResourceId, ImageViewDesc, Arc)>, + has_created_views: bool, placements_resolved: bool, commands: CommandList, - batches: Vec>, + batches: Vec, + execution_order: Vec, barriers: Vec, - barrier_indices_by_command: HashMap>, + barrier_indices_by_command: BarriersByCommand, ordinary_barrier_count: usize, lifetimes: HashMap, interference: ResourceInterference, dependency_edges: Vec>, + node_of_command: Vec, + node_representatives: Vec, happens_before: HappensBefore, uses_by_resource: HashMap>, + frontiers: ResourceFrontiers, + render_pass_spans: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -2050,14 +3646,38 @@ enum ResolvedResource { } impl CompiledGraph { - pub fn batches(&self) -> &[Vec] { - &self.batches + /// Topological batches of commands. Each returned slice references a + /// contiguous range of the full execution order. + pub fn batches(&self) -> impl Iterator { + self.batches + .iter() + .map(|batch| &self.execution_order[batch.start..batch.start + batch.count]) + } + + /// Full recording order, with each pass expanded as begin, interiors in + /// original record order, then end. + pub fn execution_order(&self) -> &[CommandIndex] { + &self.execution_order + } + + /// Render-pass spans in original command-recording order. + pub fn render_pass_spans(&self) -> &[RenderPassSpan] { + &self.render_pass_spans } pub fn barriers(&self) -> &[PipelineBarrier] { &self.barriers } + /// Returns a cached view created for a resolved graph texture. + pub fn created_view( + &self, + texture: ResourceId, + desc: &ImageViewDesc, + ) -> Option<&Arc> { + self.views.get(&(texture, desc.clone())) + } + pub fn resource_lifetimes(&self) -> &HashMap { &self.lifetimes } @@ -2067,7 +3687,8 @@ impl CompiledGraph { } /// Whether command `a` is transitively ordered before command `b` by - /// resource-hazard edges. Commands in the same batch are not ordered. + /// resource-hazard edges. Commands in the same render-pass span map to one + /// node and are deliberately unordered for aliasing/interference analysis. pub fn happens_before(&self, a: CommandIndex, b: CommandIndex) -> bool { self.happens_before.contains(a.index(), b.index()) } @@ -2077,7 +3698,8 @@ impl CompiledGraph { &self.happens_before } - /// The retained command adjacency used to build the compile-time closure. + /// Node-space adjacency over DAG-node representatives. Render-pass edges + /// refer to the node represented by its begin command index. pub fn dependency_edges(&self) -> &[Vec] { &self.dependency_edges } @@ -2089,20 +3711,32 @@ impl CompiledGraph { } /// Replace placement-specific alias barriers while retaining compile-time ordinary barriers. - fn rebuild_barriers(&mut self) { - let compiler = GraphCompiler::new( + fn rebuild_barriers(&mut self) -> Result<(), GraphError> { + let compiler = GraphCompiler::new_with_nodes( &self.resources, self.commands.side_effects.command_accesses(), self.commands.len(), + &self.commands.side_effects, + self.node_of_command.clone(), + self.node_representatives.clone(), + &self.render_pass_spans, ); self.barriers.truncate(self.ordinary_barrier_count); compiler.build_alias_barriers( &mut self.barriers, &self.lifetimes, - &self.uses_by_resource, + &self.frontiers, &self.happens_before, ); - self.barrier_indices_by_command = grouped_barrier_indices(&self.barriers); + hoist_span_barriers( + &mut self.barriers, + &self.render_pass_spans, + self.commands.len(), + ); + validate_intra_pass_barriers(&self.barriers, &self.render_pass_spans)?; + self.barrier_indices_by_command = + grouped_barrier_indices(&self.barriers, self.commands.len()); + Ok(()) } /// Tests whether two use sets can safely reuse a placement in the supplied @@ -2128,8 +3762,70 @@ impl CompiledGraph { /// must become semaphore edges as well as the corresponding memory barriers. pub fn resolve(&mut self, device: &Device) -> Result<(), GraphError> { if self.plan_placements(device)? { - self.rebuild_barriers(); + self.rebuild_barriers()?; } + self.resolve_created_views()?; + Ok(()) + } + + fn resolve_created_views(&mut self) -> Result<(), GraphError> { + if !self.has_created_views { + return Ok(()); + } + + let mut staged = HashMap::new(); + let mut staged_bindings = Vec::new(); + for command in &self.commands.commands { + let Some(desc) = command.render_pass_desc() else { + continue; + }; + let attachments = desc + .color + .iter() + .map(|attachment| (attachment.texture.id(), &attachment.view)) + .chain( + desc.depth + .into_iter() + .map(|attachment| (attachment.texture.id(), &attachment.view)), + ); + for (texture, view) in attachments { + let AttachmentView::Created(view_desc) = view else { + continue; + }; + if !self.lifetimes.contains_key(&texture) + || self.views.contains_key(&(texture, view_desc.clone())) + || staged.contains_key(&(texture, view_desc.clone())) + { + continue; + } + let image = match self.resources.get(texture.index()) { + Some(ResourceDesc::ImportedTexture { resource, .. }) => resource, + _ => match self.resolved.get(&texture) { + Some(ResolvedResource::Image(resource)) => resource, + _ => { + return Err(GraphError::ViewCreationFailed(format!( + "created attachment texture {texture:?} is unresolved" + ))); + } + }, + }; + let view = image + .create_view(view_desc.clone()) + .map(Arc::new) + .map_err(|error| GraphError::ViewCreationFailed(error.to_string()))?; + if view.image().as_ref() != image.as_ref() { + return Err(GraphError::ViewCreationFailed(format!( + "created view for texture {texture:?} refers to a different image" + ))); + } + staged.insert((texture, view_desc.clone()), view.clone()); + let mut binding_desc = view_desc.clone(); + binding_desc.name = None; + staged_bindings.push((texture, binding_desc, view)); + } + } + self.views.extend(staged); + self.created_view_bindings.extend(staged_bindings); Ok(()) } @@ -2155,8 +3851,16 @@ impl CompiledGraph { let command_accesses = self.commands.side_effects.command_accesses(); let mut accesses_by_resource = vec![Vec::new(); self.resources.len()]; - for access in command_accesses.iter().flatten() { - accesses_by_resource[access.resource.index()].push(*access); + for (command, accesses) in command_accesses.iter().enumerate() { + for access in accesses { + if !self + .commands + .side_effects + .is_lifetime_only(command.into(), access.resource) + { + accesses_by_resource[access.resource.index()].push(*access); + } + } } let mut staged_resources = Vec::new(); let mut staged_lifetimes = self.lifetimes.clone(); @@ -2353,7 +4057,7 @@ impl CompiledGraph { &pools[pool_index].occupied, &staged_lifetimes, &self.interference, - &self.uses_by_resource, + &self.frontiers, &self.happens_before, ) .ok_or(GraphError::PlacementFailed(member.id))?; @@ -2381,22 +4085,12 @@ impl CompiledGraph { return None; } - let is_ordered = self - .uses_by_resource - .get(&member.id) - .zip(self.uses_by_resource.get(other_id)) - .is_some_and(|(new_uses, existing_uses)| { - !self.interference.interferes(member.id, *other_id) - && (resources_alias_in_order( - new_uses, - existing_uses, - &self.happens_before, - ) || resources_alias_in_order( - existing_uses, - new_uses, - &self.happens_before, - )) - }); + let is_ordered = !self.interference.interferes(member.id, *other_id) + && self.frontiers.alias_in_order_either( + member.id, + *other_id, + &self.happens_before, + ); if !is_ordered { intersects_unsafe_lifetime = true; @@ -2532,13 +4226,20 @@ impl CompiledGraph { } } + for (texture, desc, view) in &self.created_view_bindings { + recorder + .views + .insert((*texture, desc.clone()), view.clone()); + } + // Batches currently record into one command buffer, which must be // submitted to one queue for current aliasing safety. TODO: separate // per-batch command buffers may be submitted sequentially on that queue; // parallel/cross-queue execution requires command-to-queue assignment // and semaphore edges for every cross-queue alias transition. + let mut in_render_pass = false; for step in recording_steps_cached( - &self.batches, + &self.execution_order, &self.barriers, &self.barrier_indices_by_command, ) { @@ -2546,7 +4247,21 @@ impl CompiledGraph { // TODO: Merge `PipelineBarrier.resources` across commands or a batch to reduce vkCmdPipelineBarrier2 calls. RecordingStep::Barrier(barrier) => recorder.pipeline_barrier(barrier), RecordingStep::Command(command_index) => { - self.commands.commands[command_index.index()].apply(recorder)?; + if let Err(error) = + self.commands.commands[command_index.index()].apply(recorder) + { + if std::mem::replace(&mut in_render_pass, false) { + unsafe { + recorder.device.cmd_end_rendering(recorder.command_buffer); + } + } + return Err(error); + } + match self.commands.commands[command_index.index()].kind() { + CommandKind::Begin => in_render_pass = true, + CommandKind::End => in_render_pass = false, + _ => {} + } } } } @@ -2560,48 +4275,54 @@ enum RecordingStep<'a> { } fn recording_steps<'a>( - batches: &'a [Vec], + execution_order: &'a [CommandIndex], barriers: &'a [PipelineBarrier], ) -> impl Iterator> + 'a { - let mut barriers_by_command: HashMap> = HashMap::new(); + let command_count = execution_order + .iter() + .map(|command| command.index()) + .chain( + barriers + .iter() + .map(|barrier| barrier.before_command.index()), + ) + .max() + .unwrap_or_default() + + 1; + let mut barriers_by_command: Vec> = + (0..command_count).map(|_| Vec::new()).collect(); for barrier in barriers { - barriers_by_command - .entry(barrier.before_command) - .or_default() - .push(barrier); + barriers_by_command[barrier.before_command.index()].push(barrier); } - batches.iter().flatten().flat_map(move |&command_index| { - barriers_by_command - .remove(&command_index) - .unwrap_or_default() + execution_order.iter().flat_map(move |&command_index| { + std::mem::take(&mut barriers_by_command[command_index.index()]) .into_iter() .map(RecordingStep::Barrier) .chain(std::iter::once(RecordingStep::Command(command_index))) }) } -fn grouped_barrier_indices(barriers: &[PipelineBarrier]) -> HashMap> { - let mut barriers_by_command: HashMap> = HashMap::new(); +fn grouped_barrier_indices( + barriers: &[PipelineBarrier], + command_count: usize, +) -> BarriersByCommand { + let mut barriers_by_command = BarriersByCommand::new(command_count); for (index, barrier) in barriers.iter().enumerate() { - barriers_by_command - .entry(barrier.before_command) - .or_default() - .push(index); + barriers_by_command.entries[barrier.before_command.index()].push(index); } barriers_by_command } fn recording_steps_cached<'a>( - batches: &'a [Vec], + execution_order: &'a [CommandIndex], barriers: &'a [PipelineBarrier], - barrier_indices_by_command: &'a HashMap>, + barrier_indices_by_command: &'a BarriersByCommand, ) -> impl Iterator> + 'a { - batches.iter().flatten().flat_map(move |&command_index| { + execution_order.iter().flat_map(move |&command_index| { barrier_indices_by_command - .get(&command_index) - .into_iter() - .flatten() + .get(command_index) + .iter() .map(move |&index| RecordingStep::Barrier(&barriers[index])) .chain(std::iter::once(RecordingStep::Command(command_index))) }) @@ -2613,12 +4334,18 @@ const DEDICATED_ALLOCATION_THRESHOLD: u64 = 64 * 1024 * 1024; fn accesses_for_resource( id: ResourceId, command_accesses: &[Vec], + side_effects: &SideEffectMap, ) -> Vec { command_accesses .iter() - .flatten() - .filter(|access| access.resource == id) - .copied() + .enumerate() + .flat_map(|(command, accesses)| { + accesses.iter().filter_map(move |access| { + (access.resource == id + && !side_effects.is_lifetime_only(command.into(), access.resource)) + .then_some(*access) + }) + }) .collect() } @@ -2797,7 +4524,10 @@ fn actual_queue_families(device: &Device, capabilities: QueueFlags) -> QueueFlag mod tests { use super::*; use crate::{ - commands::{ClearTexture, CopyBuffer, Draw, DrawArgs}, + commands::{ + AttachmentView, BeginRendering, ClearTexture, ColorAttachment, CopyBuffer, Draw, + DrawArgs, EndRendering, + }, resources::{ BufferRange, BufferSlice, DescriptorResourceIdentity, GraphBufferDesc, GraphTextureDesc, IndexBinding, IndexFormat, IndirectBinding, ResourceDesc, @@ -2812,6 +4542,34 @@ mod tests { pipeline::{DescriptorSet, DescriptorSetLayoutBindingDesc, Sampler}, }; + fn ordering_from_accesses( + resources: &[ResourceDesc], + accesses: &[Vec], + ) -> CommandOrderings { + let mut commands = CommandList::new(); + for command_accesses in accesses { + let command = AllowedAccess( + command_accesses + .iter() + .map(|access| (access.resource, access.scope)) + .collect(), + ); + commands.push(command); + } + build_command_orderings(resources, &commands.commands, &mut commands.side_effects).unwrap() + } + + fn topo_batches_for_test( + compiler: &GraphCompiler<'_>, + ) -> Result>, GraphError> { + let orderings = ordering_from_accesses(compiler.resources, compiler.accesses); + Ok(orderings + .batches + .iter() + .map(|batch| orderings.execution_order[batch.start..batch.start + batch.count].to_vec()) + .collect()) + } + static RECORDING_MARKER_APPLIED: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); @@ -2845,6 +4603,26 @@ mod tests { } } + struct AllowedAccess(Vec<(ResourceId, SubresourceScope)>); + + impl crate::commands::Command for AllowedAccess { + fn side_effects(&self, command_index: CommandIndex, map: &mut SideEffectMap) { + for &(resource, scope) in &self.0 { + map.insert(command_index, resource, scope); + } + } + } + + impl crate::recorder::RecordedCommand for AllowedAccess { + fn allowed_in_render_pass(&self) -> bool { + true + } + + fn apply(&mut self, _recorder: &mut CommandRecorder<'_>) -> Result<(), RecordError> { + Ok(()) + } + } + fn test_buffer(handle: u64, size: u64) -> Arc { Arc::new(BufferResource::for_testing( vk::Buffer::from_raw(handle), @@ -2868,6 +4646,25 @@ mod tests { )) } + fn empty_render_pass() -> BeginRendering { + BeginRendering { + color: Vec::new(), + depth: None, + } + } + + fn created_color_attachment(texture: Texture, layout: vk::ImageLayout) -> ColorAttachment { + ColorAttachment { + texture, + view: AttachmentView::Created(renderer_resources::images::ImageViewDesc::color_2d()), + layout, + load_op: vk::AttachmentLoadOp::DONT_CARE, + store_op: vk::AttachmentStoreOp::STORE, + clear_value: vk::ClearValue::default(), + range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), + } + } + fn test_view(image: Arc) -> Arc { Arc::new(ImageView::for_testing( vk::ImageView::from_raw(1), @@ -3002,7 +4799,7 @@ mod tests { }], ), ]); - let closure = HappensBefore::from_dag( + let closure = HappensBefore::from_command_dag( &[vec![2], vec![2], vec![]], &[ vec![CommandIndex(0), CommandIndex(1)], @@ -3025,7 +4822,7 @@ mod tests { &occupied, &lifetimes, &interference, - &uses, + &ResourceFrontiers::from_uses(&uses, &closure), &closure, ), Some(48) @@ -3067,7 +4864,7 @@ mod tests { }], ), ]); - let closure = HappensBefore::from_dag( + let closure = HappensBefore::from_command_dag( &[vec![], vec![], vec![]], &[vec![CommandIndex(0), CommandIndex(1), CommandIndex(2)]], ); @@ -3087,7 +4884,7 @@ mod tests { &occupied, &lifetimes, &interference, - &uses, + &ResourceFrontiers::from_uses(&uses, &closure), &closure, ), Some(48) @@ -3174,11 +4971,32 @@ mod tests { ResourceDesc::CreatedBuffer(GraphBufferDesc::default()), ]; let accesses = vec![vec![resource_access_for(used)]]; - let compiler = GraphCompiler::new(&resources, &accesses, 1); - let batches = compiler.topo_batches().unwrap(); - let closure = HappensBefore::from_dag(&[vec![]], &batches); + let orderings = ordering_from_accesses(&resources, &accesses); + let batches = orderings + .batches + .iter() + .map(|batch| orderings.execution_order[batch.start..batch.start + batch.count].to_vec()) + .collect::>(); + let closure = HappensBefore::from_command_dag(&[vec![]], &batches); + let compiler = GraphCompiler::new_with_nodes( + &resources, + &accesses, + 1, + &SideEffectMap::default(), + orderings.node_of_command, + orderings.node_representatives, + &orderings.spans, + ); let uses = compiler.resource_uses(); - let (lifetimes, _) = compiler.bind_lifetimes(&batches, &closure, &uses); + let frontiers = ResourceFrontiers::from_uses(&uses, &closure); + let uses_by_resource = compiler.resource_uses(); + let (lifetimes, _) = compiler.bind_lifetimes_for_test( + &batches, + &frontiers, + &closure, + &uses_by_resource, + &[], + ); assert!(!lifetimes.contains_key(&unused)); assert!(lifetimes.contains_key(&used)); assert!(!should_plan_created_resource( @@ -3560,6 +5378,10 @@ mod tests { )], )); graph.push(RecordingMarker(imported)); + graph.push(BeginRendering { + color: Vec::new(), + depth: None, + }); graph.push(Draw { args: DrawArgs::Direct { vertex_count: 3, @@ -3579,6 +5401,7 @@ mod tests { }], push_constants: Vec::new(), }); + graph.push(EndRendering); let mut compiled = graph.compile().unwrap(); assert!(compiled.resource_lifetimes()[&imported.id()].imported); @@ -3746,6 +5569,10 @@ mod tests { dst: uniform, dst_range: range, }); + graph.push(BeginRendering { + color: Vec::new(), + depth: None, + }); graph.push(Draw { args: DrawArgs::Direct { vertex_count: 3, @@ -3760,19 +5587,24 @@ mod tests { descriptor_sets: vec![destination], push_constants: Vec::new(), }); + graph.push(EndRendering); let accesses = graph.commands.side_effects.command_accesses(); assert!( derive_buffer_usage( - &accesses_for_resource(uniform.id(), accesses), + &accesses_for_resource(uniform.id(), accesses, &graph.commands.side_effects), vk::BufferUsageFlags::empty() ) .contains(vk::BufferUsageFlags::UNIFORM_BUFFER) ); let compiled = graph.compile().unwrap(); + let batches = compiled.batches().collect::>(); assert_eq!( - compiled.batches(), - &[vec![CommandIndex(0)], vec![CommandIndex(1)]] + batches, + [ + &[CommandIndex(0)][..], + &[CommandIndex(1), CommandIndex(2), CommandIndex(3)][..], + ] ); let barrier = compiled .barriers() @@ -3812,6 +5644,10 @@ mod tests { let DescriptorResourceAccess::Texture { texture, .. } = descriptor_set.accesses[0] else { panic!("sampled image maps to texture access"); }; + graph.push(BeginRendering { + color: Vec::new(), + depth: None, + }); graph.push(Draw { args: DrawArgs::Direct { vertex_count: 3, @@ -3826,16 +5662,23 @@ mod tests { descriptor_sets: vec![descriptor_set], push_constants: Vec::new(), }); + graph.push(EndRendering); let accesses = graph.commands.side_effects.command_accesses(); assert!( derive_image_usage( - &accesses_for_resource(texture.id(), accesses), + &accesses_for_resource(texture.id(), accesses, &graph.commands.side_effects), vk::ImageUsageFlags::empty() ) .contains(vk::ImageUsageFlags::SAMPLED) ); let compiled = graph.compile().unwrap(); - let barrier = &compiled.barriers()[0].resources[0]; + let barrier = compiled + .barriers() + .iter() + .find(|barrier| barrier.before_command == CommandIndex(0)) + .expect("initial sampled-image transition is hoisted before BeginRendering"); + assert_eq!(barrier.resources.len(), 1); + let barrier = &barrier.resources[0]; assert_eq!(barrier.old_layout, Some(vk::ImageLayout::GENERAL)); assert_eq!( barrier.new_layout, @@ -4038,6 +5881,30 @@ mod tests { assert!(!kahn.is_complete()); } + #[test] + fn kahn_handles_holes() { + let mut kahn = Kahn::new(6); + kahn.add_edge(3, 2); + kahn.add_edge(4, 2); + kahn.add_edge(2, 1); + kahn.add_node(5); + + assert_eq!(vec![3, 4, 5], kahn.pop_batch()); + for node in [3, 4, 5] { + kahn.complete(node); + } + assert_eq!(vec![2], kahn.pop_batch()); + for node in [2] { + kahn.complete(node); + } + assert_eq!(vec![1], kahn.pop_batch()); + for node in [1] { + kahn.complete(node); + } + dbg!(&kahn); + assert!(kahn.is_complete()); + } + #[test] fn interference_tracks_overlapping_inclusive_lifetimes() { let lifetimes = HashMap::from([ @@ -4122,12 +5989,30 @@ mod tests { vec![resource_access_for(ResourceId::from_u32(0))], vec![resource_access_for(ResourceId::from_u32(2))], ]; - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); - let (batches, edges) = compiler.topo_batches_and_dependencies().unwrap(); - let happens_before = HappensBefore::from_dag(&edges, &batches); + let orderings = ordering_from_accesses(&resources, &accesses); + let batches = orderings + .batches + .iter() + .map(|batch| orderings.execution_order[batch.start..batch.start + batch.count].to_vec()) + .collect::>(); + let happens_before = HappensBefore::from_command_dag(&orderings.dependency_edges, &batches); + let compiler = GraphCompiler::new_with_nodes( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + orderings.node_of_command, + orderings.node_representatives, + &orderings.spans, + ); let uses_by_resource = compiler.resource_uses(); - let (lifetimes, interference) = - compiler.bind_lifetimes(&batches, &happens_before, &uses_by_resource); + let (lifetimes, interference) = compiler.bind_lifetimes_for_test( + &batches, + &ResourceFrontiers::from_uses(&uses_by_resource, &happens_before), + &happens_before, + &uses_by_resource, + &[], + ); assert_eq!(lifetimes[&ResourceId::from_u32(0)].first_batch, 0); assert_eq!(lifetimes[&ResourceId::from_u32(0)].last_batch, 2); @@ -4144,7 +6029,7 @@ mod tests { vec![CommandIndex(1)], vec![CommandIndex(2)], ]; - let happens_before = HappensBefore::from_dag(&edges, &batches); + let happens_before = HappensBefore::from_command_dag(&edges, &batches); assert!(happens_before.happens_before(CommandIndex(0), CommandIndex(1))); assert!(happens_before.happens_before(CommandIndex(0), CommandIndex(2))); @@ -4158,7 +6043,7 @@ mod tests { fn same_batch_resource_uses_interfere() { let edges = vec![vec![], vec![]]; let batches = vec![vec![CommandIndex(0), CommandIndex(1)]]; - let happens_before = HappensBefore::from_dag(&edges, &batches); + let happens_before = HappensBefore::from_command_dag(&edges, &batches); let a = [ResourceUse { command: CommandIndex(0), stage: vk::PipelineStageFlags2::COPY, @@ -4174,7 +6059,10 @@ mod tests { (ResourceId::from_u32(0), a.to_vec()), (ResourceId::from_u32(1), b.to_vec()), ]); - let interference = ResourceInterference::from_resource_uses(&use_sets, &happens_before); + let interference = ResourceInterference::from_frontiers( + &ResourceFrontiers::from_uses(&use_sets, &happens_before), + &happens_before, + ); assert!(interference.interferes(ResourceId::from_u32(0), ResourceId::from_u32(1))); assert!(!resources_alias_in_order(&a, &b, &happens_before)); assert!(!resources_alias_in_order(&b, &a, &happens_before)); @@ -4189,7 +6077,7 @@ mod tests { vec![CommandIndex(0), CommandIndex(2)], vec![CommandIndex(1)], ]; - let happens_before = HappensBefore::from_dag(&edges, &batches); + let happens_before = HappensBefore::from_command_dag(&edges, &batches); let resource_a = [ResourceUse { command: CommandIndex(2), stage: vk::PipelineStageFlags2::COPY, @@ -4211,7 +6099,10 @@ mod tests { (ResourceId::from_u32(0), resource_a.to_vec()), (ResourceId::from_u32(1), resource_b.to_vec()), ]); - let interference = ResourceInterference::from_resource_uses(&use_sets, &happens_before); + let interference = ResourceInterference::from_frontiers( + &ResourceFrontiers::from_uses(&use_sets, &happens_before), + &happens_before, + ); assert!(interference.interferes(ResourceId::from_u32(0), ResourceId::from_u32(1))); } @@ -4239,9 +6130,22 @@ mod tests { resource_access_for(ResourceId::from_u32(1)), ], ]; - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); - let (batches, edges) = compiler.topo_batches_and_dependencies().unwrap(); - let happens_before = HappensBefore::from_dag(&edges, &batches); + let orderings = ordering_from_accesses(&resources, &accesses); + let batches = orderings + .batches + .iter() + .map(|batch| orderings.execution_order[batch.start..batch.start + batch.count].to_vec()) + .collect::>(); + let happens_before = HappensBefore::from_command_dag(&orderings.dependency_edges, &batches); + let compiler = GraphCompiler::new_with_nodes( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + orderings.node_of_command, + orderings.node_representatives, + &orderings.spans, + ); let uses = compiler.resource_uses(); assert!(happens_before.happens_before(CommandIndex(0), CommandIndex(2))); @@ -4281,12 +6185,7 @@ mod tests { let compiled = graph.compile().expect("graph compiles"); assert_eq!( - compiled - .batches() - .iter() - .flatten() - .copied() - .collect::>(), + compiled.batches().flatten().copied().collect::>(), vec![CommandIndex(0)] ); assert!(compiled.barriers().is_empty()); @@ -4295,6 +6194,852 @@ mod tests { assert!(!compiled.resource_lifetimes()[&src.id()].imported); } + #[test] + fn render_pass_spans_validate_classification_and_balance() { + let mut graph = RenderGraph::new(); + graph.push(BeginRendering { + color: Vec::new(), + depth: None, + }); + graph.push(EndRendering); + let compiled = graph.compile().unwrap(); + assert_eq!(compiled.render_pass_spans().len(), 1); + assert_eq!(compiled.render_pass_spans()[0].begin, CommandIndex(0)); + assert_eq!(compiled.render_pass_spans()[0].end, CommandIndex(1)); + + let mut unbalanced = RenderGraph::new(); + unbalanced.push(EndRendering); + assert!(matches!( + unbalanced.compile(), + Err(GraphError::UnbalancedRenderPass) + )); + + let mut unclosed = RenderGraph::new(); + unclosed.push(empty_render_pass()); + assert!(matches!( + unclosed.compile(), + Err(GraphError::UnbalancedRenderPass) + )); + + let mut nested = RenderGraph::new(); + nested.push(empty_render_pass()); + nested.push(empty_render_pass()); + assert!(matches!( + nested.compile(), + Err(GraphError::NestedRenderPass) + )); + } + + #[test] + fn render_pass_span_is_one_node_and_expands_in_record_order() { + let mut graph = RenderGraph::new(); + let source = graph.create_buffer(GraphBufferDesc { + size: 8, + ..Default::default() + }); + let shared = graph.create_buffer(GraphBufferDesc { + size: 8, + ..Default::default() + }); + let range = BufferRange { offset: 0, size: 8 }; + graph.push(CopyBuffer { + src: source, + src_range: range, + dst: shared, + dst_range: range, + }); + graph.push(empty_render_pass()); + graph.push(Draw { + args: DrawArgs::Direct { + vertex_count: 3, + instance_count: 1, + first_vertex: 0, + first_instance: 0, + }, + pipeline_layout: vk::PipelineLayout::null(), + vertices: Vec::new(), + index: None, + indirect: None, + descriptor_sets: Vec::new(), + push_constants: Vec::new(), + }); + graph.push(EndRendering); + graph.push(CopyBuffer { + src: source, + src_range: range, + dst: shared, + dst_range: range, + }); + graph.commands.side_effects.insert( + CommandIndex(2), + shared.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::VERTEX_SHADER, + access: vk::AccessFlags2::SHADER_READ, + subresource: SubresourceScopeInner::Buffer(range), + }, + ); + let compiled = graph.compile().expect("graph compiles"); + assert_eq!( + compiled.execution_order(), + [ + CommandIndex(0), + CommandIndex(1), + CommandIndex(2), + CommandIndex(3), + CommandIndex(4), + ] + ); + assert_eq!( + compiled.batches().collect::>(), + [ + &[CommandIndex(0)][..], + &[CommandIndex(1), CommandIndex(2), CommandIndex(3)][..], + &[CommandIndex(4)][..], + ] + ); + assert_eq!(compiled.dependency_edges().len(), 3); + assert!(compiled.happens_before(CommandIndex(0), CommandIndex(2))); + assert!(compiled.happens_before(CommandIndex(2), CommandIndex(4))); + assert!(!compiled.happens_before(CommandIndex(1), CommandIndex(2))); + assert!(!compiled.happens_before(CommandIndex(2), CommandIndex(3))); + assert_eq!(compiled.resource_lifetimes()[&shared.id()].first_batch, 0); + assert_eq!(compiled.resource_lifetimes()[&shared.id()].last_batch, 2); + } + + #[test] + fn render_pass_rejects_graphics_outside_and_transfer_inside() { + let mut outside = RenderGraph::new(); + outside.push(Draw { + args: DrawArgs::Direct { + vertex_count: 3, + instance_count: 1, + first_vertex: 0, + first_instance: 0, + }, + pipeline_layout: vk::PipelineLayout::null(), + vertices: Vec::new(), + index: None, + indirect: None, + descriptor_sets: Vec::new(), + push_constants: Vec::new(), + }); + assert!(matches!( + outside.compile(), + Err(GraphError::DrawOutsideRenderPass(CommandIndex(0))) + )); + + let mut inside = RenderGraph::new(); + let source = inside.create_buffer(GraphBufferDesc { + size: 16, + ..Default::default() + }); + let destination = inside.create_buffer(GraphBufferDesc { + size: 16, + ..Default::default() + }); + inside.push(empty_render_pass()); + inside.push(CopyBuffer { + src: source, + src_range: BufferRange { + offset: 0, + size: 16, + }, + dst: destination, + dst_range: BufferRange { + offset: 0, + size: 16, + }, + }); + inside.push(EndRendering); + assert!(matches!( + inside.compile(), + Err(GraphError::CommandNotAllowedInRenderPass(CommandIndex(1))) + )); + } + + #[test] + fn render_pass_attachment_validation_and_lifetime_access() { + let mut graph = RenderGraph::new(); + let texture = graph.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + ..Default::default() + }); + let mut begin = empty_render_pass(); + begin.color.push(created_color_attachment( + texture, + vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + )); + graph.push(begin); + graph.push(EndRendering); + let compiled = graph.compile().unwrap(); + assert!( + compiled + .commands + .side_effects + .is_lifetime_only(CommandIndex(1), texture.id()) + ); + let uses = compiled.resource_uses(texture.id()).unwrap(); + assert!(uses.iter().any(|use_| use_.command == CommandIndex(1))); + assert!(!compiled.barriers().iter().any(|barrier| { + barrier.before_command == CommandIndex(1) + && barrier + .resources + .iter() + .any(|resource| resource.id == texture.id()) + })); + + let mut duplicate = RenderGraph::new(); + let texture = duplicate.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + ..Default::default() + }); + let mut begin = empty_render_pass(); + begin + .color + .push(created_color_attachment(texture, vk::ImageLayout::GENERAL)); + begin + .color + .push(created_color_attachment(texture, vk::ImageLayout::GENERAL)); + duplicate.push(begin); + duplicate.push(EndRendering); + assert!(matches!( + duplicate.compile(), + Err(GraphError::InvalidRenderPassAttachment(_)) + )); + + let mut invalid_layout = RenderGraph::new(); + let texture = invalid_layout.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + ..Default::default() + }); + let mut begin = empty_render_pass(); + begin.color.push(created_color_attachment( + texture, + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + )); + invalid_layout.push(begin); + invalid_layout.push(EndRendering); + assert!(matches!( + invalid_layout.compile(), + Err(GraphError::InvalidRenderPassAttachment(_)) + )); + + let mut sentinel = RenderGraph::new(); + let texture = sentinel.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + ..Default::default() + }); + let mut begin = empty_render_pass(); + let mut attachment = created_color_attachment(texture, vk::ImageLayout::GENERAL); + let AttachmentView::Created(view) = &mut attachment.view else { + unreachable!(); + }; + view.mip_range.end = vk::REMAINING_MIP_LEVELS; + begin.color.push(attachment); + sentinel.push(begin); + sentinel.push(EndRendering); + assert!(matches!( + sentinel.compile(), + Err(GraphError::InvalidRenderPassAttachment(_)) + )); + + let mut mismatch = RenderGraph::new(); + let graph_image = test_image(602); + let foreign_image = test_image(603); + let texture = mismatch + .import_texture(graph_image, ResourceState::default()) + .unwrap(); + let mut begin = empty_render_pass(); + begin.color.push(ColorAttachment::imported( + texture, + test_view(foreign_image), + vk::ImageLayout::GENERAL, + vk::AttachmentLoadOp::DONT_CARE, + vk::AttachmentStoreOp::STORE, + vk::ClearValue::default(), + )); + mismatch.push(begin); + mismatch.push(EndRendering); + assert!(matches!( + mismatch.compile(), + Err(GraphError::InvalidRenderPassAttachment(_)) + )); + } + + #[test] + fn render_pass_feedback_loop_rejects_overlapping_interior_texture_access() { + let mut graph = RenderGraph::new(); + let texture = graph.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + mip_levels: 2, + ..Default::default() + }); + let mut begin = empty_render_pass(); + begin + .color + .push(created_color_attachment(texture, vk::ImageLayout::GENERAL)); + graph.push(begin); + struct TextureAccess(Texture); + impl crate::commands::Command for TextureAccess { + fn side_effects(&self, command: CommandIndex, map: &mut SideEffectMap) { + map.insert( + command, + self.0.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, + access: vk::AccessFlags2::SHADER_READ, + subresource: SubresourceScopeInner::Texture { + range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), + layout: vk::ImageLayout::GENERAL, + }, + }, + ); + } + } + impl crate::recorder::RecordedCommand for TextureAccess { + fn allowed_in_render_pass(&self) -> bool { + true + } + + fn apply(&mut self, _: &mut CommandRecorder<'_>) -> Result<(), RecordError> { + Ok(()) + } + } + graph.push(TextureAccess(texture)); + graph.push(EndRendering); + + assert!(matches!( + graph.compile(), + Err(GraphError::RenderPassFeedbackLoop(_)) + )); + } + + #[test] + fn intra_pass_same_layout_barriers_are_allowed_but_layout_transitions_are_rejected() { + struct ImageUse { + texture: Texture, + layout: vk::ImageLayout, + access: vk::AccessFlags2, + } + impl crate::commands::Command for ImageUse { + fn side_effects(&self, command: CommandIndex, map: &mut SideEffectMap) { + map.insert( + command, + self.texture.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COMPUTE_SHADER, + access: self.access, + subresource: SubresourceScopeInner::Texture { + range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), + layout: self.layout, + }, + }, + ); + } + } + impl crate::recorder::RecordedCommand for ImageUse { + fn allowed_in_render_pass(&self) -> bool { + true + } + + fn apply(&mut self, _: &mut CommandRecorder<'_>) -> Result<(), RecordError> { + Ok(()) + } + } + + let make_graph = |second_layout| { + let mut graph = RenderGraph::new(); + let texture = graph.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + ..Default::default() + }); + graph.push(empty_render_pass()); + graph.push(ImageUse { + texture, + layout: vk::ImageLayout::GENERAL, + access: vk::AccessFlags2::SHADER_STORAGE_WRITE, + }); + graph.push(ImageUse { + texture, + layout: second_layout, + access: vk::AccessFlags2::SHADER_STORAGE_READ, + }); + graph.push(EndRendering); + graph + }; + + let same_layout = make_graph(vk::ImageLayout::GENERAL) + .compile() + .expect("same-layout in-pass barrier is supported"); + assert!(same_layout.barriers().iter().any(|barrier| { + barrier.before_command == CommandIndex(2) + && barrier.resources.iter().any(|resource| { + resource.old_layout == Some(vk::ImageLayout::GENERAL) + && resource.new_layout == Some(vk::ImageLayout::GENERAL) + }) + })); + + assert!(matches!( + make_graph(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL).compile(), + Err(GraphError::UnsupportedIntraPassDependency) + )); + } + + #[test] + fn hazard_free_foreign_command_is_scheduled_outside_render_pass_span() { + let mut graph = RenderGraph::new(); + graph.push(empty_render_pass()); + graph.push(AllowedAccess(Vec::new())); + graph.push(EndRendering); + graph.push(AllowedAccess(Vec::new())); + + let compiled = graph.compile().expect("graph compiles"); + let begin = compiled + .execution_order() + .iter() + .position(|&command| command == CommandIndex(0)) + .unwrap(); + let end = compiled + .execution_order() + .iter() + .position(|&command| command == CommandIndex(2)) + .unwrap(); + let foreign = compiled + .execution_order() + .iter() + .position(|&command| command == CommandIndex(3)) + .unwrap(); + assert!(foreign < begin || foreign > end); + } + + #[test] + fn interior_access_barriers_are_hoisted_and_coalesced() { + let mut graph = RenderGraph::new(); + let source = graph.create_buffer(GraphBufferDesc { + size: 32, + ..Default::default() + }); + let produced = graph.create_buffer(GraphBufferDesc { + size: 32, + ..Default::default() + }); + let range = BufferRange { + offset: 0, + size: 32, + }; + let producer = graph.push(CopyBuffer { + src: source, + src_range: range, + dst: produced, + dst_range: range, + }); + graph.push(empty_render_pass()); + graph.push(AllowedAccess(vec![( + produced.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::VERTEX_SHADER, + access: vk::AccessFlags2::SHADER_READ, + subresource: SubresourceScopeInner::Buffer(range), + }, + )])); + graph.push(AllowedAccess(vec![( + produced.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, + access: vk::AccessFlags2::UNIFORM_READ, + subresource: SubresourceScopeInner::Buffer(range), + }, + )])); + graph.push(EndRendering); + + let compiled = graph.compile().expect("graph compiles"); + let begin = compiled + .barriers() + .iter() + .find(|barrier| barrier.before_command == CommandIndex(1)) + .expect("outside-producer dependency is hoisted to begin"); + let resource = begin + .resources + .iter() + .find(|resource| resource.id == produced.id()) + .expect("producer buffer barrier is present"); + assert_eq!(resource.origin, BarrierOrigin::Command(producer)); + + let consumers = begin + .resources + .iter() + .filter(|resource| resource.id == produced.id()) + .collect::>(); + assert_eq!(consumers.len(), 1); + assert_eq!( + consumers[0].dst_stage, + vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER + ); + assert_eq!( + consumers[0].dst_access, + vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::UNIFORM_READ + ); + } + + #[test] + fn initial_storage_image_transition_is_hoisted_before_begin() { + let mut graph = RenderGraph::new(); + let texture = graph.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + extra_usage: vk::ImageUsageFlags::STORAGE, + ..Default::default() + }); + graph.push(empty_render_pass()); + graph.push(AllowedAccess(vec![( + texture.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COMPUTE_SHADER, + access: vk::AccessFlags2::SHADER_STORAGE_WRITE, + subresource: SubresourceScopeInner::Texture { + range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), + layout: vk::ImageLayout::GENERAL, + }, + }, + )])); + graph.push(EndRendering); + + let compiled = graph.compile().expect("graph compiles"); + let begin = compiled + .barriers() + .iter() + .find(|barrier| barrier.before_command == CommandIndex(0)) + .expect("initial image state transition is hoisted to begin"); + let resource = begin + .resources + .iter() + .find(|resource| resource.id == texture.id()) + .expect("storage image transition is present"); + assert_eq!(resource.origin, BarrierOrigin::Initial); + assert_eq!(resource.old_layout, Some(vk::ImageLayout::UNDEFINED)); + assert_eq!(resource.new_layout, Some(vk::ImageLayout::GENERAL)); + } + + #[test] + fn outside_image_transition_to_interior_readers_is_coalesced() { + let mut graph = RenderGraph::new(); + let texture = graph.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + ..Default::default() + }); + let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); + let producer = graph.push(AllowedAccess(vec![( + texture.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COPY, + access: vk::AccessFlags2::TRANSFER_WRITE, + subresource: SubresourceScopeInner::Texture { + range, + layout: vk::ImageLayout::GENERAL, + }, + }, + )])); + graph.push(empty_render_pass()); + graph.push(AllowedAccess(vec![( + texture.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::VERTEX_SHADER, + access: vk::AccessFlags2::SHADER_READ, + subresource: SubresourceScopeInner::Texture { + range, + layout: vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + }, + }, + )])); + graph.push(AllowedAccess(vec![( + texture.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, + access: vk::AccessFlags2::SHADER_SAMPLED_READ, + subresource: SubresourceScopeInner::Texture { + range, + layout: vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + }, + }, + )])); + graph.push(EndRendering); + + let compiled = graph.compile().expect("outside transition is hoisted"); + let begin = compiled + .barriers() + .iter() + .find(|barrier| barrier.before_command == CommandIndex(1)) + .expect("transition is keyed at begin"); + let resources = begin + .resources + .iter() + .filter(|resource| resource.id == texture.id()) + .collect::>(); + assert_eq!(resources.len(), 1); + assert_eq!(resources[0].origin, BarrierOrigin::Command(producer)); + assert_eq!(resources[0].old_layout, Some(vk::ImageLayout::GENERAL)); + assert_eq!( + resources[0].new_layout, + Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) + ); + assert_eq!( + resources[0].dst_stage, + vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER + ); + assert_eq!( + resources[0].dst_access, + vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_SAMPLED_READ + ); + } + + #[test] + fn interior_storage_buffer_hazard_emits_barrier_without_splitting_pass() { + let mut graph = RenderGraph::new(); + let buffer = graph.create_buffer(GraphBufferDesc { + size: 32, + extra_usage: vk::BufferUsageFlags::STORAGE_BUFFER, + ..Default::default() + }); + let range = BufferRange { + offset: 0, + size: 32, + }; + graph.push(empty_render_pass()); + graph.push(AllowedAccess(vec![( + buffer.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COMPUTE_SHADER, + access: vk::AccessFlags2::SHADER_STORAGE_WRITE, + subresource: SubresourceScopeInner::Buffer(range), + }, + )])); + graph.push(AllowedAccess(vec![( + buffer.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COMPUTE_SHADER, + access: vk::AccessFlags2::SHADER_STORAGE_READ, + subresource: SubresourceScopeInner::Buffer(range), + }, + )])); + graph.push(EndRendering); + + let compiled = graph.compile().expect("same-pass buffer barrier is legal"); + assert_eq!( + compiled.batches().collect::>(), + [&[ + CommandIndex(0), + CommandIndex(1), + CommandIndex(2), + CommandIndex(3), + ][..]] + ); + assert_eq!(compiled.render_pass_spans().len(), 1); + assert!(compiled.barriers().iter().any(|barrier| { + barrier.before_command == CommandIndex(2) + && barrier.resources.iter().any(|resource| { + resource.id == buffer.id() + && resource.src_access == vk::AccessFlags2::SHADER_STORAGE_WRITE + && resource.dst_access == vk::AccessFlags2::SHADER_STORAGE_READ + }) + })); + } + + #[test] + fn interior_storage_image_same_layout_hazard_emits_image_barrier() { + let mut graph = RenderGraph::new(); + let texture = graph.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + extra_usage: vk::ImageUsageFlags::STORAGE, + ..Default::default() + }); + let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); + graph.push(empty_render_pass()); + graph.push(AllowedAccess(vec![( + texture.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COMPUTE_SHADER, + access: vk::AccessFlags2::SHADER_STORAGE_WRITE, + subresource: SubresourceScopeInner::Texture { + range, + layout: vk::ImageLayout::GENERAL, + }, + }, + )])); + graph.push(AllowedAccess(vec![( + texture.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COMPUTE_SHADER, + access: vk::AccessFlags2::SHADER_STORAGE_READ, + subresource: SubresourceScopeInner::Texture { + range, + layout: vk::ImageLayout::GENERAL, + }, + }, + )])); + graph.push(EndRendering); + + let compiled = graph.compile().expect("same-layout image barrier is legal"); + assert!(compiled.barriers().iter().any(|barrier| { + barrier.before_command == CommandIndex(2) + && barrier.resources.iter().any(|resource| { + resource.id == texture.id() + && resource.old_layout == Some(vk::ImageLayout::GENERAL) + && resource.new_layout == Some(vk::ImageLayout::GENERAL) + }) + })); + } + + #[test] + fn interior_only_resource_lifetime_maps_to_pass_batch() { + let mut graph = RenderGraph::new(); + let buffer = graph.create_buffer(GraphBufferDesc { + size: 16, + ..Default::default() + }); + graph.push(empty_render_pass()); + graph.push(AllowedAccess(vec![( + buffer.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::VERTEX_SHADER, + access: vk::AccessFlags2::SHADER_READ, + subresource: SubresourceScopeInner::Buffer(BufferRange { + offset: 0, + size: 16, + }), + }, + )])); + graph.push(EndRendering); + + let compiled = graph.compile().expect("graph compiles"); + let batch = compiled + .batches() + .position(|commands| commands.contains(&CommandIndex(0))) + .unwrap(); + let lifetime = compiled.resource_lifetimes()[&buffer.id()]; + assert_eq!(lifetime.first_batch, batch); + assert_eq!(lifetime.last_batch, batch); + assert_ne!(lifetime.first_batch, usize::MAX); + } + + #[test] + fn render_pass_attachment_and_interior_image_interfere_until_end() { + let mut graph = RenderGraph::new(); + let attachment = graph.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + ..Default::default() + }); + let storage = graph.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + extra_usage: vk::ImageUsageFlags::STORAGE, + ..Default::default() + }); + let mut begin = empty_render_pass(); + begin.color.push(created_color_attachment( + attachment, + vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + )); + let begin_index = graph.push(begin); + let interior = graph.push(AllowedAccess(vec![( + storage.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COMPUTE_SHADER, + access: vk::AccessFlags2::SHADER_STORAGE_WRITE, + subresource: SubresourceScopeInner::Texture { + range: texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1), + layout: vk::ImageLayout::GENERAL, + }, + }, + )])); + let end_index = graph.push(EndRendering); + + let compiled = graph.compile().expect("graph compiles"); + assert!( + compiled + .interference() + .interferes(attachment.id(), storage.id()) + ); + assert!(!compiled.barriers().iter().any(|barrier| { + barrier.before_command == interior + && barrier + .resources + .iter() + .any(|resource| resource.id == attachment.id() || resource.id == storage.id()) + })); + let uses = compiled.resource_uses(attachment.id()).unwrap(); + assert!(uses.iter().any(|use_| use_.command == begin_index)); + assert!(uses.iter().any(|use_| use_.command == end_index)); + let execution_order = compiled.execution_order(); + let end_position = execution_order + .iter() + .position(|&command| command == end_index) + .unwrap(); + let last_attachment_use = uses + .iter() + .map(|use_| { + execution_order + .iter() + .position(|&command| command == use_.command) + .unwrap() + }) + .max() + .unwrap(); + assert_eq!(last_attachment_use, end_position); + } + + #[test] + fn dependencies_order_commands_around_the_whole_pass_but_not_within_it() { + let mut graph = RenderGraph::new(); + let source = graph.create_buffer(GraphBufferDesc { + size: 16, + ..Default::default() + }); + let shared = graph.create_buffer(GraphBufferDesc { + size: 16, + ..Default::default() + }); + let range = BufferRange { + offset: 0, + size: 16, + }; + let producer = graph.push(CopyBuffer { + src: source, + src_range: range, + dst: shared, + dst_range: range, + }); + graph.push(empty_render_pass()); + let interior_a = graph.push(AllowedAccess(vec![( + shared.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::VERTEX_SHADER, + access: vk::AccessFlags2::SHADER_READ, + subresource: SubresourceScopeInner::Buffer(range), + }, + )])); + let interior_b = graph.push(AllowedAccess(vec![( + shared.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, + access: vk::AccessFlags2::SHADER_READ, + subresource: SubresourceScopeInner::Buffer(range), + }, + )])); + graph.push(EndRendering); + let consumer = graph.push(AllowedAccess(vec![( + shared.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COMPUTE_SHADER, + access: vk::AccessFlags2::SHADER_STORAGE_WRITE, + subresource: SubresourceScopeInner::Buffer(range), + }, + )])); + + let compiled = graph.compile().expect("graph compiles"); + assert!(compiled.happens_before(producer, interior_a)); + assert!(compiled.happens_before(interior_a, consumer)); + assert!(!compiled.happens_before(interior_a, interior_b)); + } + #[test] fn draw_vertex_index_and_indirect_reads_follow_copies_and_derive_usage() { let mut graph = RenderGraph::new(); @@ -4326,6 +7071,10 @@ mod tests { dst_range: range, }); } + graph.push(BeginRendering { + color: Vec::new(), + depth: None, + }); graph.push(Draw { args: DrawArgs::Indexed { index_count: 3, @@ -4360,37 +7109,42 @@ mod tests { descriptor_sets: Vec::new(), push_constants: Vec::new(), }); + graph.push(EndRendering); let accesses = graph.commands.side_effects.command_accesses(); assert_eq!( derive_buffer_usage( - &accesses_for_resource(vertex.id(), accesses), + &accesses_for_resource(vertex.id(), accesses, &graph.commands.side_effects), vk::BufferUsageFlags::empty() ), vk::BufferUsageFlags::TRANSFER_DST | vk::BufferUsageFlags::VERTEX_BUFFER ); assert_eq!( derive_buffer_usage( - &accesses_for_resource(index.id(), accesses), + &accesses_for_resource(index.id(), accesses, &graph.commands.side_effects), vk::BufferUsageFlags::empty() ), vk::BufferUsageFlags::TRANSFER_DST | vk::BufferUsageFlags::INDEX_BUFFER ); assert_eq!( derive_buffer_usage( - &accesses_for_resource(indirect.id(), accesses), + &accesses_for_resource(indirect.id(), accesses, &graph.commands.side_effects), vk::BufferUsageFlags::empty() ), vk::BufferUsageFlags::TRANSFER_DST | vk::BufferUsageFlags::INDIRECT_BUFFER ); let compiled = graph.compile().expect("graph compiles"); - assert_eq!(compiled.batches().len(), 2); + let batches = compiled.batches().collect::>(); + assert_eq!(batches.len(), 2); assert_eq!( - compiled.batches()[0], + batches[0], [CommandIndex(0), CommandIndex(1), CommandIndex(2)] ); - assert_eq!(compiled.batches()[1], [CommandIndex(3)]); + assert_eq!( + batches[1], + [CommandIndex(3), CommandIndex(4), CommandIndex(5)] + ); let draw_barrier = compiled .barriers() .iter() @@ -4466,6 +7220,10 @@ mod tests { }, ], ); + graph.push(BeginRendering { + color: Vec::new(), + depth: None, + }); graph.push(Draw { args: DrawArgs::Direct { vertex_count: 3, @@ -4480,18 +7238,19 @@ mod tests { descriptor_sets: vec![set], push_constants: Vec::new(), }); + graph.push(EndRendering); let accesses = graph.commands.side_effects.command_accesses(); assert!( derive_buffer_usage( - &accesses_for_resource(uniform.id(), accesses), + &accesses_for_resource(uniform.id(), accesses, &graph.commands.side_effects), vk::BufferUsageFlags::empty() ) .contains(vk::BufferUsageFlags::UNIFORM_BUFFER) ); assert!( derive_image_usage( - &accesses_for_resource(texture.id(), accesses), + &accesses_for_resource(texture.id(), accesses, &graph.commands.side_effects), vk::ImageUsageFlags::empty() ) .contains(vk::ImageUsageFlags::SAMPLED) @@ -4670,12 +7429,7 @@ mod tests { let compiled = graph.compile().expect("graph compiles"); assert_eq!( - compiled - .batches() - .iter() - .flatten() - .copied() - .collect::>(), + compiled.batches().flatten().copied().collect::>(), vec![CommandIndex(0), CommandIndex(1)] ); assert_eq!(compiled.barriers().len(), 1); @@ -4713,9 +7467,16 @@ mod tests { vk::AccessFlags2::SHADER_WRITE, )], ]; - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); - let batches = compiler.topo_batches().expect("graph is acyclic"); - let barriers = compiler.build_barriers(&batches); + let compiler = GraphCompiler::new( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + &[], + ); + let batches = topo_batches_for_test(&compiler).expect("graph is acyclic"); + let execution_order = flatten_batches(&batches); + let barriers = compiler.build_barriers(&execution_order); assert_eq!(barriers.len(), 2); assert_eq!(barriers[0].before_command, CommandIndex(1)); @@ -4789,38 +7550,116 @@ mod tests { vk::AccessFlags2::UNIFORM_READ, )], ]; - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let compiler = GraphCompiler::new( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + &[], + ); let batches = vec![vec![CommandIndex(0)], vec![CommandIndex(1)]]; - let happens_before = HappensBefore::from_dag(&[vec![1], vec![]], &batches); + let happens_before = HappensBefore::from_command_dag(&[vec![1], vec![]], &batches); let uses = compiler.resource_uses(); + let frontiers = ResourceFrontiers::from_uses(&uses, &happens_before); let lifetimes = HashMap::from([ (old, placed_lifetime(0, 64, false)), (new, placed_lifetime(0, 64, true)), ]); - let mut barriers = compiler.build_barriers(&batches); + let mut barriers = compiler.build_barriers(&flatten_batches(&batches)); - compiler.build_alias_barriers(&mut barriers, &lifetimes, &uses, &happens_before); + compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); let barrier = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(1)) .unwrap(); - assert_eq!(barrier.memory_barriers.len(), 1); - let memory = barrier.memory_barriers[0]; + assert!(barrier.memory_barriers.is_empty()); + let alias = barrier + .resources + .iter() + .find(|resource| resource.id == new) + .unwrap(); assert_eq!( - memory.src_stage, + alias.src_stage, vk::PipelineStageFlags2::COPY | vk::PipelineStageFlags2::COMPUTE_SHADER ); assert_eq!( - memory.src_access, + alias.src_access, vk::AccessFlags2::TRANSFER_WRITE | vk::AccessFlags2::SHADER_STORAGE_WRITE ); - assert_eq!(memory.dst_stage, vk::PipelineStageFlags2::FRAGMENT_SHADER); - assert_eq!(memory.dst_access, vk::AccessFlags2::UNIFORM_READ); + assert_eq!(alias.dst_stage, vk::PipelineStageFlags2::FRAGMENT_SHADER); + assert_eq!(alias.dst_access, vk::AccessFlags2::UNIFORM_READ); + assert!(matches!( + alias.range, + Subresource::Buffer(BufferRange { + offset: 0, + size: vk::WHOLE_SIZE, + }) + )); + assert_eq!(alias.old_layout, None); + assert_eq!(alias.new_layout, None); + assert_eq!(alias.queue_transfer, None); + assert_eq!(alias.origin, BarrierOrigin::Initial); + + // Existing first-use resource dependencies retain their destination/range/origin, + // while alias source masks are added to their own source masks. + let existing_resource = BarrierResource { + id: new, + range: Subresource::Buffer(BufferRange { + offset: 8, + size: 16, + }), + origin: BarrierOrigin::Command(CommandIndex(0)), + src_stage: vk::PipelineStageFlags2::VERTEX_SHADER, + src_access: vk::AccessFlags2::SHADER_READ, + dst_stage: vk::PipelineStageFlags2::COPY, + dst_access: vk::AccessFlags2::TRANSFER_WRITE, + old_layout: None, + new_layout: None, + queue_transfer: None, + }; + let mut barriers = vec![PipelineBarrier { + before_command: CommandIndex(1), + memory_barriers: Vec::new(), + resources: tinyvec::tiny_vec!([BarrierResource; 2] => existing_resource), + }]; + + compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); + + let alias = barriers + .iter() + .find(|barrier| barrier.before_command == CommandIndex(1)) + .unwrap() + .resources + .iter() + .find(|resource| resource.id == new) + .unwrap(); + assert_eq!( + alias.src_stage, + vk::PipelineStageFlags2::VERTEX_SHADER + | vk::PipelineStageFlags2::COPY + | vk::PipelineStageFlags2::COMPUTE_SHADER + ); + assert_eq!( + alias.src_access, + vk::AccessFlags2::SHADER_READ + | vk::AccessFlags2::TRANSFER_WRITE + | vk::AccessFlags2::SHADER_STORAGE_WRITE + ); + assert_eq!(alias.dst_stage, vk::PipelineStageFlags2::COPY); + assert_eq!(alias.dst_access, vk::AccessFlags2::TRANSFER_WRITE); + assert!(matches!( + alias.range, + Subresource::Buffer(BufferRange { + offset: 8, + size: 16, + }) + )); + assert_eq!(alias.origin, BarrierOrigin::Command(CommandIndex(0))); } #[test] - fn alias_buffer_read_only_source_uses_execution_only_memory_barrier() { + fn alias_buffer_read_only_source_uses_execution_only_scoped_barrier() { let old = ResourceId::from_u32(0); let new = ResourceId::from_u32(1); let resources = [ @@ -4843,26 +7682,39 @@ mod tests { vk::AccessFlags2::TRANSFER_WRITE, )], ]; - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let compiler = GraphCompiler::new( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + &[], + ); let batches = vec![vec![CommandIndex(0)], vec![CommandIndex(1)]]; - let happens_before = HappensBefore::from_dag(&[vec![1], vec![]], &batches); + let happens_before = HappensBefore::from_command_dag(&[vec![1], vec![]], &batches); let uses = compiler.resource_uses(); + let frontiers = ResourceFrontiers::from_uses(&uses, &happens_before); let lifetimes = HashMap::from([ (old, placed_lifetime(0, 8, false)), (new, placed_lifetime(0, 8, true)), ]); - let mut barriers = compiler.build_barriers(&batches); + let mut barriers = compiler.build_barriers(&flatten_batches(&batches)); - compiler.build_alias_barriers(&mut barriers, &lifetimes, &uses, &happens_before); + compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); - let memory = barriers + let barrier = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(1)) - .unwrap() - .memory_barriers[0]; - assert_eq!(memory.src_stage, vk::PipelineStageFlags2::VERTEX_SHADER); - assert_eq!(memory.src_access, vk::AccessFlags2::NONE); - assert_eq!(memory.dst_access, vk::AccessFlags2::TRANSFER_WRITE); + .unwrap(); + assert!(barrier.memory_barriers.is_empty()); + let alias = barrier + .resources + .iter() + .find(|resource| resource.id == new) + .unwrap(); + assert_eq!(alias.src_stage, vk::PipelineStageFlags2::VERTEX_SHADER); + assert_eq!(alias.src_access, vk::AccessFlags2::NONE); + assert_eq!(alias.dst_stage, vk::PipelineStageFlags2::COPY); + assert_eq!(alias.dst_access, vk::AccessFlags2::TRANSFER_WRITE); } #[test] @@ -4890,17 +7742,24 @@ mod tests { vk::AccessFlags2::SHADER_SAMPLED_READ, )], ]; - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let compiler = GraphCompiler::new( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + &[], + ); let batches = vec![vec![CommandIndex(0)], vec![CommandIndex(1)]]; - let happens_before = HappensBefore::from_dag(&[vec![1], vec![]], &batches); + let happens_before = HappensBefore::from_command_dag(&[vec![1], vec![]], &batches); let uses = compiler.resource_uses(); + let frontiers = ResourceFrontiers::from_uses(&uses, &happens_before); let lifetimes = HashMap::from([ (old, placed_lifetime(0, 128, false)), (new, placed_lifetime(0, 128, true)), ]); - let mut barriers = compiler.build_barriers(&batches); + let mut barriers = compiler.build_barriers(&flatten_batches(&batches)); - compiler.build_alias_barriers(&mut barriers, &lifetimes, &uses, &happens_before); + compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); let barrier = barriers .iter() @@ -4913,6 +7772,7 @@ mod tests { .find(|resource| resource.id == new) .unwrap(); assert_eq!(image.old_layout, Some(vk::ImageLayout::UNDEFINED)); + assert_eq!(image.origin, BarrierOrigin::Initial); assert_eq!( image.src_stage, vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT @@ -4959,13 +7819,20 @@ mod tests { vk::PipelineStageFlags2::COMPUTE_SHADER, vk::AccessFlags2::SHADER_WRITE, )]); - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); + let compiler = GraphCompiler::new( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + &[], + ); let batches = (0..5) .map(|index| vec![CommandIndex(index)]) .collect::>(); let edges = vec![vec![4], vec![4], vec![4], vec![4], vec![]]; - let happens_before = HappensBefore::from_dag(&edges, &batches); + let happens_before = HappensBefore::from_command_dag(&edges, &batches); let uses = compiler.resource_uses(); + let frontiers = ResourceFrontiers::from_uses(&uses, &happens_before); let lifetimes = HashMap::from([ (old_ids[0], placed_lifetime(0, 8, false)), (old_ids[1], placed_lifetime(4, 8, false)), @@ -4973,32 +7840,35 @@ mod tests { (old_ids[3], placed_lifetime(32, 8, false)), (new, placed_lifetime(8, 16, true)), ]); - let mut barriers = compiler.build_barriers(&batches); + let mut barriers = compiler.build_barriers(&flatten_batches(&batches)); - compiler.build_alias_barriers(&mut barriers, &lifetimes, &uses, &happens_before); + compiler.build_alias_barriers(&mut barriers, &lifetimes, &frontiers, &happens_before); let barrier = barriers .iter() .find(|barrier| barrier.before_command == CommandIndex(4)) .unwrap(); // Source masks are unioned across all intersecting occupants into one barrier. - assert_eq!(barrier.memory_barriers.len(), 1); + assert!(barrier.memory_barriers.is_empty()); + let alias = barrier + .resources + .iter() + .find(|resource| resource.id == new) + .unwrap(); assert_eq!( - barrier.memory_barriers[0].src_stage, + alias.src_stage, vk::PipelineStageFlags2::FRAGMENT_SHADER | vk::PipelineStageFlags2::COMPUTE_SHADER ); - assert_eq!( - barrier.memory_barriers[0].src_access, - vk::AccessFlags2::TRANSFER_WRITE - ); - assert_eq!( - barrier.memory_barriers[0].dst_stage, - vk::PipelineStageFlags2::COMPUTE_SHADER - ); - assert_eq!( - barrier.memory_barriers[0].dst_access, - vk::AccessFlags2::SHADER_WRITE - ); + assert_eq!(alias.src_access, vk::AccessFlags2::TRANSFER_WRITE); + assert_eq!(alias.dst_stage, vk::PipelineStageFlags2::COMPUTE_SHADER); + assert_eq!(alias.dst_access, vk::AccessFlags2::SHADER_WRITE); + assert!(matches!( + alias.range, + Subresource::Buffer(BufferRange { + offset: 0, + size: vk::WHOLE_SIZE, + }) + )); } fn placed_lifetime(offset: u64, size: u64, aliased: bool) -> ResourceLifetime { @@ -5048,9 +7918,16 @@ mod tests { vk::AccessFlags2::SHADER_SAMPLED_READ, )], ]; - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); - let batches = compiler.topo_batches().expect("graph is acyclic"); - let barriers = compiler.build_barriers_with_initial_states(&batches, &initial_states); + let compiler = GraphCompiler::new( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + &[], + ); + let batches = topo_batches_for_test(&compiler).expect("graph is acyclic"); + let barriers = compiler + .build_barriers_with_initial_states(&flatten_batches(&batches), &initial_states); assert_eq!(barriers.len(), 1); assert_eq!(barriers[0].before_command, CommandIndex(1)); @@ -5099,9 +7976,15 @@ mod tests { vk::AccessFlags2::SHADER_SAMPLED_READ, )], ]; - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); - let batches = compiler.topo_batches().expect("graph is acyclic"); - let barriers = compiler.build_barriers(&batches); + let compiler = GraphCompiler::new( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + &[], + ); + let batches = topo_batches_for_test(&compiler).expect("graph is acyclic"); + let barriers = compiler.build_barriers(&flatten_batches(&batches)); assert_eq!(barriers.len(), 3); let current_read_barrier = &barriers[2]; @@ -5116,6 +7999,126 @@ mod tests { assert_eq!(barrier.new_layout, Some(vk::ImageLayout::GENERAL)); } + #[test] + fn same_target_readers_chain_the_current_image_layout() { + let resource = ResourceId::from_u32(0); + let resources = [ResourceDesc::CreatedTexture(GraphTextureDesc::default())]; + let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); + let accesses = vec![ + vec![texture_scoped_access( + resource, + range, + vk::ImageLayout::GENERAL, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_WRITE, + )], + vec![texture_scoped_access( + resource, + range, + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + vk::PipelineStageFlags2::VERTEX_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + vec![texture_scoped_access( + resource, + range, + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + vk::PipelineStageFlags2::FRAGMENT_SHADER, + vk::AccessFlags2::SHADER_READ, + )], + ]; + let compiler = GraphCompiler::new( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + &[], + ); + let order = [CommandIndex(0), CommandIndex(1), CommandIndex(2)]; + let barriers = compiler.build_barriers(&order); + + let second_reader = barriers + .iter() + .find(|barrier| barrier.before_command == CommandIndex(2)) + .expect("the producer conflicts with the second reader"); + assert_eq!(second_reader.resources.len(), 1); + assert_eq!( + second_reader.resources[0].old_layout, + Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) + ); + assert_eq!( + second_reader.resources[0].new_layout, + Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) + ); + assert_eq!( + second_reader.resources[0].origin, + BarrierOrigin::Command(CommandIndex(0)) + ); + } + + #[test] + fn hoisted_equal_image_transitions_union_destination_scopes_in_order() { + let id = ResourceId::from_u32(0); + let range = texture_range(vk::ImageAspectFlags::COLOR, 0, 1, 0, 1); + let mut barriers = vec![ + PipelineBarrier { + before_command: CommandIndex(2), + memory_barriers: Vec::new(), + resources: [BarrierResource { + id, + range: Subresource::Texture(range), + origin: BarrierOrigin::Command(CommandIndex(0)), + src_stage: vk::PipelineStageFlags2::TRANSFER, + src_access: vk::AccessFlags2::TRANSFER_WRITE, + dst_stage: vk::PipelineStageFlags2::VERTEX_SHADER, + dst_access: vk::AccessFlags2::SHADER_READ, + old_layout: Some(vk::ImageLayout::GENERAL), + new_layout: Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL), + queue_transfer: None, + }] + .into_iter() + .collect(), + }, + PipelineBarrier { + before_command: CommandIndex(3), + memory_barriers: Vec::new(), + resources: [BarrierResource { + id, + range: Subresource::Texture(range), + origin: BarrierOrigin::Command(CommandIndex(0)), + src_stage: vk::PipelineStageFlags2::TRANSFER, + src_access: vk::AccessFlags2::TRANSFER_WRITE, + dst_stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, + dst_access: vk::AccessFlags2::SHADER_SAMPLED_READ, + old_layout: Some(vk::ImageLayout::GENERAL), + new_layout: Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL), + queue_transfer: None, + }] + .into_iter() + .collect(), + }, + ]; + let spans = [RenderPassSpan { + begin: CommandIndex(1), + end: CommandIndex(4), + }]; + hoist_span_barriers(&mut barriers, &spans, 5); + let merged = barriers + .iter() + .find(|barrier| barrier.before_command == CommandIndex(1)) + .unwrap(); + + assert_eq!(merged.resources.len(), 1); + assert_eq!( + merged.resources[0].dst_stage, + vk::PipelineStageFlags2::VERTEX_SHADER | vk::PipelineStageFlags2::FRAGMENT_SHADER + ); + assert_eq!( + merged.resources[0].dst_access, + vk::AccessFlags2::SHADER_READ | vk::AccessFlags2::SHADER_SAMPLED_READ + ); + } + #[test] fn overlapping_pure_reads_without_layout_change_need_no_barrier() { let resource = ResourceId::from_u32(0); @@ -5139,10 +8142,20 @@ mod tests { vk::AccessFlags2::UNIFORM_READ, )], ]; - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); - let batches = compiler.topo_batches().expect("graph is acyclic"); + let compiler = GraphCompiler::new( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + &[], + ); + let batches = topo_batches_for_test(&compiler).expect("graph is acyclic"); - assert!(compiler.build_barriers(&batches).is_empty()); + assert!( + compiler + .build_barriers(&flatten_batches(&batches)) + .is_empty() + ); } #[test] @@ -5168,9 +8181,15 @@ mod tests { vk::AccessFlags2::UNIFORM_READ, )], ]; - let compiler = GraphCompiler::new(&resources, &accesses, accesses.len()); - let batches = compiler.topo_batches().expect("graph is acyclic"); - let barriers = compiler.build_barriers(&batches); + let compiler = GraphCompiler::new( + &resources, + &accesses, + accesses.len(), + &SideEffectMap::default(), + &[], + ); + let batches = topo_batches_for_test(&compiler).expect("graph is acyclic"); + let barriers = compiler.build_barriers(&flatten_batches(&batches)); assert_eq!(batches, [vec![CommandIndex(0)], vec![CommandIndex(1)]]); assert_eq!(barriers.len(), 1); @@ -5234,19 +8253,14 @@ mod tests { let compiled = graph.compile().expect("graph compiles"); assert_eq!( - compiled.batches(), - &[ - vec![CommandIndex(0), CommandIndex(2)], - vec![CommandIndex(1)] + compiled.batches().collect::>(), + [ + &[CommandIndex(0), CommandIndex(2)][..], + &[CommandIndex(1)][..] ] ); assert_eq!( - compiled - .batches() - .iter() - .flatten() - .copied() - .collect::>(), + compiled.batches().flatten().copied().collect::>(), vec![CommandIndex(0), CommandIndex(2), CommandIndex(1)] ); assert_eq!(compiled.barriers().len(), 1); @@ -5255,7 +8269,6 @@ mod tests { let batch_for = |command| { compiled .batches() - .iter() .position(|batch| batch.contains(&command)) .expect("command appears in a batch") }; @@ -5272,17 +8285,18 @@ mod tests { let first_barrier = PipelineBarrier { before_command: CommandIndex(1), memory_barriers: Vec::new(), - resources: Vec::new(), + resources: tinyvec::TinyVec::new(), }; let second_barrier = PipelineBarrier { before_command: CommandIndex(1), memory_barriers: Vec::new(), - resources: Vec::new(), + resources: tinyvec::TinyVec::new(), }; let barriers = [first_barrier, second_barrier]; let batches = [vec![CommandIndex(0), CommandIndex(1)]]; - let steps = recording_steps(&batches, &barriers).collect::>(); + let execution_order = flatten_batches(&batches); + let steps = recording_steps(&execution_order, &barriers).collect::>(); assert!(matches!(steps[0], RecordingStep::Command(CommandIndex(0)))); assert!( matches!(steps[1], RecordingStep::Barrier(barrier) if std::ptr::eq(barrier, &barriers[0])) @@ -5334,13 +8348,14 @@ mod tests { let compiled = graph.compile().expect("graph compiles"); assert_eq!( - compiled.batches(), - &[ - vec![CommandIndex(0), CommandIndex(1)], - vec![CommandIndex(2), CommandIndex(3)] + compiled.batches().collect::>(), + [ + &[CommandIndex(0), CommandIndex(1)][..], + &[CommandIndex(2), CommandIndex(3)][..], ] ); - let steps = recording_steps(compiled.batches(), compiled.barriers()).collect::>(); + let steps = + recording_steps(compiled.execution_order(), compiled.barriers()).collect::>(); let command_two_position = steps .iter() .position(|step| matches!(step, RecordingStep::Command(CommandIndex(2)))) @@ -5394,11 +8409,11 @@ mod tests { let compiled = graph.compile().expect("graph compiles"); assert_eq!( - compiled.batches(), - &[ - vec![CommandIndex(0)], - vec![CommandIndex(1)], - vec![CommandIndex(2)] + compiled.batches().collect::>(), + [ + &[CommandIndex(0)][..], + &[CommandIndex(1)][..], + &[CommandIndex(2)][..], ] ); assert!(compiled.happens_before(CommandIndex(0), CommandIndex(2))); @@ -5407,12 +8422,10 @@ mod tests { let source_command = CommandIndex(barrier.before_command.index() - 1); let source_batch = compiled .batches() - .iter() .position(|batch| batch.contains(&source_command)) .expect("conflicting source appears in a batch"); let target_batch = compiled .batches() - .iter() .position(|batch| batch.contains(&barrier.before_command)) .expect("barrier target appears in a batch"); assert!(source_batch < target_batch); @@ -5503,8 +8516,8 @@ mod tests { }, }); let accesses = graph.commands.side_effects.command_accesses(); - let src_accesses = accesses_for_resource(src.id(), accesses); - let dst_accesses = accesses_for_resource(dst.id(), accesses); + let src_accesses = accesses_for_resource(src.id(), accesses, &graph.commands.side_effects); + let dst_accesses = accesses_for_resource(dst.id(), accesses, &graph.commands.side_effects); assert_eq!( derive_buffer_usage(&src_accesses, vk::BufferUsageFlags::empty()), @@ -5535,8 +8548,11 @@ mod tests { layout: vk::ImageLayout::TRANSFER_DST_OPTIMAL, color: [0.0; 4], }); - let accesses = - accesses_for_resource(texture.id(), graph.commands.side_effects.command_accesses()); + let accesses = accesses_for_resource( + texture.id(), + graph.commands.side_effects.command_accesses(), + &graph.commands.side_effects, + ); assert_eq!( derive_image_usage(&accesses, vk::ImageUsageFlags::empty()), @@ -5564,8 +8580,11 @@ mod tests { size: 16, }, }); - let accesses = - accesses_for_resource(buffer.id(), graph.commands.side_effects.command_accesses()); + let accesses = accesses_for_resource( + buffer.id(), + graph.commands.side_effects.command_accesses(), + &graph.commands.side_effects, + ); assert_eq!( derive_buffer_usage(&accesses, vk::BufferUsageFlags::UNIFORM_BUFFER), diff --git a/crates/renderer_rendergraph/src/graph_compiler.rs b/crates/renderer_rendergraph/src/graph_compiler.rs new file mode 100644 index 0000000..a44d496 --- /dev/null +++ b/crates/renderer_rendergraph/src/graph_compiler.rs @@ -0,0 +1,250 @@ +use ash::vk; + +use crate::{ + CommandIndex, GraphError, SideEffectMap, + graph::{Kahn, RenderPassSpan}, + recorder::BoxedCommand, + resources::{ResourceDesc, SubresourceScope, SubresourceScopeInner}, +}; + +struct SpanBuilder { + spans: Vec, + current: Option, +} + +impl SpanBuilder { + fn new() -> Self { + Self { + spans: Vec::new(), + current: None, + } + } + + fn in_span(&self) -> bool { + self.current.is_some() + } + + fn current_begin(&self) -> Option { + self.current + } + + fn begin(&mut self, index: CommandIndex) -> Result<(), GraphError> { + if self.current.is_some() { + Err(GraphError::NestedRenderPass) + } else { + self.current = Some(index); + Ok(()) + } + } + + fn end(&mut self, end: CommandIndex) -> Result<(), GraphError> { + if let Some(begin) = self.current.take() { + self.spans.push(RenderPassSpan { begin, end }); + Ok(()) + } else { + Err(GraphError::UnbalancedRenderPass) + } + } + + fn finish(self) -> Result, GraphError> { + if self.current.is_some() { + Err(GraphError::UnbalancedRenderPass) + } else { + Ok(self.spans) + } + } +} + +pub(crate) struct Batch { + pub(crate) start: usize, + pub(crate) count: usize, +} + +pub(crate) struct CommandOrderings { + pub(crate) execution_order: Vec, + pub(crate) batches: Vec, + pub(crate) spans: Vec, + pub(crate) node_of_command: Vec, + pub(crate) node_representatives: Vec, + pub(crate) dependency_edges: Vec>, +} + +pub(crate) fn build_command_orderings( + resources: &[ResourceDesc], + commands: &[BoxedCommand], + side_effects: &mut SideEffectMap, +) -> Result { + let per_command_resource_access = side_effects.command_accesses(); + let mut kahn = Kahn::new(commands.len()); + let mut access_by_resource: Vec)>> = + vec![Vec::new(); resources.len()]; + + let mut span_builder = SpanBuilder::new(); + let mut node_of_command = vec![usize::MAX; commands.len()]; + let mut node_representatives = Vec::new(); + + for (index, (command, accesses)) in commands + .iter() + .zip(per_command_resource_access) + .enumerate() + .map(|(i, c)| (CommandIndex::from(i), c)) + { + let rep = span_builder.current_begin().unwrap_or(index); + kahn.add_node(rep.index()); + + // Node ids follow first encounter of each representative in command order. + let node = if node_of_command[rep.index()] == usize::MAX { + let node = node_representatives.len(); + node_representatives.push(rep); + node + } else { + node_of_command[rep.index()] + }; + node_of_command[index.index()] = node; + + for access in accesses { + let Some(resource_access) = access_by_resource.get_mut(access.resource.index()) else { + // `validate_resources` reports invalid ids after dependency analysis. + continue; + }; + + let priors = resource_access + .iter() + .take_while(|&&(c, _)| c < rep) + .filter(|(_, scopes)| scopes.iter().any(|scope| scope.conflicts(&access.scope))); + + for &(prior, _) in priors { + kahn.add_edge(prior.index(), rep.index()); + } + + if let Some(&mut (last, ref mut scopes)) = resource_access.last_mut() + && last == rep + { + scopes.push(access.scope); + } else { + resource_access.push((rep, vec![access.scope])) + } + } + + match command.kind() { + crate::CommandKind::Begin => { + if let Some(desc) = command.render_pass_desc() { + crate::graph::validate_render_pass_attachments(resources, desc)?; + } + span_builder.begin(index)?; + } + crate::CommandKind::Graphics => { + if !command.allowed_in_render_pass() { + return Err(GraphError::CommandNotAllowedInRenderPass(index)); + } + if !span_builder.in_span() { + return Err(GraphError::DrawOutsideRenderPass(index)); + } + } + crate::CommandKind::Transfer + | crate::CommandKind::Compute + | crate::CommandKind::Other => { + if span_builder.in_span() && !command.allowed_in_render_pass() { + return Err(GraphError::CommandNotAllowedInRenderPass(index)); + } + } + crate::CommandKind::End => { + span_builder.end(index)?; + } + } + } + + // spans are ordered by beginning command + let spans = span_builder.finish()?; + + for span in &spans { + // Snapshot accesses intentionally predate lifetime-only attachment injection: + // these accesses extend lifetimes but must not add hazard edges. + if let Some(desc) = commands[span.begin.index()].render_pass_desc() { + for attachment in desc.color { + side_effects.insert_lifetime_only( + span.end, + attachment.texture.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT, + access: vk::AccessFlags2::COLOR_ATTACHMENT_WRITE, + subresource: SubresourceScopeInner::Texture { + range: attachment.range, + layout: attachment.layout, + }, + }, + ); + } + + if let Some(attachment) = desc.depth { + side_effects.insert_lifetime_only( + span.end, + attachment.texture.id(), + SubresourceScope { + stage: vk::PipelineStageFlags2::EARLY_FRAGMENT_TESTS + | vk::PipelineStageFlags2::LATE_FRAGMENT_TESTS, + access: vk::AccessFlags2::DEPTH_STENCIL_ATTACHMENT_WRITE, + subresource: SubresourceScopeInner::Texture { + range: attachment.range, + layout: attachment.layout, + }, + }, + ); + } + } + } + + let mut execution_order: Vec = Vec::with_capacity(commands.len()); + let mut batches = Vec::::new(); + + let mut span_iter = spans.iter().peekable(); + + while !kahn.is_complete() { + let first = execution_order.len(); + + for cmd in kahn.iter_batch().map(CommandIndex::from) { + use core::cmp::Ordering; + match span_iter.peek().map(|s| s.begin.cmp(&cmd)) { + Some(Ordering::Equal) => { + let span = span_iter.next().unwrap(); + execution_order + .extend((span.begin.index()..=span.end.index()).map(CommandIndex::from)); + } + _ => execution_order.push(cmd), + } + } + + let count = execution_order.len() - first; + + if count == 0 { + return Err(GraphError::DependencyCycle); + } + + for command in &execution_order[first..] { + kahn.complete(command.index()); + } + + batches.push(Batch { + start: first, + count, + }); + } + + let mut dependency_edges = vec![Vec::new(); node_representatives.len()]; + for (node, representative) in node_representatives.iter().enumerate() { + let representative = representative.index(); + let node_edges = &mut dependency_edges[node]; + for &edge in &kahn.edges[representative] { + node_edges.push(node_of_command[edge]); + } + } + + Ok(CommandOrderings { + execution_order, + batches, + spans, + node_of_command, + node_representatives, + dependency_edges, + }) +} diff --git a/crates/renderer_rendergraph/src/lib.rs b/crates/renderer_rendergraph/src/lib.rs index a84f97b..5d90568 100644 --- a/crates/renderer_rendergraph/src/lib.rs +++ b/crates/renderer_rendergraph/src/lib.rs @@ -3,12 +3,18 @@ pub mod commands; pub mod graph; +mod graph_compiler; pub mod recorder; pub mod resources; +pub use commands::{ + AttachmentView, BeginRendering, ColorAttachment, CommandKind, DepthStencilAttachment, + EndRendering, RenderPassDesc, +}; pub use graph::{ - CompiledGraph, GraphError, HappensBefore, MemoryBarrier, Placement, RenderGraph, - ResourceInterference, ResourceLifetime, ResourceUse, aliasing_permitted, + BarrierOrigin, CompiledGraph, GraphError, HappensBefore, MemoryBarrier, Placement, RenderGraph, + RenderPassSpan, ResourceFrontiers, ResourceInterference, ResourceLifetime, ResourceUse, + aliasing_permitted, }; pub use recorder::{CommandIndex, CommandList, CommandRecorder, SideEffectMap}; pub use resources::{ diff --git a/crates/renderer_rendergraph/src/recorder.rs b/crates/renderer_rendergraph/src/recorder.rs index c06f2e7..cf1700d 100644 --- a/crates/renderer_rendergraph/src/recorder.rs +++ b/crates/renderer_rendergraph/src/recorder.rs @@ -1,6 +1,11 @@ -use std::{cell::RefCell, collections::BTreeMap}; +use std::{ + cell::RefCell, + collections::{BTreeMap, BTreeSet, HashMap}, + sync::Arc, +}; use ash::vk; +use renderer_resources::images::{ImageView, ImageViewDesc}; use crate::resources::{ BufferRange, ResourceId, Subresource, SubresourceScope, SubresourceScopeInner, @@ -38,6 +43,7 @@ pub struct ResourceAccess { #[derive(Debug, Default, Clone)] pub struct SideEffectMap { accesses: Vec>, + lifetime_only: BTreeSet<(usize, ResourceId)>, } impl SideEffectMap { @@ -81,6 +87,23 @@ impl SideEffectMap { accesses.push(ResourceAccess { resource, scope }); } + /// Record an access used only to extend a resource's lifetime through a command. + pub fn insert_lifetime_only( + &mut self, + command_index: CommandIndex, + resource: ResourceId, + scope: SubresourceScope, + ) { + self.insert(command_index, resource, scope); + self.lifetime_only.insert((command_index.index(), resource)); + } + + /// Whether this command/resource access exists only for lifetime analysis. + pub fn is_lifetime_only(&self, command_index: CommandIndex, resource: ResourceId) -> bool { + self.lifetime_only + .contains(&(command_index.index(), resource)) + } + pub fn accesses(&self, command_index: CommandIndex) -> &[ResourceAccess] { self.accesses .get(command_index.0) @@ -92,10 +115,12 @@ impl SideEffectMap { } } +pub type BoxedCommand = Box; + /// A safe, type-erased temporary command buffer. Each command is owned until /// compilation/recording, while its access declarations are stored separately. pub struct CommandList { - pub(crate) commands: Vec>, + pub(crate) commands: Vec, pub side_effects: SideEffectMap, } @@ -105,6 +130,18 @@ pub trait RecordedCommand { Ok(()) } + fn kind(&self) -> crate::commands::CommandKind { + crate::commands::CommandKind::Other + } + + fn allowed_in_render_pass(&self) -> bool { + false + } + + fn render_pass_desc(&self) -> Option> { + None + } + fn apply(&mut self, recorder: &mut CommandRecorder<'_>) -> Result<(), RecordError>; } @@ -149,6 +186,7 @@ impl Default for CommandList { pub enum RecordError { MissingBuffer(ResourceId), MissingImage(ResourceId), + MissingView(ResourceId), UnresolvedCreatedBuffer(ResourceId), UnresolvedCreatedImage(ResourceId), DescriptorSet(String), @@ -161,6 +199,7 @@ pub struct CommandRecorder<'a> { pub command_buffer: vk::CommandBuffer, pub buffers: BTreeMap, pub images: BTreeMap, + pub views: HashMap<(ResourceId, ImageViewDesc), Arc>, } #[derive(Default)] @@ -181,6 +220,7 @@ impl<'a> CommandRecorder<'a> { command_buffer, buffers: BTreeMap::new(), images: BTreeMap::new(), + views: HashMap::new(), } } @@ -198,6 +238,18 @@ impl<'a> CommandRecorder<'a> { .ok_or(RecordError::MissingImage(id)) } + /// Resolve a cached image view by graph resource and concrete view description. + pub fn view( + &self, + id: ResourceId, + desc: &ImageViewDesc, + ) -> Result, RecordError> { + self.views + .get(&(id, image_view_desc_without_name(desc))) + .cloned() + .ok_or(RecordError::MissingView(id)) + } + pub fn pipeline_barrier(&self, barrier: &super::graph::PipelineBarrier) { BARRIER_SCRATCH.with(|scratch| { let mut scratch = scratch.borrow_mut(); @@ -288,6 +340,19 @@ impl<'a> CommandRecorder<'a> { } } +fn image_view_desc_without_name(desc: &ImageViewDesc) -> ImageViewDesc { + ImageViewDesc { + name: None, + flags: desc.flags, + kind: desc.kind, + format: desc.format, + components: desc.components, + aspect: desc.aspect, + mip_range: desc.mip_range, + layer_range: desc.layer_range, + } +} + pub(crate) fn scope_range(scope: SubresourceScope) -> Subresource { match scope.subresource { SubresourceScopeInner::Buffer(range) => Subresource::Buffer(range), @@ -337,7 +402,7 @@ mod tests { dst_stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, dst_access: vk::AccessFlags2::SHADER_READ, }], - resources: Vec::new(), + resources: tinyvec::TinyVec::new(), }; MEMORY_BARRIER_COUNT.store(0, Ordering::SeqCst); MEMORY_BARRIER_SRC_STAGE.store(0, Ordering::SeqCst); diff --git a/crates/renderer_rendergraph/tests/graph_validation.rs b/crates/renderer_rendergraph/tests/graph_validation.rs index 806f68f..d5417b0 100644 --- a/crates/renderer_rendergraph/tests/graph_validation.rs +++ b/crates/renderer_rendergraph/tests/graph_validation.rs @@ -6,16 +6,19 @@ use renderer_device::{ testing::TestVulkan, }; use renderer_rendergraph::{ - RenderGraph, - commands::Command, + BeginRendering, ColorAttachment, EndRendering, RenderGraph, commands::CopyBuffer, + commands::{ClearTexture, Command}, recorder::{CommandIndex, CommandRecorder, RecordError, RecordedCommand, SideEffectMap}, resources::{ BufferRange, GraphBufferDesc, GraphTextureDesc, ResourceId, ResourceState, SubresourceScope, SubresourceScopeInner, }, }; -use renderer_resources::buffers::{Buffer, BufferDesc}; +use renderer_resources::{ + buffers::{Buffer, BufferDesc}, + images::ImageViewDesc, +}; struct ScopedNoop(Vec<(ResourceId, SubresourceScope)>); @@ -33,6 +36,26 @@ impl RecordedCommand for ScopedNoop { } } +struct PassNoop(Vec<(ResourceId, SubresourceScope)>); + +impl Command for PassNoop { + fn side_effects(&self, command_index: CommandIndex, map: &mut SideEffectMap) { + for &(resource, scope) in &self.0 { + map.insert(command_index, resource, scope); + } + } +} + +impl RecordedCommand for PassNoop { + fn allowed_in_render_pass(&self) -> bool { + true + } + + fn apply(&mut self, _recorder: &mut CommandRecorder<'_>) -> Result<(), RecordError> { + Ok(()) + } +} + fn buffer_scope() -> SubresourceScope { SubresourceScope { stage: vk::PipelineStageFlags2::COPY, @@ -61,6 +84,47 @@ fn image_scope() -> SubresourceScope { } } +fn pass_buffer_read_scope() -> SubresourceScope { + SubresourceScope { + stage: vk::PipelineStageFlags2::VERTEX_SHADER, + access: vk::AccessFlags2::SHADER_READ, + subresource: SubresourceScopeInner::Buffer(BufferRange { + offset: 0, + size: 256, + }), + } +} + +fn pass_texture_read_scope() -> SubresourceScope { + SubresourceScope { + stage: vk::PipelineStageFlags2::FRAGMENT_SHADER, + access: vk::AccessFlags2::SHADER_SAMPLED_READ, + subresource: SubresourceScopeInner::Texture { + range: vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + }, + layout: vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + }, + } +} + +fn supports_render_pass_test(test: &TestVulkan) -> bool { + let features = test.device().enabled_features().core13; + if features.synchronization2 != vk::TRUE { + eprintln!("Skipping graph render pass: synchronization2 is unsupported"); + return false; + } + if features.dynamic_rendering != vk::TRUE { + eprintln!("Skipping graph render pass: dynamic_rendering is unsupported"); + return false; + } + true +} + fn submit_graph(test: &TestVulkan, graph: &mut renderer_rendergraph::CompiledGraph) { let device = test.device(); let pool_info = vk::CommandPoolCreateInfo::default() @@ -179,7 +243,7 @@ fn imported_buffer_copy_records_under_validation() { } #[test] -fn resolved_buffers_alias_and_submit_with_memory_barrier() { +fn resolved_buffers_alias_and_submit_with_scoped_barrier() { let Some(test) = TestVulkan::acquire() else { return; }; @@ -233,9 +297,21 @@ fn resolved_buffers_alias_and_submit_with_memory_barrier() { assert_eq!(first_placement.pool, second_placement.pool); assert_eq!(first_placement.offset, second_placement.offset); assert!(first_placement.aliased && second_placement.aliased); - assert!(compiled.barriers().iter().any(|barrier| { - barrier.before_command == CommandIndex(1) && !barrier.memory_barriers.is_empty() - })); + let alias_barrier = compiled + .barriers() + .iter() + .find(|barrier| barrier.before_command == CommandIndex(1)) + .expect("incoming buffer's first use has an alias barrier"); + assert!(alias_barrier.memory_barriers.is_empty()); + let incoming = alias_barrier + .resources + .iter() + .find(|resource| resource.id == second.id()) + .expect("alias barrier is scoped to the incoming buffer"); + assert_eq!(incoming.src_stage, vk::PipelineStageFlags2::COPY); + assert_eq!(incoming.src_access, vk::AccessFlags2::TRANSFER_WRITE); + assert_eq!(incoming.dst_stage, vk::PipelineStageFlags2::COPY); + assert_eq!(incoming.dst_access, vk::AccessFlags2::TRANSFER_WRITE); submit_graph(&test, &mut compiled); test.assert_no_validation_errors(); @@ -328,3 +404,149 @@ fn resolved_images_alias_and_submit_with_undefined_transition() { .expect("wait before shared allocation teardown"); test.assert_no_validation_errors(); } + +#[test] +fn created_render_pass_records_submits_and_validates() { + let Some(test) = TestVulkan::acquire() else { + return; + }; + if !supports_render_pass_test(&test) { + return; + } + let device = test.device(); + let make_buffer = || { + Arc::new( + Buffer::new( + device.clone(), + BufferDesc { + size: 256, + usage: vk::BufferUsageFlags::TRANSFER_SRC | vk::BufferUsageFlags::TRANSFER_DST, + queue_families: QueueFlags::empty(), + mem_location: MemoryLocation::CpuToGpu, + ..Default::default() + }, + ) + .expect("create render-pass test buffer"), + ) + }; + let producer_source = make_buffer(); + let producer_destination = make_buffer(); + let unrelated_destination = make_buffer(); + + let mut graph = RenderGraph::new(); + let target = graph.create_texture(GraphTextureDesc { + format: vk::Format::R8G8B8A8_UNORM, + extent: vk::Extent3D { + width: 16, + height: 16, + depth: 1, + }, + ..Default::default() + }); + let producer_source = graph + .import_buffer(producer_source, ResourceState::default()) + .expect("import producer source"); + let producer_destination = graph + .import_buffer(producer_destination, ResourceState::default()) + .expect("import producer destination"); + let unrelated_destination = graph + .import_buffer(unrelated_destination, ResourceState::default()) + .expect("import unrelated destination"); + let buffer_range = BufferRange { + offset: 0, + size: 256, + }; + let image_range = vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + }; + + graph.push(ClearTexture { + texture: target, + range: image_range, + layout: vk::ImageLayout::TRANSFER_DST_OPTIMAL, + color: [0.1, 0.2, 0.3, 1.0], + }); + let producer = graph.push(CopyBuffer { + src: producer_source, + src_range: buffer_range, + dst: producer_destination, + dst_range: buffer_range, + }); + let begin = graph.push(BeginRendering { + color: vec![ColorAttachment::created( + target, + ImageViewDesc::color_2d(), + vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + vk::AttachmentLoadOp::LOAD, + vk::AttachmentStoreOp::STORE, + vk::ClearValue::default(), + )], + depth: None, + }); + let interior = graph.push(PassNoop(vec![( + producer_destination.id(), + pass_buffer_read_scope(), + )])); + let end = graph.push(EndRendering); + let post_end = graph.push(ScopedNoop(vec![(target.id(), pass_texture_read_scope())])); + let unrelated = graph.push(CopyBuffer { + src: producer_destination, + src_range: buffer_range, + dst: unrelated_destination, + dst_range: buffer_range, + }); + + let mut compiled = graph.compile().expect("compile render-pass graph"); + let execution_order = compiled.execution_order(); + let position = |command| { + execution_order + .iter() + .position(|&item| item == command) + .unwrap() + }; + assert!(position(begin) < position(interior)); + assert!(position(interior) < position(end)); + assert!(position(unrelated) > position(end)); + let begin_barrier = compiled + .barriers() + .iter() + .find(|barrier| barrier.before_command == begin) + .expect("incoming barriers are hoisted before BeginRendering"); + assert!(begin_barrier.resources.iter().any(|resource| { + resource.id == target.id() + && resource.old_layout == Some(vk::ImageLayout::TRANSFER_DST_OPTIMAL) + && resource.new_layout == Some(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL) + })); + assert!(begin_barrier.resources.iter().any(|resource| { + resource.id == producer_destination.id() + && resource.origin == renderer_rendergraph::BarrierOrigin::Command(producer) + })); + assert!(compiled.barriers().iter().any(|barrier| { + barrier.before_command == post_end + && barrier.resources.iter().any(|resource| { + resource.id == target.id() + && resource.old_layout == Some(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL) + && resource.new_layout == Some(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) + }) + })); + + compiled + .resolve(device) + .expect("resolve created attachment view"); + assert!( + compiled + .created_view(target.id(), &ImageViewDesc::color_2d()) + .is_some() + ); + submit_graph(&test, &mut compiled); + test.assert_no_validation_errors(); + drop(compiled); + device + .wait_idle() + .expect("wait before render target teardown"); + test.assert_no_validation_errors(); +} diff --git a/crates/renderer_resources/src/images.rs b/crates/renderer_resources/src/images.rs index 4adb734..bbac6a0 100644 --- a/crates/renderer_resources/src/images.rs +++ b/crates/renderer_resources/src/images.rs @@ -1,6 +1,5 @@ use std::{ borrow::Cow, - mem::ManuallyDrop, ops::{Add, Sub}, sync::Arc, }; @@ -555,10 +554,7 @@ impl Image { // } // } - pub fn create_view( - self: &Arc, - mut desc: ImageViewDesc, - ) -> crate::Result> { + pub fn create_view(self: &Arc, mut desc: ImageViewDesc) -> crate::Result { // validate if !view_kind_compatible(self.desc.kind, desc.kind) { tracing::error!( @@ -618,7 +614,7 @@ impl Image { let device = self.image.device(); let view = unsafe { device.raw.create_image_view(&create_info, None)? }; - Ok(ManuallyDrop::new(ImageView { + Ok(ImageView { view: ImageViewHandle::Owned(DeviceObject::new_debug_named( device.clone(), view, @@ -626,7 +622,7 @@ impl Image { )), desc, image: self.clone(), - })) + }) } } @@ -1189,26 +1185,6 @@ fn image_view_subresource_range(desc: &ImageViewDesc) -> vk::ImageSubresourceRan .layer_count(desc.layer_range.len()) } -#[cfg(test)] -mod descriptor_range_tests { - use super::*; - - #[test] - fn normalized_range_accessor_resolves_remaining_sentinels() { - let mut desc = ImageViewDesc::color_2d() - .with_mip_range(1..) - .with_layer_range(2..); - desc.mip_range.set_max_end(6); - desc.layer_range.set_max_end(9); - let range = image_view_subresource_range(&desc); - assert_eq!(range.aspect_mask, vk::ImageAspectFlags::COLOR); - assert_eq!(range.base_mip_level, 1); - assert_eq!(range.level_count, 5); - assert_eq!(range.base_array_layer, 2); - assert_eq!(range.layer_count, 7); - } -} - pub struct QueueOwnership { pub src: u32, pub dst: u32, @@ -1336,3 +1312,23 @@ impl ImageSubresourceExt for vk::ImageSubresource { MipRange::single(self.mip_level) } } + +#[cfg(test)] +mod descriptor_range_tests { + use super::*; + + #[test] + fn normalized_range_accessor_resolves_remaining_sentinels() { + let mut desc = ImageViewDesc::color_2d() + .with_mip_range(1..) + .with_layer_range(2..); + desc.mip_range.set_max_end(6); + desc.layer_range.set_max_end(9); + let range = image_view_subresource_range(&desc); + assert_eq!(range.aspect_mask, vk::ImageAspectFlags::COLOR); + assert_eq!(range.base_mip_level, 1); + assert_eq!(range.level_count, 5); + assert_eq!(range.base_array_layer, 2); + assert_eq!(range.layer_count, 7); + } +} diff --git a/design-plans/rendergraph-render-passes.md b/design-plans/rendergraph-render-passes.md new file mode 100644 index 0000000..5df288f --- /dev/null +++ b/design-plans/rendergraph-render-passes.md @@ -0,0 +1,575 @@ +# Plan: Render-Pass Support (`BeginRendering` / `EndRendering`) in `renderer_rendergraph` + +Status: proposed — rev 5 (rev-1 correctness, rev-2 performance, rev-3 adversarial, and rev-4 re-review applied; rev-4 verdict was REJECT, findings resolved here) +Date: 2026-10-27 +Scope: `crates/renderer_rendergraph` + +## Goal + +Add dynamic-rendering render-pass support to the rendergraph: `BeginRendering` +and `EndRendering` commands that bracket a span of graph commands, with the +compiler guaranteeing that every command recorded between the two is emitted +inside the rendering instance — i.e. commands that must be in a rendergraph +pass are ordered by the begin/end rendering commands, and no foreign command +may be scheduled between them. + +## Current state (verified against code) + +- Commands implement `Command` (declares `side_effects` → `SubresourceScope` + accesses) + `RecordedCommand` (`preflight`/`apply`), and are pushed via + `RenderGraph::push` into `CommandList` (`commands.rs`, `recorder.rs`). +- `GraphCompiler::dependencies()` (`graph.rs:820-863`) builds hazard edges + purely from resource conflicts; `topo_batches_and_dependencies()` + (`graph.rs:870-889`) Kahn-sorts commands into `batches`; + `record_commands` (`graph.rs:2500-2554`) flattens batches, emitting barriers + (keyed `before_command` via `grouped_barrier_indices`) immediately before + each command. +- There is no notion of a render pass. Nothing prevents the flattener from + emitting an unrelated command between two commands the user recorded + adjacently: Kahn batching interleaves hazard-free commands freely. +- `CommandRecorder` (`recorder.rs:159-164`) maps `ResourceId → vk::Buffer` / + `vk::Image` only; there is no image-view plumbing. Descriptor imports read + `Arc` bindings at import time (`graph.rs:507-513`) but retain + only the underlying image (`StagedImportResource::Texture(view.image().clone())`, + `graph.rs:524-532`) — attachment views are new plumbing, not an existing + pattern to reuse. +- `CommandList` stores type-erased `Box` (`recorder.rs:97-109`); + the `Command` trait exposes only `side_effects` (`commands.rs:14-16`), so + there is currently no way to classify a command or introspect its render-pass + descriptor. +- The graph documents a single-queue submission contract (`graph.rs:8-10`); + ordinary barriers set `queue_transfer: None` (`graph.rs:972-982, 1001-1012`). +- Compile-time cost profile (recomputed independently): `dependencies()` is + `O(Σ_r u_r²)` pair checks (plus a per-pair scope cross-product factor); + `HappensBefore::from_dag` allocates one `FixedBitSet` row per node + (`graph.rs:213-217`, `O(N²/64)` memory) and unions per edge + (`O(E·N/64)` time). `ResourceInterference::from_resource_uses` **already** + computes terminal/initial frontier sets once per resource + (`graph.rs:1348-1356`); the remaining recomputations are in + `first_fit_placement` (`graph.rs:1642-1649`), the `overlapping` loop in + `plan_placements` (`graph.rs:2384-2399`), and `build_alias_barriers` + (`graph.rs:1088, 1109-1117`). Contraction does not remove those; see D8. +- The crate is not yet referenced by `renderer`/`bevy_vulkan_render`; the + current per-frame path is the legacy `renderer::render_graph`. If this + crate replaces it, `compile()` becomes per-frame, so compile cost matters. +- `dynamic_rendering` and `synchronization2` are requested/enabled + (`crates/renderer/src/lib.rs:352-357`, `crates/renderer_device/src/testing.rs:166-172`). +- **Pre-existing defect (reproduced):** `build_barriers` records only the last + *conflicting* producer's layout (`graph.rs:951-982`) and never chains + `old_layout` through earlier barriers. Two same-layout readers of one + producer yield two image barriers both claiming `oldLayout = producerLayout`; + the second is stale (Vulkan requires `oldLayout` be the current layout or + `UNDEFINED`). Fixing this generally (per-subresource current-layout map) is + part of Task 3 — the plan's own tests hit it. +- `Image::create_view` returns `Result>` + (`renderer_resources/src/images.rs:558-621`); storing it directly leaks the + `vk::ImageView` and the `Arc` (the existing renderer calls `.raw()` + on the `ManuallyDrop`, `crates/renderer/src/rendering/mod.rs:332`, which + leaks today). New code must use `ManuallyDrop::into_inner`. +- `ImageView` holds `Arc` (`images.rs:1123-1127`), so a view cache + keeps its image alive; drop-order interaction with `CompiledGraph` is benign + provided the `ManuallyDrop` is unwrapped. + +## Design decisions + +### D1 — Dynamic rendering, not `VkRenderPass` + +New commands `BeginRendering` / `EndRendering` wrapping `vkCmdBeginRendering` +/ `vkCmdEndRendering` (Vulkan 1.3 `dynamic_rendering`). Attachments are graph +`Texture` handles plus a view specification (see D9 for the created-texture +problem). + +### D2 — The pass span is an atomic scheduling unit, contracted *before* hazard-edge construction + +The compiler contracts the `[begin, end]` command span into a single DAG node +("pass node"). This is the mechanism that satisfies the ordering contract: + +- No foreign command can ever be scheduled inside the span (Kahn batching + would otherwise interleave hazard-free commands into it). +- The whole span is ordered relative to everything else by the begin/end + hazard edges. + +Contraction happens **before** `dependencies()` builds hazard edges. This +removes the `O(Σ u_r²)` interior-draw edge construction (edges that would be +discarded as self-edges anyway) and shrinks the `O(N²/64)` closure memory/time +by roughly `(N_c/N)²`. + +The contraction mapping used by every consumer: + +- **Type-safe node indexing:** use `NodeId(NonZeroU32)` newtypes and + `Option` rather than raw `u32` sentinels mixed into `CommandIndex` + spaces. To minimize churn, keep `batches: Vec>` with the + `BeginRendering` command index as the documented pass-node representative; + `spans: Vec` on `CompiledGraph` records `{ begin, end, interior }` + for expansion. Document the semantic change to the public `batches()` / + `dependency_edges()` accessors. Closure rows are indexed `0..N_c`. +- **HappensBefore is span-aware internally:** `HappensBefore` (public, + `lib.rs:10`) stores the command→node map and span slices and translates in + `contains`/`happens_before` (`graph.rs:244-251`), so command-space queries + keep working and interior indices never silently return `false` from a + node-space row lookup. +- **Intra-span record order is NOT a happens-before fact.** Interior commands + map to the pass node and same-node uses are treated as **unordered** by the + closure used for aliasing/interference. This is a memory-correctness + requirement, not a style choice: if intra-span record order were visible to + `resources_alias_in_order`, a created attachment `A` (single use at `Begin`) + and a created storage image `B` used only by an interior command would be + judged ordered, `first_fit_placement` could place `B` over `A`'s memory, and + `build_alias_barriers` would either emit no barrier or an illegal in-pass + `UNDEFINED → layout` transition — discarding a live render target mid-pass. + Intra-span record order is needed only by `build_barriers` (which iterates + execution order directly) and by recording; it must not leak into the + aliasing/interference closure. +- **Attachment liveness is modeled through `End`.** `EndRendering` carries + synthetic accesses (the attachment layout, `COLOR_ATTACHMENT_WRITE` / + `DEPTH_STENCIL_ATTACHMENT_WRITE` at the attachment stage) used **only** for + lifetime/alias analysis — excluded from barrier generation (the begin's + merged scope already covers synchronization) and from usage derivation. + This makes `terminal_uses(attachment) == End`, so an attachment is never + judged dead before the store. +- **Barrier analysis:** `build_barriers` walks scheduled command indices and + reads only `accesses[command_index]` (`graph.rs:930-934`). Barrier analysis + runs over the *uncontracted* access list in **execution order** (see D6), + with each interior command's scheduling position mapped to the pass node. + This preserves, e.g., the barrier for an outside producer consumed only by + an interior `Draw`. +- **Lifetimes:** `bind_lifetimes` indexes `batch_for_command` for every + original access (`graph.rs:1258-1275`); interior commands map to the pass + node's batch index so they are never left at `usize::MAX`. Placement uses + command-level interference, not batch intervals, so placement quality is + unaffected; external consumers of `resource_lifetimes()` get coarser + intervals for interior commands (documented). +- **Self-edges:** intra-span hazard pairs are skipped during edge + construction (they become intra-span execution-order constraints handled by + `build_barriers`, not DAG edges). Note `dependencies()` only adds + prior→current edges (`graph.rs:825-861`), so the DAG is acyclic regardless; + only a literal self-edge would trip `Kahn`, and skipping intra-span pairs + prevents exactly that. + +### D3 — Access declaration: one merged scope per attachment; interior accesses preserved + +- `BeginRendering` declares **one access per attachment** with + `access = COLOR_ATTACHMENT_WRITE | (LOAD ? COLOR_ATTACHMENT_READ : 0)` at + the same stage (`COLOR_ATTACHMENT_OUTPUT` / `EARLY|LATE_FRAGMENT_TESTS`), + subresource range from the view (concrete range rules in D9). Declaring + read and write as two separate accesses is a blocker: + `SideEffectMap::insert` deliberately keeps opposite-polarity scopes separate + (`recorder.rs:55-82`), and `build_barriers` then emits **two** + `ImageMemoryBarrier2` entries for the same image/range/layout in one + `vkCmdPipelineBarrier2` (reproduced). A single merged scope is handled + correctly by `SideEffectMap`, `conflicts`, `derive_image_usage` + (`graph.rs:2696-2709`), and the barrier builder. +- `EndRendering`'s synthetic accesses (D2) are tagged so barrier generation + and usage derivation skip them. +- Interior command accesses are *not* replaced by the begin declarations; + they remain in the access list (see D2 barrier analysis) so the compiler + can emit barriers for outside producers consumed inside the pass. +- Because the span is contracted, any later consumer is ordered after the + *entire pass node*, i.e. after `EndRendering` — matching store-visibility + semantics (verified: the consumer's barrier takes + `COLOR_ATTACHMENT_OUTPUT`/`COLOR_ATTACHMENT_WRITE` as src, and hoisting a + barrier before `BeginRendering` with those dst stages is sufficient + visibility for LOAD reads and in-pass attachment reads). +- **Intra-span hazard policy (narrowed):** + - **Allowed inside a span:** any barrier with `old_layout == new_layout` — + memory barriers, buffer barriers, *and* image barriers with no layout + change (a storage-image read/write hazard at `GENERAL` needs an + `ImageMemoryBarrier2`; buffer barriers do not synchronize image + accesses). These are legal inside a rendering instance. + - **Rejected at compile time** (`GraphError::UnsupportedIntraPassDependency`): + intra-span hazards requiring an image layout transition (`old != new`). + The rejection operates on the **generated barrier plan** (D5), not on an + independent declared-layout scan. + - **Feedback loops:** defined as any interior access (read *or* write) + whose `(aspect, mip, layer)` set intersects a `Begin` attachment + subresource, excluding the begin's own load operation. A storage-image + write to an attachment subresource is equally a feedback loop; an + interior read of a disjoint mip/layer is legal. Genuine feedback loops + require splitting. + - In-pass barrier emission is part of Task 4, not a follow-up: the recorder + emits the barrier between the two interior commands in execution order. +- **Aliasing vs. interior first uses is a placement constraint, not a + validation message.** For image resources, add an interference edge (or an + explicit `aliasing_allowed` predicate consulted by `plan_placements` / + `first_fit_placement`) whenever a required alias transition would land at a + command that maps to a pass node — covering interior *and* boundary + (`Begin`/`End`) positions. There is no "reject aliasing" hook after + `plan_placements` commits offsets, so this must be enforced before + placement. Attachments themselves are safe: with D2's `End` modeling, + their first use is `Begin` and their terminal use is `End`, both boundary + commands of the pass node itself. + +### D4 — Barrier placement: hoist cross-boundary barriers to BeginRendering, with per-subresource layout handling + +Barriers whose `before_command` targets an interior command but whose origin +lies outside the span are re-keyed to the span's `BeginRendering` command, +after batching when spans are known. Constraints: + +- **Barrier origin must be explicit.** `BarrierResource` (`graph.rs:84-95`) + records only masks/layout — no source command and no initial-vs-command + origin — so a post-pass hoisting pass cannot distinguish an outside-producer + barrier from an initial-state transition. Add + `src_command: Option` or a + `BarrierOrigin::{Initial, Command(CommandIndex)}` to `BarrierResource`. + **Initial-state transitions for interior first uses are hoisted to + `Begin`** and are not covered by the intra-span rejection (a transient + storage image first used inside a pass is the common case, not an edge + case). +- **Coalesce hoisted image transitions per `(resource, exact range, + target_layout)` into one barrier** with a unioned `dst_stage/dst_access` + (safe: the dst scope is a wait target; unioning only makes all consumers + wait). Layout chaining is defined **per `(aspect, mip, layer)`**, not per + barrier: for overlapping-but-unequal ranges, transition only the + subresources still in the producer layout; for disjoint ranges, emit + independent `producer → layout` transitions. The earlier "chain different + target layouts" rule is removed: for same-pass overlapping ranges it is + unreachable (D3 rejects the required in-pass transition), and for disjoint + ranges chaining `producer → A → B` is simply wrong (the second range is + still in the producer layout). +- The general `build_barriers` fix (per-subresource current-layout map, not + "last conflicting access's layout") makes non-hoisted multi-reader cases + correct too; D4's rules then only govern the hoisted subset. +- After hoisting, rebuild `barrier_indices_by_command` and + `ordinary_barrier_count` (`O(B)`), and apply the same coalescing in + `rebuild_barriers` (`graph.rs:2092-2106`) for alias barriers. + **Sequencing:** hoisting runs before `build_alias_barriers`, and hoisted + barriers live strictly within the ordinary prefix so `rebuild_barriers`' + truncation to `ordinary_barrier_count` (`graph.rs:2098`) cannot drop or + duplicate them; `rebuild_barriers` re-runs hoisting after the alias rebuild. +- Consumers after the pass keep their own barrier keys; expansion emits them + after `EndRendering` naturally. No barrier targets `EndRendering` itself + (its synthetic accesses are excluded from barrier generation). +- In-pass barriers (per D3) are emitted between the two interior commands in + execution order; they are not hoisted. + +### D5 — Compile-time validation on the generated barrier plan, all linear + +New `GraphError` variants for: + +- unbalanced begin/end; nested/overlapping spans (dynamic rendering cannot + nest); `EndRendering` before its `BeginRendering` in record order; +- graphics commands outside any span (`GraphError::DrawOutsideRenderPass`), + and **transfer/clear commands inside a span** (`CopyBuffer`, + `ClearTexture`, and any custom command not marked render-pass-legal — + `vkCmdCopyBuffer2`/`vkCmdClearColorImage` are illegal inside a rendering + instance). Classification uses a `fn kind(&self) -> CommandKind` method on + **`RecordedCommand`** (not `Command` — a method on `Command` is + unreachable after `Box` erasure), returning + `CommandKind::{Begin, End, Graphics, Transfer, Compute, Other}` plus a + `fn allowed_in_render_pass(&self) -> bool` (default false; `Draw` → true; + test/custom no-op commands may override so Task 5's in-pass barrier tests + can exist). **Note:** five existing unit tests compile graphs containing + `Draw` with no pass (pushes at `graph.rs:3563, 3749, 3815, 4329, 4469`) + and must be updated (wrap in begin/end or make the restriction opt-in) — + added to Task 5. +- unsupported intra-span dependencies (per D3). **The check operates on the + generated barrier plan after hoisting/chaining, not on an independent + declared-layout scan:** after hoisting, any barrier whose `before_command` + is interior and whose `old_layout != new_layout` is either hoistable + (initial/outside origin) or must be rejected. This is exact, `O(B)`, and + removes the fragile per-subresource validation machinery whose + `previous`-selection (`graph.rs:951-982`) can diverge from declared + layouts (e.g. producer `P` at layout `P0`, outside reader `R1` transitions + to `A`, interior reader `R2` reads at `A`: a declared-layout scan sees + `A → A` and accepts, while the un-fixed builder would emit `P0 → A` at + `R2` inside the pass). +- attachment consistency: each attachment view must refer to the same image + as its graph `Texture` — compare **raw handles** (`Image: PartialEq` + compares raw handles, `images.rs:484-488`; `Arc::ptr_eq` would + false-positive), consistent with import dedup by raw handle + (`graph.rs:389-399`). Applies to the imported case at compile time and to + the created case after view creation (D9). The recorder binds graph images + by `ResourceId` (`graph.rs:2513-2529`); this check closes that gap. +- attachment layout legality: the caller-specified layout must be a legal + rendering layout (`COLOR_ATTACHMENT_OPTIMAL`, `GENERAL`, etc.). The + previously proposed "layout must match compiler-produced state" rule is + vacuous — the barrier's `new_layout` is derived from the begin's own + declared access, so no independent state exists. +- duplicate attachment subresources: two attachments covering the same + subresource would be silently merged by `SideEffectMap::insert` + (`recorder.rs:55-82`), making them invisible from `command_accesses`; + reject via the introspection hook (below), using a per-subresource set + (`O(covered subresources)`, not pairwise `O(a²)`). +- queue ownership: passes cannot cross queues or require ownership + transfers; the single-queue submission contract (`graph.rs:8-10`) applies + unchanged, and any access requiring a queue transfer inside or across a + span is rejected until paired transfer/barrier support exists. +- **Introspection hook:** commands are type-erased, and `CommandKind` carries + no data, so Task 2's attachment validations (view/image identity, duplicate + subresources, layout legality) need a `fn render_pass_desc(&self) -> + Option<&RenderPassDesc>` method on `RecordedCommand` (or a validation hook + called from `compile()`, or an attachment summary stored in + `CommandList::push`, which is generic over `C: Command`). Without one, + these checks are not implementable as written. +- **Error-path safety:** `record_commands` returning mid-pass must not leave + an unclosed rendering instance; track an `in_render_pass` flag in the + recorder/compiled graph, end the rendering instance (or discard the command + buffer) on the error path, and reset the flag (`graph.rs:2540-2553`). + +### D6 — Execution order is precomputed at compile time + +Compile an `execution_order: Vec` once: expanded contracted +batches — begin → interiors in **original record order** → end. Barrier +analysis and recording both iterate it; `batches` is retained (begin-index +representative) for future parallel recording. This avoids per-frame +expansion branches and keeps `recording_steps_cached` (`graph.rs:2595-2608`) +allocation-free. Interior commands are emitted in record order, not topo +order: ordering within the pass is the user's responsibility, matching the +"ordered by begin/end" contract. + +**Batch-index assertion fix:** `build_barriers_with_initial_states` asserts +`batch_indices[previous] < batch_indices[current]` (`graph.rs:962-966`). +With interior commands mapped to the pass node's batch index, an allowed +intra-span hazard gives equal batch indices → debug panic. Use +**execution-order positions** for that assertion, not batch indices. + +### D7 — Recording is allocation-free per frame + +- `BeginRendering::apply` precomputes `vk::RenderingAttachmentInfo`s in the + command (or uses `ArrayVec`/thread-local scratch, mirroring + `pipeline_barrier`'s `BARRIER_SCRATCH` pattern, `recorder.rs:217-292`) so + no `Vec` is built per frame. Resolving attachment images per frame is + `O(a log R)` `BTreeMap` lookups — negligible. +- **View plumbing into `apply`:** `RecordedCommand::apply` receives only + `CommandRecorder` (`recorder.rs:108`), which cannot see `CompiledGraph`'s + view cache. Extend `CommandRecorder` with a view map + (`ResourceId → Arc`, or `ResourceId → Arc` plus + command-local view cache populated via `&mut self`) populated by + `record_commands`. `BeginRendering::apply` looks up the view; a missing or + unresolved view is a new `RecordError::MissingView`. Views are created + with `ManuallyDrop::into_inner(create_view(...))` — never store the + `ManuallyDrop` directly (leak). +- **Resolve idempotence:** views are created only after a fully successful + placement pass, atomically; a failed view creation must not leave a + partially-populated cache that later makes `Arc::get_mut` + (`graph.rs:2472-2480`) fail on a retried placement. Repeated `resolve()` + calls are idempotent; `create_view` failures map to `GraphError` variants. + +### D8 — Frontier reuse for placement and alias barriers (performance amendment, corrected) + +`ResourceInterference::from_resource_uses` already caches frontier sets per +resource (`graph.rs:1348-1356`); the plan does not claim to change it. The +actual recomputation hot spots contraction does not touch are +`first_fit_placement` (`graph.rs:1642-1649`), the `overlapping` loop in +`plan_placements` (`graph.rs:2384-2399`), and `build_alias_barriers` +(`terminal_uses` per old resource per new resource at `graph.rs:1109-1117`; +`initial_uses` per new at `graph.rs:1088`). Compute terminal/initial +frontiers **once per resource** and reuse them across all three sites, so +the performance rationale is honest: contraction removes the dependency-edge +and closure quadratics; frontier reuse removes the placement/alias ones. + +### D9 — Attachment API for created textures + +`RenderGraph::create_texture` returns only a `Texture` id (`graph.rs:352-354`); +the `Arc` is created privately in `plan_placements`/`resolve` +(`graph.rs:2129-2495`) and `CompiledGraph` exposes no accessor, so a caller +can never supply an `Arc` for a transient render target — the +main use case. Resolution: + +- `ColorAttachment.view` is an enum: + `AttachmentView::Created(ImageViewDesc) | Imported(Arc)`. +- **`Created(ImageViewDesc)` must carry concrete aspect/mip/layer ranges** + (`REMAINING_*` sentinels are rejected at validation): `side_effects` runs + at `push` time (`recorder.rs:119-124`), before `resolve()` creates any + image, `image_view_subresource_range` is private (`images.rs:1183`), and + `create_view` normalizes/mutates the desc (`set_max_end`, format fill-in, + `images.rs:566-589`; `MipRange` addition panics on overflow in debug for + `base > 0` with a `REMAINING` count, `images.rs:1319-1333`). The declared + access range therefore comes from the desc, and is validated against the + image's mip/layer counts at compile time (from `GraphTextureDesc` / + imported `ImageDesc`). +- The crate creates and caches views after `resolve()` (`Image::create_view` + needs `&Arc`, `images.rs:558`), keyed by `(Texture, ImageViewDesc)`; + `BeginRendering::apply` consumes the cached view via the D7 recorder view + map. +- The D5 view/image consistency check applies to the imported case at + compile time and to the created case after view creation. + +## Work items (all in `crates/renderer_rendergraph`) + +### Task 1 — New commands in `src/commands.rs` + +- `BeginRendering { color: Vec, depth: Option, ... }` + with `ColorAttachment { texture: Texture, view: AttachmentView (D9), + layout: vk::ImageLayout, load_op, store_op, clear_value }` (depth/stencil + analogous). `EndRendering` (unit struct). +- `Command::side_effects`: `BeginRendering` inserts **one merged scope per + attachment** (write + conditional LOAD read, same stage, concrete range + from the desc/view — D3, D9). `EndRendering` inserts synthetic + lifetime-only accesses (D2), tagged to be skipped by barrier generation + and usage derivation. +- Add `fn kind(&self) -> CommandKind` and + `fn allowed_in_render_pass(&self) -> bool` and + `fn render_pass_desc(&self) -> Option<&RenderPassDesc>` to + `RecordedCommand` (D5): `Draw` → `Graphics`/allowed; begin/end → + `Begin`/`End`; `CopyBuffer`/`ClearTexture` → `Transfer`/not allowed; + default `Other`/not allowed. +- `RecordedCommand::apply`: `BeginRendering` builds `vk::RenderingInfo` + + precomputed `vk::RenderingAttachmentInfo`s (resolving `vk::Image` via + `recorder.image(...)`, using the cached/created view via the D7 view map) + and calls `cmd_begin_rendering`; `EndRendering` calls `cmd_end_rendering`. + Allocation-free per D7. +- Export both from `lib.rs`. + +### Task 2 — Span tracking & validation in `src/graph.rs` + +- Derive spans in `compile()` by scanning command record order using + `CommandKind`; build `node_of_command` and `spans`. +- Validation pass implementing the D5 rules, including the raw-handle + view/image consistency check (imported case), layout legality, duplicate + attachment subresources (via the introspection hook), and the + classification-based inside/outside-pass checks. Linear per D5. +- Update the five existing unit tests that compile `Draw` without a pass + (pushes at `graph.rs:3563, 3749, 3815, 4329, 4469`). + +### Task 3 — Span contraction in the compiler + +- Contract spans **before** `dependencies()` per D2: aggregate access scopes + per pass node, skip intra-span pairs during edge construction, begin-index + node representation, contracted `dependency_edges`; `HappensBefore` stores + the command→node map and translates in queries. +- Fix the pre-existing stale-`old_layout` defect in `build_barriers` + (`graph.rs:951-982`) with a per-subresource current-layout map (D4). +- Add `BarrierOrigin` to `BarrierResource` (D4). +- Barrier analysis runs over the uncontracted access list in execution order + with interior positions mapped to the pass node (D2); switch the + `batch_indices` debug assertion to execution-order positions (D6). +- `bind_lifetimes`, `HappensBefore::from_dag`, `uses_by_resource`, and + `build_alias_barriers` thread the span map through per D2; all closure + consumers route through the span-aware query; same-node uses are unordered + for aliasing/interference (D2 blocker fix). +- Precompute `execution_order` per D6. +- Frontier reuse per D8 (`first_fit_placement`, `plan_placements` overlapping + loop, `build_alias_barriers`). + +### Task 4 — Barrier hoisting, in-pass barriers & recording + +- Hoist cross-boundary barriers per D4, including initial-state transitions + for interior first uses (`BarrierOrigin::Initial`); coalesce per + `(resource, exact range, target_layout)` with unioned dst masks; + per-`(aspect, mip, layer)` layout handling for overlapping/disjoint ranges. +- Enforce the aliasing placement constraint per D3 (interference edge / + `aliasing_allowed` predicate) before placement commits offsets. +- Emit in-pass barriers (any `old_layout == new_layout`) between interior + commands in execution order for allowed intra-span hazards (D3). +- Hoisting sequenced before `build_alias_barriers`, hoisted barriers within + the ordinary prefix, `rebuild_barriers` re-runs hoisting (D4). +- `recording_steps_cached` iterates `execution_order`, expanding pass nodes: + barriers (hoisted) → `BeginRendering` → interior commands in record order + (with in-pass barriers interleaved) → `EndRendering`. +- `record_commands`: expanded steps, D7 view map population, and the D5 + error-path safety (`in_render_pass` flag, no unclosed rendering instance + on mid-pass `RecordError`). + +### Task 5 — Tests + +- Unit tests in `graph.rs`: + - span validation errors (unbalanced, nested, draw-outside-pass, + transfer-inside-pass, view/image mismatch, layout legality, duplicate + attachment subresources, feedback-loop rejection); + - contraction correctness (foreign command never appears between begin and + end in `execution_order`; interior accesses still produce barriers; + lifetimes resolved for interior commands); + - barrier hoisting (barrier for an outside producer consumed by an interior + command is keyed to begin; **two interior consumers of one + outside-produced image coalesce into one barrier with unioned dst masks + and correct per-subresource `old_layout`** — regression test for the + reproduced blocker; also cover the non-hoisted two-reader case fixed by + the general `build_barriers` change); + - initial-state hoisting (transient storage image first used inside a pass + gets its `UNDEFINED → GENERAL` transition before `BeginRendering`); + - in-pass barrier emission (interior storage-buffer *and* storage-image + same-layout hazards produce barriers between the interior commands, no + pass split; layout-transition hazard is rejected); + - **aliasing safety (regression test for the rev-4 blocker):** created + attachment `A` + interior-only storage image `B` must **not** share + memory, and no barrier may target an interior command for `A`/`B`; + attachment `terminal_uses == End`; + - ordering of both incoming and outgoing dependencies around the whole + span; + - span-aware closure queries (`happens_before` with interior endpoints; + same-node uses unordered for aliasing). +- Integration tests in `tests/graph_validation.rs` (following the existing + `TestVulkan` pattern): + - skip when `core13.dynamic_rendering != TRUE` in addition to the existing + `synchronization2` check (`tests/graph_validation.rs:108-111`); + - the previously proposed `Draw`-based graph is **not constructible** + (`CopyBuffer` accepts only buffers, `commands.rs:30-79`; `Draw` has no + pipeline handle and never binds one, `commands.rs:143-153, 226-335`). + Use a custom no-op command (like `ScopedNoop`) with a texture read + access as the post-`End` consumer: + `ClearTexture → BeginRendering(color=target) → allowed no-op (texture + read) → EndRendering → no-op (texture read)` compiles, records, + submits, and validates. Adding a real `BindPipeline` command (with a + documented sequence edge to the following `Draw`, since pipeline binding + declares no resources) is an optional separate work item; + - a case proving a hazard-free foreign command is emitted *after* + `EndRendering`, not inside the pass; + - an outside-producer → interior-consumer case asserting the hoisted + barrier appears before `BeginRendering`. + +### Task 6 — Existing test/API churn + +- `batches()` assertions and helpers that assume command-space batches: + `graph.rs:3774, 4285, 4388-4393, 4674, 5237, 5245, 5257, 5337, 5343, 5397, + 5409, 5414` (including `recording_steps(compiled.batches(), …)` at 5343). +- `GraphCompiler::new` / `build_barriers` signature changes (execution order + + span map) break direct unit-test call sites at `graph.rs:3177-3178, 4125, + 4242, 4716-4718, 4792-4800, 4846-4854, 4893-4901, 4962-4976, 5051-5052, + 5102-5104, 5142-5145, 5171-5173`; `HappensBefore::from_dag` is called + directly by a test (`graph.rs:4140-4155`) and its signature changes. +- Wrapping a `Draw` in a pass shifts barrier indices in tests such as + `sampled_image_import_sets_usage_and_layout_barrier` (asserts + `compiled.barriers()[0]`, `graph.rs:3837-3845`). +- Provide compatibility constructors where cheap, or update all listed call + sites; `cargo test -p renderer_rendergraph` must pass at the end. + +## Sequencing + +Tasks 1–2 and 3–4 share types, so execute strictly in order: **1 → 2 → 3 → +4 → 5 → 6**, with `cargo check -p renderer_rendergraph` after each and +`cargo test -p renderer_rendergraph` at the end. No changes to other crates; +`renderer`'s legacy `render_graph` module is out of scope. + +## Performance summary (from rev-2 review, corrected in revs 3–5) + +- **Contraction before `dependencies()`** removes the interior-draw + `O(Σ u_r²)` edge construction and shrinks the `O(N²/64)` closure + memory/time by roughly `(N_c/N)²` (a pass spanning 90% of commands → + ~100× less closure work). +- **Contraction does not touch** the placement/alias frontiers; D8's + frontier reuse addresses `first_fit_placement`, `plan_placements`' + overlapping loop, and `build_alias_barriers` (interference frontier + caching already exists). +- **Barrier hoisting is cost-neutral to slightly positive** provided + coalescing follows D4 (per-target-layout, per-subresource handling, + unioned dst masks only for identical ranges/layouts). +- **In-pass barriers for same-layout hazards** avoid the significant + tiled-GPU regression of forced pass splits (attachment store/load + round-trips); only layout transitions and feedback loops force splits. +- **Validation is linear** with the barrier-plan-based intra-span check + (`O(B)`) and per-subresource-set duplicate detection; the existing + layout-conflict precheck grows by `O(a²)` pair iterations per pass + (28 for 8 attachments with the merged single access) — negligible. +- **Recording stays allocation-free** via precomputed attachment infos, + thread-local scratch, precomputed `execution_order`, and begin-index node + representation. + +## Risks / open points + +- **API shape:** raw `push(BeginRendering)` / `push(EndRendering)` vs. a + scoped `render_pass(desc, |graph| ...)` builder that pushes begin, runs the + closure, and pushes end while recording span command indices. The scoped + builder makes unbalanced spans unrepresentable; raw push is more flexible. + Decide before Task 1. +- **Created-texture views (D9):** view creation after `resolve()` adds a + cache keyed by `(Texture, ImageViewDesc)`; `ImageView` holds `Arc` + so lifetime is safe; resolve idempotence and atomic creation per D7. +- **Public API semantics:** `batches()` / `dependency_edges()` keep + command-space indices with the begin index as pass-node representative; + document the semantics and consider adding `execution_order()` as the + primary accessor. +- **`BindPipeline`:** out of scope for this plan; without it, submitted + tests cannot include real draws. If added later, it needs a sequence edge + to the following `Draw` (pipeline binding declares no resources, so the + hazard graph will not order it).