From 6274b6e5a8a3c3e8535253bc7c984c3cd7be2166 Mon Sep 17 00:00:00 2001 From: Janis Date: Thu, 9 Jan 2025 20:02:24 +0100 Subject: [PATCH] rendergraph resolves faster Access: layout is nolonger stored in option rendergraph resolved with linear vectors instead of maps --- crates/renderer/src/egui.rs | 20 +- crates/renderer/src/lib.rs | 3 +- crates/renderer/src/render_graph.rs | 826 +++++++++++++++++++++++----- crates/renderer/src/util.rs | 707 +----------------------- 4 files changed, 694 insertions(+), 862 deletions(-) diff --git a/crates/renderer/src/egui.rs b/crates/renderer/src/egui.rs index 3be5620..2efb061 100644 --- a/crates/renderer/src/egui.rs +++ b/crates/renderer/src/egui.rs @@ -184,7 +184,7 @@ pub fn egui_pre_pass( rg.import_image( img, Access { - layout: Some(vk::ImageLayout::GENERAL), + layout: vk::ImageLayout::GENERAL, ..Access::undefined() }, ), @@ -229,12 +229,12 @@ pub fn egui_pre_pass( Access { stage: vk::PipelineStageFlags2::NONE, mask: vk::AccessFlags2::empty(), - layout: Some(vk::ImageLayout::UNDEFINED), + layout: vk::ImageLayout::UNDEFINED, }, Access { stage: vk::PipelineStageFlags2::TRANSFER, mask: vk::AccessFlags2::TRANSFER_WRITE, - layout: Some(vk::ImageLayout::TRANSFER_DST_OPTIMAL), + layout: vk::ImageLayout::TRANSFER_DST_OPTIMAL, }, None, ) @@ -270,12 +270,12 @@ pub fn egui_pre_pass( Access { stage: vk::PipelineStageFlags2::TRANSFER, mask: vk::AccessFlags2::TRANSFER_WRITE, - layout: Some(vk::ImageLayout::TRANSFER_DST_OPTIMAL), + layout: vk::ImageLayout::TRANSFER_DST_OPTIMAL, }, Access { stage: vk::PipelineStageFlags2::TRANSFER, mask: vk::AccessFlags2::TRANSFER_READ, - layout: Some(vk::ImageLayout::TRANSFER_SRC_OPTIMAL), + layout: vk::ImageLayout::TRANSFER_SRC_OPTIMAL, }, None, ); @@ -285,12 +285,12 @@ pub fn egui_pre_pass( Access { stage: vk::PipelineStageFlags2::NONE, mask: vk::AccessFlags2::empty(), - layout: Some(vk::ImageLayout::GENERAL), + layout: vk::ImageLayout::GENERAL, }, Access { stage: vk::PipelineStageFlags2::TRANSFER, mask: vk::AccessFlags2::TRANSFER_WRITE, - layout: Some(vk::ImageLayout::TRANSFER_DST_OPTIMAL), + layout: vk::ImageLayout::TRANSFER_DST_OPTIMAL, }, None, ); @@ -334,12 +334,12 @@ pub fn egui_pre_pass( Access { stage: vk::PipelineStageFlags2::TRANSFER, mask: vk::AccessFlags2::TRANSFER_WRITE, - layout: Some(vk::ImageLayout::TRANSFER_DST_OPTIMAL), + layout: vk::ImageLayout::TRANSFER_DST_OPTIMAL, }, Access { stage: vk::PipelineStageFlags2::ALL_COMMANDS, mask: vk::AccessFlags2::empty(), - layout: Some(vk::ImageLayout::GENERAL), + layout: vk::ImageLayout::GENERAL, }, None, ) @@ -374,7 +374,7 @@ pub fn egui_pre_pass( ( *id, Access { - layout: Some(vk::ImageLayout::GENERAL), + layout: vk::ImageLayout::GENERAL, ..Access::undefined() }, ) diff --git a/crates/renderer/src/lib.rs b/crates/renderer/src/lib.rs index f23101a..846d7af 100644 --- a/crates/renderer/src/lib.rs +++ b/crates/renderer/src/lib.rs @@ -2535,7 +2535,6 @@ impl Renderer { let mut rg = render_graph::RenderGraph::new(); let (textures_to_remove, cmds) = util::timed("record command buffer", || { let framebuffer = rg.import_image(frame.image.clone(), Access::undefined()); - rg.mark_as_output(framebuffer); render_graph::clear_pass(&mut rg, clear_color, framebuffer); egui_pass::egui_pre_pass( @@ -2558,6 +2557,8 @@ impl Renderer { )?; render_graph::present_pass(&mut rg, framebuffer); + rg.mark_as_output(framebuffer, Access::present()); + Result::Ok((textures_to_remove, rg.resolve(dev.clone())?)) })?; diff --git a/crates/renderer/src/render_graph.rs b/crates/renderer/src/render_graph.rs index 0984de7..a3d876a 100644 --- a/crates/renderer/src/render_graph.rs +++ b/crates/renderer/src/render_graph.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] use std::{ - collections::{BTreeMap, BTreeSet}, + collections::BTreeMap, fmt::{Debug, Display}, sync::Arc, }; @@ -15,12 +15,6 @@ use crate::{ SwapchainFrame, }; use ash::vk; -use bytemuck::Contiguous; -use itertools::Itertools; -use petgraph::{ - graph::NodeIndex, - visit::{EdgeRef, IntoNodeReferences, NodeRef}, -}; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] @@ -125,7 +119,7 @@ impl RenderContext<'_> { pub struct Access { pub stage: vk::PipelineStageFlags2, pub mask: vk::AccessFlags2, - pub layout: Option, + pub layout: vk::ImageLayout, } impl core::ops::BitOr for Access { @@ -211,14 +205,14 @@ impl Access { Self { stage: vk::PipelineStageFlags2::NONE, mask: vk::AccessFlags2::empty(), - layout: None, + layout: vk::ImageLayout::UNDEFINED, } } pub fn undefined() -> Self { Self { stage: vk::PipelineStageFlags2::NONE, mask: vk::AccessFlags2::empty(), - layout: Some(vk::ImageLayout::UNDEFINED), + layout: vk::ImageLayout::UNDEFINED, } } /// Only use this for `Ord`! @@ -226,7 +220,7 @@ impl Access { Self { stage: vk::PipelineStageFlags2::from_raw(u64::MAX), mask: vk::AccessFlags2::from_raw(u64::MAX), - layout: None, + layout: vk::ImageLayout::UNDEFINED, } } /// Only use this for `Ord`! @@ -234,63 +228,63 @@ impl Access { Self { stage: vk::PipelineStageFlags2::from_raw(u64::MAX), mask: vk::AccessFlags2::from_raw(u64::MAX), - layout: Some(vk::ImageLayout::from_raw(i32::MAX)), + layout: vk::ImageLayout::from_raw(i32::MAX), } } pub fn general() -> Self { Self { stage: vk::PipelineStageFlags2::NONE, mask: vk::AccessFlags2::empty(), - layout: Some(vk::ImageLayout::GENERAL), + layout: vk::ImageLayout::GENERAL, } } pub fn transfer_read() -> Self { Self { stage: vk::PipelineStageFlags2::TRANSFER, mask: vk::AccessFlags2::TRANSFER_READ, - layout: Some(vk::ImageLayout::TRANSFER_SRC_OPTIMAL), + layout: vk::ImageLayout::TRANSFER_SRC_OPTIMAL, } } pub fn transfer_write() -> Self { Self { stage: vk::PipelineStageFlags2::TRANSFER, mask: vk::AccessFlags2::TRANSFER_WRITE, - layout: Some(vk::ImageLayout::TRANSFER_DST_OPTIMAL), + layout: vk::ImageLayout::TRANSFER_DST_OPTIMAL, } } pub fn vertex_read() -> Self { Self { stage: vk::PipelineStageFlags2::VERTEX_ATTRIBUTE_INPUT, mask: vk::AccessFlags2::VERTEX_ATTRIBUTE_READ, - layout: None, + layout: vk::ImageLayout::UNDEFINED, } } pub fn index_read() -> Self { Self { stage: vk::PipelineStageFlags2::INDEX_INPUT, mask: vk::AccessFlags2::INDEX_READ, - layout: None, + layout: vk::ImageLayout::UNDEFINED, } } pub fn indirect_read() -> Self { Self { stage: vk::PipelineStageFlags2::DRAW_INDIRECT, mask: vk::AccessFlags2::INDIRECT_COMMAND_READ, - layout: None, + layout: vk::ImageLayout::UNDEFINED, } } pub fn color_attachment_read_only() -> Self { Self { stage: vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT, mask: vk::AccessFlags2::COLOR_ATTACHMENT_READ, - layout: Some(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL), + layout: vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, } } pub fn color_attachment_write_only() -> Self { Self { stage: vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT, mask: vk::AccessFlags2::COLOR_ATTACHMENT_WRITE, - layout: Some(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL), + layout: vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, } } pub fn color_attachment_read_write() -> Self { @@ -298,14 +292,14 @@ impl Access { stage: vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT, mask: vk::AccessFlags2::COLOR_ATTACHMENT_WRITE | vk::AccessFlags2::COLOR_ATTACHMENT_READ, - layout: Some(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL), + layout: vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, } } pub fn present() -> Self { Self { stage: vk::PipelineStageFlags2::NONE, mask: vk::AccessFlags2::empty(), - layout: Some(vk::ImageLayout::PRESENT_SRC_KHR), + layout: vk::ImageLayout::PRESENT_SRC_KHR, } } } @@ -351,20 +345,21 @@ def_monotonic_id!(pub RenderGraphPassId); #[derive(Debug)] pub struct RenderGraph { resources: Vec, - accesses: Vec, pass_descs: Vec, /// the rendergraph produces these resources. Any passes on which these /// outputs do not depend are pruned. - outputs: Vec, + outputs: BTreeMap, } impl RenderGraph { pub fn new() -> Self { + let mut pass_descs = Vec::new(); + pass_descs.push(PassDesc::default()); + Self { resources: Vec::new(), - pass_descs: Vec::new(), - accesses: Vec::new(), - outputs: Vec::new(), + pass_descs, + outputs: BTreeMap::new(), } } @@ -372,16 +367,20 @@ impl RenderGraph { GraphResourceId(self.resources.len() as u32) } + fn input_pass_mut(&mut self) -> &mut PassDesc { + &mut self.pass_descs[0] + } + pub fn add_resource(&mut self, desc: GraphResourceDesc) -> GraphResourceId { let id = self.get_next_resource_id(); self.resources.push(desc.into()); - self.accesses.push(Access::undefined()); id } - pub fn mark_as_output(&mut self, id: GraphResourceId) { - // TODO: dedup - self.outputs.push(id); + + pub fn mark_as_output(&mut self, id: GraphResourceId, access: Access) { + _ = self.outputs.try_insert(id, access); } + pub fn import_resource(&mut self, res: GraphResource, access: Access) -> GraphResourceId { if let Some(i) = self .resources @@ -392,21 +391,25 @@ impl RenderGraph { } else { let id = self.get_next_resource_id(); self.resources.push(res); - self.accesses.push(access); + self.input_pass_mut().writes.push((id, access)); id } } + pub fn import_image(&mut self, image: Arc, access: Access) -> GraphResourceId { let res = GraphResource::ImportedImage(image); self.import_resource(res, access) } + pub fn import_buffer(&mut self, buffer: Arc, access: Access) -> GraphResourceId { let res = GraphResource::ImportedBuffer(buffer); self.import_resource(res, access) } + pub fn import_framebuffer(&mut self, frame: Arc) -> GraphResourceId { self.import_resource(GraphResource::Framebuffer(frame), Access::undefined()) } + pub fn add_pass(&mut self, pass: PassDesc) { self.pass_descs.push(pass); } @@ -418,21 +421,27 @@ impl RenderGraph { &mut self, device: device::Device, ) -> crate::Result>> { - let now = std::time::Instant::now(); + let output_reads = self + .outputs + .iter() + .map(|(rid, access)| (*rid, *access)) + .collect::>(); + self.add_pass(PassDesc { + reads: output_reads, + writes: vec![], + ..Default::default() + }); - let mut refmap = util::asdf::NodeRefsMap::new(self.resources.len(), self.pass_descs.len()); + let topo = util::timed("Resolving Render Graph", || { + let mut refmap = + graph_resolver::NodeRefsMap::new(self.resources.len(), self.pass_descs.len()); - refmap.allocate_ref_ranges(&self.pass_descs); - refmap.ref_passes(&self.pass_descs); - refmap.ref_inputs(&self.resources); - refmap.ref_outputs(&self.outputs); - let dag = refmap.build_dag(); - let topo = refmap.toposort_dag(dag); - - eprintln!( - "resolved render graph in {}ms", - now.elapsed().as_micros() as f32 / 1e3 - ); + refmap.allocate_ref_ranges(&self.pass_descs); + refmap.ref_passes(&self.pass_descs); + let dag = refmap.build_dag(); + let topo = refmap.toposort_dag(dag); + topo + }); // create internal resources: for (i, res) in self.resources.iter_mut().enumerate() { @@ -463,8 +472,7 @@ impl RenderGraph { .map(|(passes, accesses)| { let passes = passes .into_iter() - .filter_map(|i| i.unpack().get_pass_idx()) - .map(|i| core::mem::take(&mut self.pass_descs[i])) + .map(|i| core::mem::take(&mut self.pass_descs[i.index()])) .collect::>(); (passes, accesses) }) @@ -512,7 +520,7 @@ impl RenderGraph { let outputs = self .outputs .iter() - .map(|id| (*id, core::mem::take(&mut self.resources[id.0 as usize]))) + .map(|(id, _)| (*id, core::mem::take(&mut self.resources[id.0 as usize]))) .collect::>(); outputs @@ -569,6 +577,626 @@ impl RenderGraph { } } +mod graph_resolver { + use std::collections::{BTreeMap, BTreeSet}; + use std::fmt::Display; + + use ash::vk; + use petgraph::visit::EdgeRef; + + use crate::render_graph::*; + + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] + #[repr(transparent)] + pub struct PassNode(u16); + + impl PassNode { + pub fn index(&self) -> usize { + self.0 as usize + } + pub fn as_u32(&self) -> u32 { + self.0 as u32 + } + pub fn pass(i: usize) -> Self { + Self(i as u16) + } + } + + #[derive(Debug, Clone, Copy)] + pub enum Barrier { + Logical, + Execution { + src: vk::PipelineStageFlags2, + dst: vk::PipelineStageFlags2, + }, + LayoutTransition { + src: (vk::PipelineStageFlags2, vk::ImageLayout), + dst: (vk::PipelineStageFlags2, vk::ImageLayout), + }, + + MakeAvailable { + src: (vk::PipelineStageFlags2, vk::AccessFlags2), + dst: vk::PipelineStageFlags2, + }, + MakeVisible { + src: vk::PipelineStageFlags2, + dst: (vk::PipelineStageFlags2, vk::AccessFlags2), + }, + MemoryBarrier { + src: (vk::PipelineStageFlags2, vk::AccessFlags2), + dst: (vk::PipelineStageFlags2, vk::AccessFlags2), + }, + } + + impl Display for Barrier { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Barrier::Logical => write!(f, "Logical"), + Barrier::Execution { .. } => write!(f, "Execution"), + Barrier::LayoutTransition { .. } => write!(f, "Layout"), + Barrier::MakeAvailable { .. } => write!(f, "MakeAvailable"), + Barrier::MakeVisible { .. } => write!(f, "MakeVisible"), + Barrier::MemoryBarrier { .. } => write!(f, "MemoryBarrier"), + } + } + } + + pub struct NodeRefsMap { + num_resources: usize, + num_passes: usize, + // bitmap of passes referencing rid + references: Vec, + + // range into ref_accesses*: start, end, index + ref_ranges: Vec<(u32, u32, u32)>, + ref_accesses: Vec<(Option, Option)>, + ref_access_passid: Vec, + } + + impl NodeRefsMap { + pub fn new(num_resources: usize, num_passes: usize) -> Self { + Self { + num_resources, + num_passes, + references: vec![0; ((num_passes) * num_resources).div_ceil(64) as usize], + ref_ranges: Vec::new(), + ref_accesses: Vec::new(), + ref_access_passid: Vec::new(), + } + } + + pub fn allocate_ref_ranges(&mut self, passes: &[PassDesc]) { + let mut rid_passcount = vec![0; self.num_resources]; + + for pass in passes.iter() { + for rid in pass + .reads + .iter() + .chain(pass.writes.iter()) + .map(|id| id.0) + .collect::>() + { + rid_passcount[rid.0 as usize] += 1; + } + } + + tracing::debug!("per-resource pass-count: {rid_passcount:?}"); + + let mut total = 0; + for num_passes in rid_passcount { + self.ref_ranges.push((total, total + num_passes, 0)); + self.ref_accesses + .extend((0..num_passes).map(|_| (None, None))); + self.ref_access_passid + .extend((0..num_passes).map(|_| PassNode(0))); + total += num_passes; + } + + tracing::debug!( + "ref_ranges and ref_accesses:\n{:?}\n{:?}\n{:?}", + self.ref_ranges, + self.ref_accesses, + self.ref_access_passid + ); + // for resourcedesc in resources: ref first pass + } + + fn get_accesses_for_rid_pass_mut( + &mut self, + rid: GraphResourceId, + pass: PassNode, + ) -> &mut (Option, Option) { + let (start, _, i) = self.ref_ranges[rid.0 as usize]; + + let idx = self.ref_access_passid[start as usize..(start + i) as usize] + .binary_search(&pass) + .unwrap_or_else(|_| { + // increase counter + self.ref_ranges[rid.0 as usize].2 += 1; + i as usize + }) + + start as usize; + + self.ref_access_passid[idx] = pass; + + &mut self.ref_accesses[idx] + } + + fn get_reads_for_rid_pass_mut( + &mut self, + rid: GraphResourceId, + pass: PassNode, + ) -> &mut Option { + &mut self.get_accesses_for_rid_pass_mut(rid, pass).0 + } + fn get_writes_for_rid_pass_mut( + &mut self, + rid: GraphResourceId, + pass: PassNode, + ) -> &mut Option { + &mut self.get_accesses_for_rid_pass_mut(rid, pass).1 + } + + fn get_accesses_for_rid_pass( + &self, + rid: GraphResourceId, + pass: PassNode, + ) -> (Option, Option) { + let (start, _, i) = self.ref_ranges[rid.0 as usize]; + + let Some(idx) = self.ref_access_passid[start as usize..(start + i) as usize] + .binary_search(&pass) + .ok() + else { + return (None, None); + }; + let idx = idx + start as usize; + + self.ref_accesses[idx] + } + + fn get_reads_for_rid_pass(&self, rid: GraphResourceId, pass: PassNode) -> Option { + self.get_accesses_for_rid_pass(rid, pass).0 + } + fn get_writes_for_rid_pass(&self, rid: GraphResourceId, pass: PassNode) -> Option { + self.get_accesses_for_rid_pass(rid, pass).1 + } + + fn reference_rid_pass(&mut self, rid: GraphResourceId, pass: PassNode) { + let bit_idx = rid.0 as usize * (self.num_passes) + pass.index(); + let word_idx = bit_idx / 64; + let word_offset = bit_idx % 64; + tracing::debug!( + bit_idx, + word_idx, + word_offset, + "pass: {pass:?} references rid: {rid:?} " + ); + self.references[word_idx] |= 1 << word_offset; + } + + pub fn ref_passes(&mut self, passes: &[PassDesc]) { + for (i, pass) in passes.iter().enumerate() { + let pass_id = PassNode::pass(i); + + for &(rid, access) in &pass.reads { + let read = self + .get_reads_for_rid_pass_mut(rid, pass_id) + .get_or_insert(Access::empty()); + *read = *read | access; + + // TODO: check for first pass as well + self.reference_rid_pass(rid, PassNode::pass(i)); + } + + for &(rid, access) in &pass.writes { + let write = self + .get_writes_for_rid_pass_mut(rid, pass_id) + .get_or_insert(Access::empty()); + *write = *write | access; + + // TODO: check for first pass as well + self.reference_rid_pass(rid, PassNode::pass(i)); + } + } + + tracing::debug!( + "ref_ranges and ref_accesses:\n{:?}\n{:?}\n{:?}", + self.ref_ranges, + self.ref_accesses, + self.ref_access_passid + ); + } + + pub fn build_dag( + &self, + ) -> petgraph::stable_graph::StableDiGraph { + struct Edge { + from: PassNode, + to: PassNode, + rid: GraphResourceId, + barrier: Barrier, + } + + #[derive(Debug, Clone, Copy)] + enum Ref { + Write(PassNode, Access), + Read(PassNode, Access), + } + + impl Ref { + fn node(&self) -> PassNode { + match self { + Ref::Write(node, _) | Ref::Read(node, _) => *node, + } + } + fn access(&self) -> Access { + match self { + Ref::Write(_, access) | Ref::Read(_, access) => *access, + } + } + } + + let mut edges = Vec::::new(); + + let bits = + crate::util::BitIter::new(&self.references, self.num_resources * (self.num_passes)) + .chunks(self.num_passes); + + tracing::debug!("building edges:"); + tracing::debug!("chunks: {bits:#?}"); + for (i, bits) in bits.enumerate() { + let rid = GraphResourceId(i as u32); + tracing::debug!("rid: {rid:?}"); + tracing::debug!("passes: {bits}"); + + let mut to_make_available = AccessMask::empty(); + let mut made_available = AccessMask::empty(); + + let mut last_ref = Option::::None; + let mut last_write = Option::::None; + + for pass in bits { + let pass = PassNode::pass(pass); + tracing::debug!("pass: {:?}", pass); + tracing::debug!("accesses: {:?}", self.get_accesses_for_rid_pass(rid, pass)); + + if let Some(last_ref) = last_ref.as_ref() { + edges.push(Edge { + from: last_ref.node(), + to: pass, + rid, + barrier: Barrier::Logical, + }); + } + + let read = self.get_reads_for_rid_pass(rid, pass); + if let Some(read) = read { + tracing::debug!("read: {:?}", read); + + let make_visible = read.into_access_mask() & !made_available; + if let Some(last_write) = last_write.as_ref() { + let from = last_write.node(); + let to = pass; + let from_write = last_write.access(); + + // if last_write is some, make visible the writes + if !make_visible.is_empty() && !from_write.stage.is_empty() { + made_available = made_available | make_visible; + + edges.push(Edge { + from, + to, + rid, + barrier: Barrier::MakeVisible { + src: from_write.stage, + dst: (make_visible.stage, make_visible.mask), + }, + }); + } + + // make available any changes + if !to_make_available.is_empty() { + edges.push(Edge { + from, + to, + rid, + barrier: Barrier::MakeAvailable { + src: (to_make_available.stage, to_make_available.mask), + dst: read.stage, + }, + }); + to_make_available = AccessMask::empty(); + } + + if make_visible.is_empty() && !to_make_available.is_empty() { + // still require a-after-b + edges.push(Edge { + from, + to, + rid, + barrier: Barrier::Execution { + src: from_write.stage, + dst: read.stage, + }, + }); + } + } + + // layout transition from previous pass, either read or write + if let Some(last_ref) = last_ref.as_ref() { + if last_ref.access().layout != read.layout { + let from = last_ref.node(); + let to = pass; + edges.push(Edge { + from, + to, + rid, + barrier: Barrier::LayoutTransition { + src: (last_ref.access().stage, last_ref.access().layout), + dst: (read.stage, read.layout), + }, + }); + } + } + } + + let write = self.get_writes_for_rid_pass(rid, pass); + if let Some(write) = write { + tracing::debug!("write: {:?}", write); + match last_ref.as_ref() { + Some(Ref::Read(node, before)) => { + // execution barrier to ward against write-after-read + + edges.push(Edge { + from: *node, + to: pass, + rid, + barrier: Barrier::Execution { + src: before.stage, + dst: write.stage, + }, + }); + } + Some(Ref::Write(node, before)) => { + // check for layout transition here + if before.layout != write.layout { + edges.push(Edge { + from: *node, + to: pass, + rid, + barrier: Barrier::LayoutTransition { + src: (before.stage, before.layout), + dst: (write.stage, write.layout), + }, + }); + } + } + _ => {} + } + } + + if let Some(read) = read { + last_ref = Some(Ref::Read(pass, read)); + } + if let Some(write) = write { + last_write = Some(Ref::Write(pass, write)); + last_ref = last_write; + } + } + } + + let mut dag = + petgraph::stable_graph::StableDiGraph::::new( + ); + + for i in 0..self.num_passes { + dag.add_node(PassNode::pass(i)); + } + + // insert edges + for edge in edges { + let Edge { + from, + to, + rid, + barrier, + } = edge; + dag.add_edge(from.as_u32().into(), to.as_u32().into(), (rid, barrier)); + } + + // prune dead ends + let mut sinks = dag + .externals(petgraph::Direction::Outgoing) + .filter(|idx| dag.node_weight(*idx) != Some(&PassNode::pass(self.num_passes - 1))) + .collect::>(); + + while let Some(sink) = sinks.pop() { + let mut neighbors = dag + .neighbors_directed(sink, petgraph::Direction::Incoming) + .detach(); + while let Some((edge, node)) = neighbors.next(&dag) { + dag.remove_edge(edge); + + if dag + .neighbors_directed(node, petgraph::Direction::Outgoing) + .count() + == 0 + { + sinks.push(node); + } + } + + tracing::debug!("removing {sink:?} from dag"); + dag.remove_node(sink); + } + + #[cfg(any(debug_assertions, test))] + std::fs::write( + "render_graph2.dot", + &format!( + "{:?}", + petgraph::dot::Dot::with_attr_getters( + &dag, + &[], + &|_graph, edgeref| { + format!( + "label = \"{},{:#?}\"", + edgeref.weight().0, + edgeref.weight().1, + ) + }, + &|_graph, noderef| { + format!( + "label = \"Pass({:?})\"", + petgraph::visit::NodeRef::weight(&noderef) + ) + } + ) + ), + ) + .expect("writing render_graph repr"); + + dag + } + + pub fn toposort_dag( + &self, + mut dag: petgraph::stable_graph::StableDiGraph, + ) -> Vec<(Vec, BTreeMap)> { + let mut topomap = Vec::new(); + + let mut sinks = dag + .externals(petgraph::Direction::Outgoing) + .collect::>(); + let mut next_sinks = BTreeSet::new(); + + loop { + tracing::debug!("sinks: {sinks:?}, next_sinks: {next_sinks:?}"); + + if sinks.is_empty() { + break; + } + + let mut passes = Vec::with_capacity(self.num_passes); + let mut barriers = BTreeMap::new(); + for &sink in sinks.iter() { + for &(rid, barrier) in dag + .edges_directed(sink, petgraph::Direction::Incoming) + .map(|edge| edge.weight()) + { + let before_and_after = match barrier { + Barrier::Logical => None, + Barrier::Execution { src, dst } => Some(( + Access { + stage: src, + ..Access::empty() + }, + Access { + stage: dst, + ..Access::empty() + }, + )), + Barrier::LayoutTransition { + src: (src, from), + dst: (dst, to), + } => Some(( + Access { + stage: src, + layout: from, + ..Access::empty() + }, + Access { + stage: dst, + layout: to, + ..Access::empty() + }, + )), + Barrier::MakeAvailable { + src: (stage, mask), + dst, + } => Some(( + Access { + stage, + mask, + ..Access::empty() + }, + Access { + stage: dst, + ..Access::empty() + }, + )), + Barrier::MakeVisible { + src, + dst: (stage, mask), + } => Some(( + Access { + stage: src, + ..Access::empty() + }, + Access { + stage, + mask, + ..Access::empty() + }, + )), + Barrier::MemoryBarrier { + src: (src_stage, src_mask), + dst: (dst_stage, dst_mask), + } => Some(( + Access { + stage: src_stage, + mask: src_mask, + ..Access::empty() + }, + Access { + stage: dst_stage, + mask: dst_mask, + ..Access::empty() + }, + )), + }; + + if let Some((before, after)) = before_and_after { + // initial access is transitioned at the beginning + // this affects imported resources only. + barriers + .entry(rid) + .and_modify(|(from, to)| { + *from = *from | before; + *to = *to | after; + }) + .or_insert((before, after)); + } + } + + dag.edges_directed(sink, petgraph::Direction::Incoming) + .for_each(|edge| { + let node = edge.source(); + if dag + .neighbors_directed(node, petgraph::Direction::Outgoing) + .all(|node| node == sink) + { + next_sinks.insert(node); + } + }); + + passes.push(*dag.node_weight(sink).unwrap()); + dag.remove_node(sink); + } + + topomap.push((passes, barriers)); + sinks.clear(); + core::mem::swap(&mut sinks, &mut next_sinks); + } + + tracing::debug!("topomap: {topomap:?}"); + + topomap + } + } +} + pub enum Barrier { Image(vk::ImageMemoryBarrier2<'static>), Buffer(vk::BufferMemoryBarrier2<'static>), @@ -638,8 +1266,8 @@ pub fn image_barrier( .dst_access_mask(after_access.mask) .dst_stage_mask(after_access.stage) .image(image) - .old_layout(before_access.layout.unwrap_or_default()) - .new_layout(after_access.layout.unwrap_or_default()) + .old_layout(before_access.layout) + .new_layout(after_access.layout) .subresource_range(vk::ImageSubresourceRange { aspect_mask: util::image_aspect_from_format(format), ..images::SUBRESOURCERANGE_ALL @@ -656,104 +1284,6 @@ pub fn image_barrier( ) } -// #[cfg(test)] -// mod tests { -// use super::*; - -// macro_rules! def_dummy_pass { -// ($name:ident: {$queue:path, $layout_in:path, $layout_out:path}) => { -// #[derive(Debug, Clone)] -// struct $name(Vec, Vec); -// impl Pass for $name { -// fn get_read_resource_access(&self, _id: RenderGraphResourceId) -> ResourceAccess { -// ResourceAccess::default() -// } -// fn get_write_resource_access(&self, _id: RenderGraphResourceId) -> ResourceAccess { -// ResourceAccess::default() -// } - -// fn get_queue_capability_requirements(&self) -> device::QueueFlags { -// $queue -// } - -// fn get_read_dependencies<'a>( -// &'a self, -// ) -> Box + 'a> { -// Box::new(self.0.iter().cloned()) -// } - -// fn get_write_dependencies<'a>( -// &'a self, -// ) -> Box + 'a> { -// Box::new(self.1.iter().cloned()) -// } - -// fn record(&self, _ctx: &RenderContext) -> crate::Result<()> { -// Ok(()) -// } -// } -// }; -// } - -// def_dummy_pass!(DepthPass: { -// device::QueueFlags::GRAPHICS, -// vk::ImageLayout::DEPTH_ATTACHMENT_OPTIMAL, -// vk::ImageLayout::DEPTH_ATTACHMENT_OPTIMAL}); -// def_dummy_pass!(RenderPass: { -// device::QueueFlags::GRAPHICS, -// vk::ImageLayout::COLORiATTACHMENT_OPTIMAL, -// vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL}); -// def_dummy_pass!(AsyncPass: { -// device::QueueFlags::ASYNC_COMPUTE, -// vk::ImageLayout::UNDEFINED, -// vk::ImageLayout::GENERAL}); -// def_dummy_pass!(PostProcessPass: { -// device::QueueFlags::ASYNC_COMPUTE, -// vk::ImageLayout::GENERAL, -// vk::ImageLayout::GENERAL}); -// def_dummy_pass!(PresentPass: { -// device::QueueFlags::PRESENT, -// vk::ImageLayout::PRESENT_SRC_KHR, -// vk::ImageLayout::UNDEFINED}); -// def_dummy_pass!(DepthVisualisationPass: { -// device::QueueFlags::ASYNC_COMPUTE, -// vk::ImageLayout::GENERAL, -// vk::ImageLayout::UNDEFINED}); - -// #[test] -// fn resolve_graph() { -// let mut graph = RenderGraph::new(); -// let gbuffer = graph.add_resource(RenderGraphResourceDesc::Image(ImageDesc { -// ..Default::default() -// })); -// let depth_image = graph.add_resource(RenderGraphResourceDesc::Image(ImageDesc { -// ..Default::default() -// })); -// let depth_visualisation = graph.add_resource(RenderGraphResourceDesc::Image(ImageDesc { -// ..Default::default() -// })); -// let compute_buffer = graph.add_resource(RenderGraphResourceDesc::Buffer(BufferDesc { -// ..Default::default() -// })); - -// graph.add_pass(DepthPass(vec![depth_image], vec![depth_image])); -// graph.add_pass(DepthVisualisationPass( -// vec![depth_image, depth_visualisation], -// vec![depth_visualisation], -// )); -// graph.add_pass(AsyncPass(vec![compute_buffer], vec![compute_buffer])); -// graph.add_pass(RenderPass( -// vec![depth_image, compute_buffer, gbuffer], -// vec![gbuffer], -// )); -// graph.add_pass(PostProcessPass(vec![gbuffer], vec![gbuffer])); -// graph.mark_as_output(gbuffer); -// graph.mark_as_output(depth_image); - -// // graph.resolve(); -// } -// } - pub fn clear_pass(rg: &mut RenderGraph, color: Rgba, target: GraphResourceId) { let reads = [(target, Access::transfer_write())].to_vec(); let writes = [(target, Access::transfer_write())].to_vec(); diff --git a/crates/renderer/src/util.rs b/crates/renderer/src/util.rs index 1be9b69..bc63064 100644 --- a/crates/renderer/src/util.rs +++ b/crates/renderer/src/util.rs @@ -424,707 +424,8 @@ bitflags::bitflags! { } } -pub mod asdf { - use std::collections::{BTreeMap, BTreeSet}; - use std::fmt::Display; - - use ash::vk; - - use crate::render_graph::*; - - #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] - #[repr(u8)] - pub enum PassNode { - First, - Pass(u16), - Last, - } - - #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] - #[repr(transparent)] - pub struct PackedPassNode(u16); - impl PackedPassNode { - pub fn unpack(self) -> PassNode { - match self.0 { - 0 => PassNode::First, - 1 => PassNode::Last, - n => PassNode::Pass(n - 2), - } - } - pub fn pack(i: PassNode) -> Self { - Self(match i { - PassNode::First => 0, - PassNode::Last => 1, - PassNode::Pass(n) => n + 2, - }) - } - } - - impl From for PackedPassNode { - fn from(value: PassNode) -> Self { - Self::pack(value) - } - } - impl From for PassNode { - fn from(value: PackedPassNode) -> Self { - PackedPassNode::unpack(value) - } - } - - impl PassNode { - pub fn dag_index(&self) -> u32 { - match self { - PassNode::First => 0, - PassNode::Last => 1, - PassNode::Pass(i) => 2 + *i as u32, - } - } - pub fn into_u32(&self, max_i: u32) -> u32 { - match self { - PassNode::First => 0, - PassNode::Last => max_i + 1, - PassNode::Pass(i) => 1 + *i as u32, - } - } - fn range_full(from: Self, to: Self, max_i: u32) -> std::ops::RangeInclusive { - from.into_u32(max_i)..=to.into_u32(max_i) - } - pub fn pass(i: usize) -> Self { - Self::Pass(i as u16) - } - pub fn get_pass_idx(&self) -> Option { - match self { - PassNode::First | PassNode::Last => None, - PassNode::Pass(i) => Some(*i as usize), - } - } - pub fn from_u32(v: u32, max_i: u32) -> Self { - match v { - 0 => Self::First, - n if n == 1 + max_i => Self::Last, - n => Self::Pass(n as u16 - 1), - } - } - } - - #[derive(Debug, Clone, Copy)] - pub enum Barrier { - Logical, - Execution { - src: vk::PipelineStageFlags2, - dst: vk::PipelineStageFlags2, - }, - LayoutTransition { - src: (vk::PipelineStageFlags2, vk::ImageLayout), - dst: (vk::PipelineStageFlags2, vk::ImageLayout), - }, - - MakeAvailable { - src: (vk::PipelineStageFlags2, vk::AccessFlags2), - dst: vk::PipelineStageFlags2, - }, - MakeVisible { - src: vk::PipelineStageFlags2, - dst: (vk::PipelineStageFlags2, vk::AccessFlags2), - }, - MemoryBarrier { - src: (vk::PipelineStageFlags2, vk::AccessFlags2), - dst: (vk::PipelineStageFlags2, vk::AccessFlags2), - }, - } - - impl Display for Barrier { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Barrier::Logical => write!(f, "Logical"), - Barrier::Execution { .. } => write!(f, "Execution"), - Barrier::LayoutTransition { .. } => write!(f, "Layout"), - Barrier::MakeAvailable { .. } => write!(f, "MakeAvailable"), - Barrier::MakeVisible { .. } => write!(f, "MakeVisible"), - Barrier::MemoryBarrier { .. } => write!(f, "MemoryBarrier"), - } - } - } - - pub struct NodeRefsMap { - num_resources: usize, - num_passes: usize, - // bitmap of passes referencing rid - references: Vec, - - // range into ref_accesses*: start, end, index - ref_ranges: Vec<(u32, u32, u32)>, - ref_accesses: Vec<(Access, Access)>, - ref_access_passid: Vec, - } - - impl NodeRefsMap { - pub fn new(num_resources: usize, num_passes: usize) -> Self { - Self { - num_resources, - num_passes, - references: vec![0; ((num_passes + 2) * num_resources).div_ceil(64) as usize], - ref_ranges: Vec::new(), - ref_accesses: Vec::new(), - ref_access_passid: Vec::new(), - } - } - - pub fn allocate_ref_ranges(&mut self, passes: &[PassDesc]) { - let mut rid_passcount = vec![0; self.num_resources]; - - for pass in passes.iter() { - for rid in pass - .reads - .iter() - .chain(pass.writes.iter()) - .map(|id| id.0) - .collect::>() - { - rid_passcount[rid.0 as usize] += 1; - } - } - - tracing::debug!("per-resource pass-count: {rid_passcount:?}"); - - let mut total = 0; - for num_passes in rid_passcount { - self.ref_ranges.push((total, total + num_passes, 0)); - self.ref_accesses - .extend((0..num_passes).map(|_| (Access::empty(), Access::empty()))); - self.ref_access_passid - .extend((0..num_passes).map(|_| PackedPassNode(0))); - total += num_passes; - } - - tracing::debug!( - "ref_ranges and ref_accesses:\n{:?}\n{:?}\n{:?}", - self.ref_ranges, - self.ref_accesses, - self.ref_access_passid - ); - // for resourcedesc in resources: ref first pass - } - - fn get_accesses_for_rid_pass_mut( - &mut self, - rid: GraphResourceId, - pass: PackedPassNode, - ) -> &mut (Access, Access) { - let (start, _, i) = self.ref_ranges[rid.0 as usize]; - - let idx = self.ref_access_passid[start as usize..(start + i) as usize] - .binary_search(&pass) - .unwrap_or_else(|_| { - // increase counter - self.ref_ranges[rid.0 as usize].2 += 1; - i as usize - }) - + start as usize; - - self.ref_access_passid[idx] = pass; - - &mut self.ref_accesses[idx] - } - - fn get_reads_for_rid_pass_mut( - &mut self, - rid: GraphResourceId, - pass: PackedPassNode, - ) -> &mut Access { - &mut self.get_accesses_for_rid_pass_mut(rid, pass).0 - } - fn get_writes_for_rid_pass_mut( - &mut self, - rid: GraphResourceId, - pass: PackedPassNode, - ) -> &mut Access { - &mut self.get_accesses_for_rid_pass_mut(rid, pass).1 - } - - fn get_accesses_for_rid_pass( - &self, - rid: GraphResourceId, - pass: PackedPassNode, - ) -> Option<(Access, Access)> { - let (start, _, i) = self.ref_ranges[rid.0 as usize]; - - let idx = self.ref_access_passid[start as usize..(start + i) as usize] - .binary_search(&pass) - .ok()? - + start as usize; - - Some(self.ref_accesses[idx]) - } - - fn get_reads_for_rid_pass( - &self, - rid: GraphResourceId, - pass: PackedPassNode, - ) -> Option { - Some(self.get_accesses_for_rid_pass(rid, pass)?.0) - } - fn get_writes_for_rid_pass( - &self, - rid: GraphResourceId, - pass: PackedPassNode, - ) -> Option { - Some(self.get_accesses_for_rid_pass(rid, pass)?.1) - } - - fn reference_rid_pass(&mut self, rid: GraphResourceId, pass: PackedPassNode) { - let bit_idx = rid.0 as usize * (self.num_passes + 2) + pass.0 as usize; - let word_idx = bit_idx / 64; - let word_offset = bit_idx % 64; - tracing::debug!( - bit_idx, - word_idx, - word_offset, - "pass: {pass:?} references rid: {rid:?} " - ); - self.references[word_idx] |= 1 << word_offset; - } - - pub fn ref_passes(&mut self, passes: &[PassDesc]) { - for (i, pass) in passes.iter().enumerate() { - let packed_pass = PassNode::pass(i).into(); - - for &(rid, access) in &pass.reads { - let read = self.get_reads_for_rid_pass_mut(rid, packed_pass); - *read = *read | access; - - // TODO: check for first pass as well - self.reference_rid_pass(rid, PassNode::pass(i).into()); - } - - for &(rid, access) in &pass.writes { - let write = self.get_writes_for_rid_pass_mut(rid, packed_pass); - *write = *write | access; - - // TODO: check for first pass as well - self.reference_rid_pass(rid, PassNode::pass(i).into()); - } - } - } - - pub fn ref_inputs(&mut self, resources: &[GraphResource]) { - for (i, resource) in resources.iter().enumerate() { - match resource { - GraphResource::ImageDesc(_) | GraphResource::BufferDesc(_) => { - self.reference_rid_pass(GraphResourceId(i as u32), PassNode::First.into()); - } - _ => {} - } - } - } - - pub fn ref_outputs(&mut self, outputs: &[GraphResourceId]) { - for &rid in outputs { - self.reference_rid_pass(rid, PassNode::Last.into()); - } - } - - pub fn build_dag( - &self, - ) -> petgraph::stable_graph::StableDiGraph - { - struct Edge { - from: PackedPassNode, - to: PackedPassNode, - rid: GraphResourceId, - barrier: Barrier, - } - - #[derive(Debug, Clone, Copy)] - enum Ref { - Write(PackedPassNode, Access), - Read(PackedPassNode, Access), - } - - impl Ref { - fn node(&self) -> PackedPassNode { - match self { - Ref::Write(node, _) | Ref::Read(node, _) => *node, - } - } - fn access(&self) -> Access { - match self { - Ref::Write(_, access) | Ref::Read(_, access) => *access, - } - } - } - - let mut edges = Vec::::new(); - - let bits = crate::util::BitIter::new( - &self.references, - self.num_resources * (self.num_passes + 2), - ) - .chunks(self.num_passes + 2); - - tracing::debug!("building edges:"); - tracing::debug!("chunks: {bits:#?}"); - for (i, bits) in bits.enumerate() { - let rid = GraphResourceId(i as u32); - tracing::debug!("rid: {rid:?}"); - tracing::debug!("passes: {bits}"); - - let mut to_make_available = AccessMask::empty(); - let mut made_available = AccessMask::empty(); - - let mut last_ref = Option::::None; - let mut last_read = Option::::None; - let mut last_write = Option::::None; - - for pass in bits { - let packed_pass = PackedPassNode(pass as u16); - tracing::debug!("pass: {:?}", packed_pass.unpack()); - - let read = self.get_reads_for_rid_pass(rid, packed_pass); - if let Some(read) = read { - tracing::debug!("read: {:?}", read); - let make_visible = read.into_access_mask() & !made_available; - if let Some(last_write) = last_write.as_ref() { - let from = last_write.node(); - let to = packed_pass; - let from_write = last_write.access(); - - // if last_write is some, make visible the writes - if !make_visible.is_empty() { - made_available = made_available | make_visible; - - edges.push(Edge { - from, - to, - rid, - barrier: Barrier::MakeVisible { - src: from_write.stage, - dst: (make_visible.stage, make_visible.mask), - }, - }); - } - - // make available any changes - if !to_make_available.is_empty() { - edges.push(Edge { - from, - to, - rid, - barrier: Barrier::MakeAvailable { - src: (to_make_available.stage, to_make_available.mask), - dst: read.stage, - }, - }); - to_make_available = AccessMask::empty(); - } - - if make_visible.is_empty() && !to_make_available.is_empty() { - // still require a-after-b - edges.push(Edge { - from, - to, - rid, - barrier: Barrier::Execution { - src: from_write.stage, - dst: read.stage, - }, - }); - } - } - - // layout transition from previous pass, either read or write - if let Some(last_ref) = last_ref.as_ref() { - if last_ref.access().layout != read.layout { - let from = last_ref.node(); - let to = packed_pass; - edges.push(Edge { - from, - to, - rid, - barrier: Barrier::LayoutTransition { - src: ( - last_ref.access().stage, - last_ref.access().layout.unwrap(), - ), - dst: (read.stage, read.layout.unwrap()), - }, - }); - } - } - } - - let write = self.get_writes_for_rid_pass(rid, packed_pass); - if let Some(write) = write { - tracing::debug!("write: {:?}", write); - match last_ref.as_ref() { - Some(Ref::Read(node, before)) => { - // execution barrier to ward against write-after-read - - edges.push(Edge { - from: *node, - to: packed_pass, - rid, - barrier: Barrier::Execution { - src: before.stage, - dst: write.stage, - }, - }); - } - Some(Ref::Write(node, before)) => { - // check for layout transition here - if before.layout != write.layout { - edges.push(Edge { - from: *node, - to: packed_pass, - rid, - barrier: Barrier::LayoutTransition { - src: (before.stage, before.layout.unwrap()), - dst: (write.stage, write.layout.unwrap()), - }, - }); - } - } - _ => {} - } - } - - if let Some(read) = read { - last_read = Some(Ref::Read(packed_pass, read)); - last_ref = last_read; - } - if let Some(write) = write { - last_write = Some(Ref::Write(packed_pass, write)); - last_ref = last_write; - } - } - } - - let mut dag = petgraph::stable_graph::StableDiGraph::< - PackedPassNode, - (GraphResourceId, Barrier), - >::new(); - - let root = dag.add_node(PassNode::First.into()); - let output = dag.add_node(PassNode::Last.into()); - - _ = (root, output); - - for i in 0..self.num_passes { - dag.add_node(PassNode::pass(i).into()); - } - - // insert edges - for edge in edges { - let Edge { - from, - to, - rid, - barrier, - } = edge; - dag.add_edge( - from.unpack().dag_index().into(), - to.unpack().dag_index().into(), - (rid, barrier), - ); - } - - #[cfg(any(debug_assertions, test))] - std::fs::write( - "render_graph2.dot", - &format!( - "{:?}", - petgraph::dot::Dot::with_attr_getters( - &dag, - &[], - &|_graph, edgeref| { - format!( - "label = \"{},{:#?}\"", - edgeref.weight().0, - edgeref.weight().1, - ) - }, - &|_graph, noderef| { - format!( - "label = \"Pass({:?})\"", - petgraph::visit::NodeRef::weight(&noderef) - ) - } - ) - ), - ) - .expect("writing render_graph repr"); - - // prune dead ends - let mut sinks = dag - .externals(petgraph::Direction::Outgoing) - .filter(|idx| dag.node_weight(*idx) != Some(&PassNode::Last.into())) - .collect::>(); - - while let Some(sink) = sinks.pop() { - let mut neighbors = dag - .neighbors_directed(sink, petgraph::Direction::Incoming) - .detach(); - while let Some((edge, node)) = neighbors.next(&dag) { - dag.remove_edge(edge); - - if dag - .neighbors_directed(node, petgraph::Direction::Outgoing) - .count() - == 0 - { - sinks.push(node); - } - } - dag.remove_node(sink); - } - - dag - } - - pub fn toposort_dag( - &self, - mut dag: petgraph::stable_graph::StableDiGraph< - PackedPassNode, - (GraphResourceId, Barrier), - >, - ) -> Vec<( - Vec, - BTreeMap, - )> { - let mut topomap = Vec::new(); - - let mut sinks = dag - .externals(petgraph::Direction::Incoming) - .collect::>(); - let mut next_sinks = vec![]; - - loop { - if sinks.is_empty() { - break; - } - - let mut passes = Vec::with_capacity(self.num_passes); - let mut barriers = BTreeMap::new(); - for &sink in &sinks { - for &(rid, barrier) in dag - .edges_directed(sink, petgraph::Direction::Incoming) - .map(|edge| edge.weight()) - { - let before_and_after = match barrier { - Barrier::Logical => None, - Barrier::Execution { src, dst } => Some(( - Access { - stage: src, - ..Access::empty() - }, - Access { - stage: dst, - ..Access::empty() - }, - )), - Barrier::LayoutTransition { - src: (src, from), - dst: (dst, to), - } => Some(( - Access { - stage: src, - layout: Some(from), - ..Access::empty() - }, - Access { - stage: dst, - layout: Some(to), - ..Access::empty() - }, - )), - Barrier::MakeAvailable { - src: (stage, mask), - dst, - } => Some(( - Access { - stage, - mask, - ..Access::empty() - }, - Access { - stage: dst, - ..Access::empty() - }, - )), - Barrier::MakeVisible { - src, - dst: (stage, mask), - } => Some(( - Access { - stage: src, - ..Access::empty() - }, - Access { - stage, - mask, - ..Access::empty() - }, - )), - Barrier::MemoryBarrier { - src: (src_stage, src_mask), - dst: (dst_stage, dst_mask), - } => Some(( - Access { - stage: src_stage, - mask: src_mask, - ..Access::empty() - }, - Access { - stage: dst_stage, - mask: dst_mask, - ..Access::empty() - }, - )), - }; - - if let Some((before, after)) = before_and_after { - // initial access is transitioned at the beginning - // this affects imported resources only. - barriers - .entry(rid) - .and_modify(|(from, to)| { - *from = *from | before; - *to = *to | after; - }) - .or_insert((before, after)); - } - } - - let mut neighbors = dag - .neighbors_directed(sink, petgraph::Direction::Incoming) - .detach(); - while let Some((edge, node)) = neighbors.next(&dag) { - dag.remove_edge(edge); - - if dag - .neighbors_directed(node, petgraph::Direction::Outgoing) - .count() - == 0 - { - next_sinks.push(node); - } - } - - passes.push(*dag.node_weight(sink).unwrap()); - dag.remove_node(sink); - } - - topomap.push((passes, barriers)); - core::mem::swap(&mut sinks, &mut next_sinks); - } - - topomap - } - } -} - #[derive(Debug, Clone)] -struct BitIter<'a> { +pub struct BitIter<'a> { bits: &'a [u64], num_bits: usize, bit_offset: usize, @@ -1146,7 +447,7 @@ impl<'a> std::fmt::Display for BitIter<'a> { } impl<'a> BitIter<'a> { - fn new(bits: &'a [u64], num_bits: usize) -> Self { + pub fn new(bits: &'a [u64], num_bits: usize) -> Self { Self { bits, num_bits, @@ -1155,7 +456,7 @@ impl<'a> BitIter<'a> { } } - fn chunks(self, chunk_size: usize) -> ChunkedBitIter<'a> { + pub fn chunks(self, chunk_size: usize) -> ChunkedBitIter<'a> { ChunkedBitIter { inner: self, chunk_size, @@ -1185,7 +486,7 @@ impl Iterator for BitIter<'_> { } #[derive(Debug)] -struct ChunkedBitIter<'a> { +pub struct ChunkedBitIter<'a> { inner: BitIter<'a>, chunk_size: usize, pos: usize,