renderer: thread-local commandpools

Cargo.toml

@@ -22,12 +22,14 @@ tracing-subscriber = {version ="0.3.18", features = ["env-filter"]}
 glam = {version = "0.29.0", features = ["bytemuck"]}
 rand = "0.8.5"
 bitflags = "2.6"
+thread_local = "1.1.8"
 
 ash = "0.38.0"
 ash-window = "0.13.0"
 vk-mem = "0.4.0"
 vk-sync = "0.1.6"
 
+arrayvec = "0.7.6"
 tinyvec = "1.8"
 indexmap = "2"
 petgraph = "0.7"

crates/renderer/Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 tinyvec = { workspace = true }
+arrayvec = { workspace = true }
 indexmap = { workspace = true }
 petgraph = { workspace = true }
 itertools = { workspace = true }
@@ -13,6 +14,7 @@ rand = { workspace = true }
 parking_lot = { workspace = true }
 glam = { workspace = true }
 bitflags = { workspace = true }
+thread_local = {workspace = true}
 
 thiserror = { workspace = true }
 anyhow = { workspace = true }

crates/renderer/src/commands.rs

@@ -70,19 +70,19 @@ impl SingleUseCommandPool {
     }
 }
 
-pub trait CommandBuffer: DeviceOwned<vk::CommandBuffer> {
+pub trait HasQueue: DeviceOwned<vk::CommandBuffer> {
     fn queue(&self) -> &Queue;
 }
 
-impl CommandBuffer for SingleUseCommand {
+impl HasQueue for SingleUseCommand {
     fn queue(&self) -> &Queue {
         &self.pool.queue
     }
 }
 
-pub struct CommandList<T: CommandBuffer>(pub Vec<T>);
+pub struct CommandList<T: HasQueue>(pub Vec<T>);
 
-impl<T: CommandBuffer> CommandList<T> {
+impl<T: HasQueue> CommandList<T> {
     /// all commands in list must be allocated from the same queue.
     pub fn submit<'a>(
         &'a self,
@@ -215,6 +215,11 @@ impl SingleUseCommand {
         self.state.state()
     }
 
+    /// Safety: commandbuffer must not be accessed from multiple threads at the same time
+    pub unsafe fn buffer(&self) -> vk::CommandBuffer {
+        self.handle()
+    }
+
     pub fn end(&self) -> VkResult<()> {
         assert_eq!(self.state(), CommandBufferState::Recording);
         unsafe {
@@ -224,14 +229,115 @@ impl SingleUseCommand {
         Ok(())
     }
 
-    /// Safety: commandbuffer must not be accessed from multiple threads at the same time
+    pub fn submit_fence(
-    pub unsafe fn buffer(&self) -> vk::CommandBuffer {
+        &self,
-        self.handle()
+        wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
+        signal: Option<vk::Semaphore>,
+        fence: Option<vk::Fence>,
+    ) -> VkResult<()> {
+        assert_eq!(self.state(), CommandBufferState::Recording);
+        unsafe { self.device().dev().end_command_buffer(self.handle())? };
+
+        let buffers = [self.handle()];
+        let mut submit_info = vk::SubmitInfo::default().command_buffers(&buffers);
+
+        if let Some(semaphore) = signal.as_ref() {
+            // SAFETY: T and [T;1] have the same layout
+            submit_info = submit_info.signal_semaphores(unsafe {
+                core::mem::transmute::<&vk::Semaphore, &[vk::Semaphore; 1]>(semaphore)
+            });
+        }
+        if let Some((semaphore, stage)) = wait.as_ref() {
+            submit_info = submit_info
+                .wait_semaphores(core::slice::from_ref(semaphore))
+                .wait_dst_stage_mask(core::slice::from_ref(stage));
         }
 
-    //pub fn copy_buffer_to_image(&self, image: &Image2D, buffer: &Buffer, )
+        let fence = fence.unwrap_or(vk::Fence::null());
+        self.pool.queue().with_locked(|queue| unsafe {
+            self.device()
+                .dev()
+                .queue_submit(queue, &[submit_info], fence)
+        })?;
 
-    pub fn image_barrier(
+        self.state.set_pending();
+
+        tracing::trace!(
+            "submitted queue {:?} and fence {:?}",
+            self.pool.queue(),
+            fence
+        );
+
+        Ok(())
+    }
+
+    pub fn submit_async<'a>(
+        &'a self,
+        wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
+        signal: Option<vk::Semaphore>,
+        fence: Arc<sync::Fence>,
+    ) -> VkResult<FenceFuture<'a>> {
+        self.submit_fence(wait, signal, Some(fence.fence()))?;
+
+        Ok(FenceFuture::new(fence))
+    }
+
+    #[allow(dead_code)]
+    pub fn submit_blocking(
+        self,
+        wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
+        signal: Option<vk::Semaphore>,
+    ) -> VkResult<()> {
+        let fence = Arc::new(sync::Fence::create(self.device().clone())?);
+        let future = self.submit_async(wait, signal, fence)?;
+        future.block()?;
+        Ok(())
+    }
+}
+
+impl traits::CommandBufferExt for SingleUseCommand {}
+
+pub mod traits {
+    use super::*;
+
+    pub trait CommandBufferExt: DeviceOwned<vk::CommandBuffer> {
+        fn submit_to_queue(
+            self,
+            queue: &Queue,
+            wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
+            signal: Option<vk::Semaphore>,
+            fence: Arc<sync::Fence>,
+        ) -> VkResult<impl std::future::Future<Output = ()>>
+        where
+            Self: Sized,
+        {
+            //assert_eq!(self.state(), CommandBufferState::Recording);
+            unsafe { self.device().dev().end_command_buffer(self.handle())? };
+
+            let buffers = [self.handle()];
+            let mut submit_info = vk::SubmitInfo::default().command_buffers(&buffers);
+
+            if let Some(semaphore) = signal.as_ref() {
+                submit_info = submit_info.signal_semaphores(core::slice::from_ref(semaphore));
+            }
+            if let Some((semaphore, stage)) = wait.as_ref() {
+                submit_info = submit_info
+                    .wait_semaphores(core::slice::from_ref(semaphore))
+                    .wait_dst_stage_mask(core::slice::from_ref(stage));
+            }
+
+            queue.with_locked(|queue| unsafe {
+                self.device()
+                    .dev()
+                    .queue_submit(queue, &[submit_info], fence.fence())
+            })?;
+
+            tracing::trace!("submitted queue {:?} and fence {:?}", queue, fence);
+
+            Ok(crate::sync::FenceFuture::new(fence))
+        }
+
+        fn image_barrier(
             &self,
             image: vk::Image,
             aspects: vk::ImageAspectFlags,
@@ -243,7 +349,8 @@ impl SingleUseCommand {
             new_layout: vk::ImageLayout,
             queue_ownership_op: Option<QueueOwnership>,
         ) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
+
             let (src_family, dst_family) = queue_ownership_op
                 .map(|t| (t.src, t.dst))
                 .unwrap_or((vk::QUEUE_FAMILY_IGNORED, vk::QUEUE_FAMILY_IGNORED));
@@ -277,17 +384,17 @@ impl SingleUseCommand {
             }
         }
 
-    pub fn blit_images(
+        fn blit_images(
             &self,
             src: &Image,
             src_region: util::Rect2D,
             dst: &Image,
             dst_region: util::Rect2D,
         ) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device().dev().cmd_blit_image(
-                self.buffer(),
+                    self.handle(),
                     src.image(),
                     vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
                     dst.image(),
@@ -310,14 +417,14 @@ impl SingleUseCommand {
             }
         }
 
-    pub fn copy_buffer_to_image(
+        fn copy_buffer_to_image(
             &self,
             buffer: vk::Buffer,
             image: vk::Image,
             layout: vk::ImageLayout,
             regions: &[vk::BufferImageCopy],
         ) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device().dev().cmd_copy_buffer_to_image(
                     self.handle(),
@@ -329,16 +436,16 @@ impl SingleUseCommand {
             }
         }
 
-    pub fn copy_buffers(&self, src: vk::Buffer, dst: vk::Buffer, regions: &[vk::BufferCopy]) {
+        fn copy_buffers(&self, src: vk::Buffer, dst: vk::Buffer, regions: &[vk::BufferCopy]) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device()
                     .dev()
                     .cmd_copy_buffer(self.handle(), src, dst, regions);
             }
         }
-    #[allow(dead_code)]
+
-    pub fn copy_images(
+        fn copy_images(
             &self,
             src: vk::Image,
             src_layout: vk::ImageLayout,
@@ -346,7 +453,7 @@ impl SingleUseCommand {
             dst_layout: vk::ImageLayout,
             regions: &[vk::ImageCopy],
         ) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device().dev().cmd_copy_image(
                     self.handle(),
@@ -359,7 +466,7 @@ impl SingleUseCommand {
             }
         }
 
-    pub fn clear_color_image(
+        fn clear_color_image(
             &self,
             image: vk::Image,
             format: vk::Format,
@@ -367,7 +474,7 @@ impl SingleUseCommand {
             color: crate::Rgba,
             subresources: &[vk::ImageSubresourceRange],
         ) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             let clear_colors = match format.get_component_kind() {
                 crate::util::FormatComponentKind::Float => vk::ClearColorValue {
                     float32: color.into_f32(),
@@ -391,39 +498,39 @@ impl SingleUseCommand {
             }
         }
 
-    pub fn begin_rendering(&self, rendering_info: vk::RenderingInfo<'_>) {
+        fn begin_rendering(&self, rendering_info: vk::RenderingInfo<'_>) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device()
                     .dev()
-                .cmd_begin_rendering(self.buffer(), &rendering_info);
+                    .cmd_begin_rendering(self.handle(), &rendering_info);
             }
         }
 
-    pub fn set_viewport(&self, viewports: &[vk::Viewport]) {
+        fn set_viewport(&self, viewports: &[vk::Viewport]) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device()
                     .dev()
-                .cmd_set_viewport(self.buffer(), 0, viewports);
+                    .cmd_set_viewport(self.handle(), 0, viewports);
             }
         }
-    pub fn set_scissors(&self, scissors: &[vk::Rect2D]) {
+        fn set_scissors(&self, scissors: &[vk::Rect2D]) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device()
                     .dev()
-                .cmd_set_scissor(self.buffer(), 0, scissors);
+                    .cmd_set_scissor(self.handle(), 0, scissors);
             }
         }
-    pub fn push_constants(
+        fn push_constants(
             &self,
             layout: &PipelineLayout,
             stage: vk::ShaderStageFlags,
             offset: u32,
             bytes: &[u8],
         ) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device().dev().cmd_push_constants(
                     self.handle(),
@@ -434,44 +541,44 @@ impl SingleUseCommand {
                 );
             }
         }
-    pub fn bind_pipeline(&self, pipeline: &Pipeline) {
+        fn bind_pipeline(&self, pipeline: &Pipeline) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device().dev().cmd_bind_pipeline(
-                self.buffer(),
+                    self.handle(),
                     pipeline.bind_point(),
                     pipeline.handle(),
                 );
             }
         }
-    pub fn bind_vertex_buffers(&self, buffers: &[vk::Buffer], offsets: &[u64]) {
+        fn bind_vertex_buffers(&self, buffers: &[vk::Buffer], offsets: &[u64]) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device()
                     .dev()
-                .cmd_bind_vertex_buffers(self.buffer(), 0, buffers, offsets);
+                    .cmd_bind_vertex_buffers(self.handle(), 0, buffers, offsets);
             }
         }
-    pub fn bind_indices(&self, buffer: vk::Buffer, offset: u64, kind: vk::IndexType) {
+        fn bind_indices(&self, buffer: vk::Buffer, offset: u64, kind: vk::IndexType) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device()
                     .dev()
-                .cmd_bind_index_buffer(self.buffer(), buffer, offset, kind);
+                    .cmd_bind_index_buffer(self.handle(), buffer, offset, kind);
             }
         }
 
-    pub fn bind_descriptor_sets(
+        fn bind_descriptor_sets(
             &self,
             layout: &PipelineLayout,
             bind_point: vk::PipelineBindPoint,
             descriptor_sets: &[vk::DescriptorSet],
         ) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             use crate::device::DeviceOwned;
             unsafe {
                 self.device().dev().cmd_bind_descriptor_sets(
-                self.buffer(),
+                    self.handle(),
                     bind_point,
                     layout.handle(),
                     0,
@@ -481,8 +588,7 @@ impl SingleUseCommand {
             }
         }
 
-    #[allow(dead_code)]
+        fn draw_indexed(
-    pub fn draw_indexed(
             &self,
             indices: u32,
             instances: u32,
@@ -490,10 +596,10 @@ impl SingleUseCommand {
             vertex_offset: i32,
             instance_offset: u32,
         ) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device().dev().cmd_draw_indexed(
-                self.buffer(),
+                    self.handle(),
                     indices,
                     instances,
                     index_offset,
@@ -503,11 +609,11 @@ impl SingleUseCommand {
             }
         }
 
-    pub fn draw_indexed_indirect(&self, buffer: vk::Buffer, offset: u64, count: u32, stride: u32) {
+        fn draw_indexed_indirect(&self, buffer: vk::Buffer, offset: u64, count: u32, stride: u32) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
                 self.device().dev().cmd_draw_indexed_indirect(
-                self.buffer(),
+                    self.handle(),
                     buffer,
                     offset,
                     count,
@@ -516,30 +622,285 @@ impl SingleUseCommand {
             }
         }
 
-    pub fn end_rendering(&self) {
+        fn end_rendering(&self) {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            // assert_eq!(self.state(), CommandBufferState::Recording);
             unsafe {
-            self.device().dev().cmd_end_rendering(self.buffer());
+                self.device().dev().cmd_end_rendering(self.handle());
+            }
+        }
+    }
+}
+
+pub use command_pools::{CommandBuffer, CommandBufferFuture, CommandPool, CommandPools};
+
+mod command_pools {
+    use std::{borrow::Cow, cell::Cell, mem::ManuallyDrop, sync::Arc};
+
+    use arrayvec::ArrayVec;
+    use ash::{prelude::VkResult, vk};
+    use parking_lot::Mutex;
+    use smol::future::FutureExt;
+    use thread_local::ThreadLocal;
+
+    use crate::{
+        define_device_owned_handle,
+        device::{Device, DeviceOwned},
+        sync,
+        util::MutexExt,
+        Queue,
+    };
+
+    #[derive(Debug, thiserror::Error)]
+    pub enum Error {
+        #[error(transparent)]
+        Vulkan(#[from] vk::Result),
+        #[error("CommandBuffer allocation failed: wrong number of command buffers.")]
+        NoCommandBuffersAllocated,
+        #[error("Attempted to submit commandbuffer to incompatible queue.")]
+        InvalidQueueSubmission,
+    }
+
+    define_device_owned_handle! {
+        #[derive()]
+        pub CommandPool(vk::CommandPool) {
+            family: u32,
+            free_list: Mutex<Cell<(Vec<vk::CommandBufferLevel>, Vec<vk::CommandBuffer>)>>,
+        } => |this|  unsafe {
+            this.device().dev().destroy_command_pool(this.handle(), None);
         }
     }
 
-    pub fn submit_fence(
+    impl !Sync for CommandPool {}
+
+    impl CommandPool {
+        pub fn new(
+            dev: Device,
+            family: u32,
+            flags: vk::CommandPoolCreateFlags,
+            name: Option<Cow<'static, str>>,
+        ) -> Result<CommandPool, vk::Result> {
+            let pool_info = vk::CommandPoolCreateInfo::default()
+                .queue_family_index(family)
+                .flags(flags);
+
+            let pool = unsafe { dev.dev().create_command_pool(&pool_info, None)? };
+
+            Self::construct(dev, pool, name, family, Mutex::new(Default::default()))
+        }
+
+        #[allow(dead_code)]
+        pub unsafe fn reset(&self) -> VkResult<()> {
+            unsafe {
+                self.device()
+                    .dev()
+                    .reset_command_pool(self.handle(), vk::CommandPoolResetFlags::empty())
+            }
+        }
+
+        pub fn free_command_buffers(&self) {
+            self.free_list_mut(|levels, buffers| {
+                unsafe {
+                    self.device()
+                        .dev()
+                        .free_command_buffers(self.handle(), &buffers);
+                }
+                levels.clear();
+                buffers.clear();
+            });
+        }
+
+        pub fn free_command_buffer(
+            self: &Arc<Self>,
+            level: vk::CommandBufferLevel,
+            buffer: vk::CommandBuffer,
+        ) {
+            self.free_list_mut(|levels, buffers| {
+                levels.push(level);
+                buffers.push(buffer);
+            });
+        }
+
+        fn free_list_mut<
+            T,
+            F: FnOnce(&mut Vec<vk::CommandBufferLevel>, &mut Vec<vk::CommandBuffer>) -> T,
+        >(
             &self,
+            cb: F,
+        ) -> T {
+            self.free_list.with_locked(|cell| {
+                let (mut levels, mut buffers) = cell.take();
+                let t = cb(&mut levels, &mut buffers);
+                cell.set((levels, buffers));
+                t
+            })
+        }
+
+        pub fn alloc(
+            self: &Arc<Self>,
+            name: Option<Cow<'static, str>>,
+            level: vk::CommandBufferLevel,
+            flags: vk::CommandBufferUsageFlags,
+        ) -> Result<CommandBuffer, Error> {
+            // TODO: is this really the right place?
+            self.free_command_buffers();
+
+            let handle = {
+                let info = vk::CommandBufferAllocateInfo::default()
+                    .command_pool(self.handle())
+                    .command_buffer_count(1)
+                    .level(level);
+
+                unsafe { self.device().dev().allocate_command_buffers(&info) }?
+                    .pop()
+                    .ok_or(Error::NoCommandBuffersAllocated)?
+            };
+
+            let begin_info = vk::CommandBufferBeginInfo::default().flags(flags);
+            unsafe {
+                self.device()
+                    .dev()
+                    .begin_command_buffer(handle, &begin_info)?;
+            }
+
+            Ok(CommandBuffer::construct(
+                self.device().clone(),
+                handle,
+                name,
+                self.clone(),
+                level,
+            )?)
+        }
+
+        pub fn alloc_primary(
+            self: &Arc<Self>,
+            name: Option<Cow<'static, str>>,
+            flags: vk::CommandBufferUsageFlags,
+        ) -> Result<CommandBuffer, Error> {
+            self.alloc(name, vk::CommandBufferLevel::PRIMARY, flags)
+        }
+    }
+
+    impl core::fmt::Debug for CommandPool {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            f.debug_struct("CommandPool")
+                .field("inner", &self.inner)
+                .field("family", &self.family)
+                .finish_non_exhaustive()
+        }
+    }
+
+    impl CommandPoolsShard {
+        fn new() -> Self {
+            Self {
+                per_family_pools: ArrayVec::new(),
+            }
+        }
+
+        fn find_or_default(&mut self, dev: &Device, family: u32) -> Arc<CommandPool> {
+            match self
+                .per_family_pools
+                .binary_search_by_key(&family, |entry| entry.0)
+            {
+                Ok(i) => self.per_family_pools[i].1.clone(),
+                Err(i) => {
+                    let pool = Arc::new(
+                        CommandPool::new(
+                            dev.clone(),
+                            family,
+                            vk::CommandPoolCreateFlags::empty(),
+                            Some(
+                                format!("commandpool-{:?}-{}", std::thread::current().id(), family)
+                                    .into(),
+                            ),
+                        )
+                        .unwrap(),
+                    );
+                    self.per_family_pools
+                        .try_insert(i, (family, pool.clone()))
+                        .expect("too many command pools in thread local");
+
+                    pool
+                }
+            }
+        }
+    }
+
+    // SAFETY: this isn't safe lmao. assumes pools are unused when dropped
+    unsafe impl Send for CommandPoolsShard {}
+
+    #[derive(Debug)]
+    struct CommandPoolsShard {
+        per_family_pools: ArrayVec<(u32, Arc<CommandPool>), 8>,
+    }
+
+    pub struct CommandPools {
+        device: Device,
+        shards: ThreadLocal<crossbeam::utils::CachePadded<core::cell::Cell<CommandPoolsShard>>>,
+    }
+
+    impl CommandPools {
+        pub fn get_for_family(&self, family: u32) -> Arc<CommandPool> {
+            let shard = self
+                .shards
+                .get_or(|| core::cell::Cell::new(CommandPoolsShard::new()).into());
+
+            let mut inner = shard.replace(CommandPoolsShard::new());
+            let pool = inner.find_or_default(&self.device, family);
+            _ = shard.replace(inner);
+
+            pool
+        }
+
+        pub fn allocate_buffer(
+            &self,
+            family: u32,
+            name: Option<Cow<'static, str>>,
+            level: vk::CommandBufferLevel,
+            flags: vk::CommandBufferUsageFlags,
+        ) -> Result<CommandBuffer, Error> {
+            self.get_for_family(family).alloc(name, level, flags)
+        }
+    }
+
+    define_device_owned_handle! {
+        #[derive(Debug)]
+        pub CommandBuffer(vk::CommandBuffer) {
+            pool: Arc<CommandPool>,
+            #[allow(unused)]
+            level: vk::CommandBufferLevel,
+        } => |this|  unsafe {
+            this.device().dev().free_command_buffers(this.pool.handle(), &[this.handle()]);
+        }
+    }
+
+    impl !Sync for CommandBuffer {}
+    impl !Send for CommandBuffer {}
+
+    impl super::traits::CommandBufferExt for CommandBuffer {}
+
+    impl CommandBuffer {
+        pub fn submit(
+            self,
+            queue: &Queue,
             wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
             signal: Option<vk::Semaphore>,
-        fence: Option<vk::Fence>,
+            fence: Arc<sync::Fence>,
-    ) -> VkResult<()> {
+        ) -> Result<CommandBufferFuture, Error> {
-        assert_eq!(self.state(), CommandBufferState::Recording);
+            let this = ManuallyDrop::new(self);
-        unsafe { self.device().dev().end_command_buffer(self.handle())? };
 
-        let buffers = [self.handle()];
+            if queue.1 != this.pool.family {
+                tracing::error!("attempted to submit commandbuffer to incompatible queue.");
+                return Err(Error::InvalidQueueSubmission);
+            }
+            unsafe {
+                this.device().dev().end_command_buffer(this.handle())?;
+            }
+
+            let buffers = [this.handle()];
             let mut submit_info = vk::SubmitInfo::default().command_buffers(&buffers);
 
             if let Some(semaphore) = signal.as_ref() {
-            // SAFETY: T and [T;1] have the same layout
+                submit_info = submit_info.signal_semaphores(core::slice::from_ref(semaphore));
-            submit_info = submit_info.signal_semaphores(unsafe {
-                core::mem::transmute::<&vk::Semaphore, &[vk::Semaphore; 1]>(semaphore)
-            });
             }
             if let Some((semaphore, stage)) = wait.as_ref() {
                 submit_info = submit_info
@@ -547,41 +908,48 @@ impl SingleUseCommand {
                     .wait_dst_stage_mask(core::slice::from_ref(stage));
             }
 
-        let fence = fence.unwrap_or(vk::Fence::null());
+            queue.with_locked(|queue| unsafe {
-        self.pool.queue().with_locked(|queue| unsafe {
+                this.device()
-            self.device()
                     .dev()
-                .queue_submit(queue, &[submit_info], fence)
+                    .queue_submit(queue, &[submit_info], fence.fence())
             })?;
-        tracing::trace!(
-            "submitted queue {:?} and fence {:?}",
-            self.pool.queue(),
-            fence
-        );
 
-        Ok(())
+            Ok(CommandBufferFuture {
+                inner: sync::FenceFuture::new(fence),
+                pool: this.pool.clone(),
+                buffer: this.handle(),
+                level: this.level,
+            })
+        }
     }
 
-    pub fn submit_async<'a>(
+    pub struct CommandBufferFuture {
-        &'a self,
+        inner: sync::FenceFuture<'static>,
-        wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
+        pool: Arc<CommandPool>,
-        signal: Option<vk::Semaphore>,
+        buffer: vk::CommandBuffer,
-        fence: Arc<sync::Fence>,
+        level: vk::CommandBufferLevel,
-    ) -> VkResult<FenceFuture<'a>> {
-        self.submit_fence(wait, signal, Some(fence.fence()))?;
-
-        Ok(FenceFuture::new(fence))
     }
 
-    #[allow(dead_code)]
+    impl CommandBufferFuture {
-    pub fn submit_blocking(
+        pub fn block(&self) -> VkResult<()> {
-        self,
+            self.inner.block()
-        wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
+        }
-        signal: Option<vk::Semaphore>,
+    }
-    ) -> VkResult<()> {
+
-        let fence = Arc::new(sync::Fence::create(self.device().clone())?);
+    impl core::future::Future for CommandBufferFuture {
-        let future = self.submit_async(wait, signal, fence)?;
+        type Output = <sync::FenceFuture<'static> as core::future::Future>::Output;
-        future.block()?;
+
-        Ok(())
+        fn poll(
+            mut self: std::pin::Pin<&mut Self>,
+            cx: &mut std::task::Context<'_>,
+        ) -> std::task::Poll<Self::Output> {
+            self.inner.poll(cx)
+        }
+    }
+
+    impl Drop for CommandBufferFuture {
+        fn drop(&mut self) {
+            self.pool.free_command_buffer(self.level, self.buffer);
+        }
     }
 }

crates/renderer/src/device.rs

@@ -26,6 +26,7 @@ pub struct DeviceQueueFamilies {
     pub(crate) present: (u32, u32),
     pub(crate) async_compute: (u32, u32),
     pub(crate) transfer: (u32, u32),
+    pub(crate) properties: Box<[vk::QueueFamilyProperties]>,
 }
 
 impl DeviceQueueFamilies {
@@ -78,12 +79,12 @@ bitflags::bitflags! {
     #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
     pub struct QueueFlags: u32 {
         const GRAPHICS = 1 << 0;
-        const PRESENT = 1 << 1;
+        const ASYNC_COMPUTE = 1 << 1;
-        const ASYNC_COMPUTE = 1 << 2;
+        const TRANSFER = 1 << 2;
-        const TRANSFER = 1 << 3;
+        const PRESENT = 1 << 3;
 
         const NONE = 0;
-        const PRESENT_GRAPHICS = 1 << 0 | 1 << 1;
+        const PRESENT_GRAPHICS = 1 << 0 | 1 << 2;
     }
 }
 
@@ -256,7 +257,7 @@ impl DeviceBuilder {
         display_handle: Option<RawDisplayHandle>,
         pdev: vk::PhysicalDevice,
     ) -> DeviceQueueFamilies {
-        let queue_families = unsafe {
+        let queue_familiy_properties = unsafe {
             instance
                 .instance
                 .get_physical_device_queue_family_properties(pdev)
@@ -318,7 +319,7 @@ impl DeviceBuilder {
         }
 
         let mut queue_families = QueueFamilies(
-            queue_families
+            queue_familiy_properties
                 .iter()
                 .enumerate()
                 .map(|(i, family)| {
@@ -416,6 +417,7 @@ impl DeviceBuilder {
             async_compute,
             transfer,
             present,
+            properties: queue_familiy_properties.into_boxed_slice(),
         };
 
         queues
@@ -887,7 +889,7 @@ impl Device {
     pub fn physical_device(&self) -> &PhysicalDevice {
         &self.0.physical
     }
-    pub fn graphics_queue(&self) -> &Queue {
+    pub fn main_queue(&self) -> &Queue {
         &self.0.main_queue
     }
     pub fn transfer_queue(&self) -> &Queue {

crates/renderer/src/egui.rs

@@ -5,6 +5,7 @@ use indexmap::IndexMap;
 
 use crate::{
     buffers::{Buffer, BufferDesc},
+    commands::traits::CommandBufferExt,
     device::{self, DeviceOwned},
     images::{Image, ImageDesc, ImageViewDesc},
     render_graph::{

crates/renderer/src/render_graph.rs

@@ -8,7 +8,8 @@ use std::{
 
 use crate::{
     buffers::{Buffer, BufferDesc},
-    commands, def_monotonic_id,
+    commands::{self, traits::CommandBufferExt},
+    def_monotonic_id,
     device::{self, DeviceOwned},
     images::{self, Image, ImageDesc},
     util::{self, Rgba, WithLifetime},
@@ -489,7 +490,7 @@ impl RenderGraph {
         })?;
 
         let pool =
-            commands::SingleUseCommandPool::new(device.clone(), device.graphics_queue().clone())?;
+            commands::SingleUseCommandPool::new(device.clone(), device.main_queue().clone())?;
 
         let resources = &self.resources;
         let cmds = topo

crates/renderer/src/rendering/mod.rs

@@ -7,7 +7,7 @@ pub use crate::egui_pass::{egui_pass, egui_pre_pass};
 
 use crate::{
     buffers::{Buffer, BufferDesc},
-    commands,
+    commands::{self, traits::CommandBufferExt},
     device::{Device, DeviceOwned},
     images::ImageViewDesc,
     pipeline,
@@ -97,7 +97,7 @@ impl Wireframe {
             },
         )?;
 
-        let pool = commands::SingleUseCommandPool::new(dev.clone(), dev.graphics_queue().clone())?;
+        let pool = commands::SingleUseCommandPool::new(dev.clone(), dev.main_queue().clone())?;
 
         let cmd = pool.alloc()?;