use std::sync::{Arc, atomic::AtomicU8};

use crate::{
    define_device_owned_handle,
    device::DeviceOwned,
    images::{Image, QueueOwnership},
    pipeline::{Pipeline, PipelineLayout},
    sync::{self, FenceFuture},
    util::{self, MutexExt},
};

use super::{Device, queue::Queue};
use ash::{prelude::*, vk};
use ash_formats::FormatHelpers as _;
use parking_lot::Mutex;

#[derive(Debug)]
pub struct SingleUseCommandPool {
    device: Device,
    pool: Mutex<vk::CommandPool>,
    queue: Queue,
}

impl Drop for SingleUseCommandPool {
    fn drop(&mut self) {
        unsafe {
            self.pool.with_locked(|&pool| {
                self.device.dev().destroy_command_pool(pool, None);
            });
        }
    }
}

impl SingleUseCommandPool {
    pub fn new(device: Device, queue: Queue) -> VkResult<Arc<Self>> {
        let pool_info = vk::CommandPoolCreateInfo::default()
            .queue_family_index(queue.family_index())
            .flags(vk::CommandPoolCreateFlags::TRANSIENT);

        let pool = unsafe { device.dev().create_command_pool(&pool_info, None)? };

        Ok(Arc::new(Self {
            device,
            pool: Mutex::new(pool),
            queue,
        }))
    }

    #[allow(dead_code)]
    pub fn reset(&self) -> VkResult<()> {
        unsafe {
            self.pool.with_locked(|pool| {
                self.device
                    .dev()
                    .reset_command_pool(*pool, vk::CommandPoolResetFlags::empty())
            })
        }
    }
    pub fn alloc(self: &Arc<Self>) -> VkResult<SingleUseCommand> {
        SingleUseCommand::new(self.device.clone(), self.clone())
    }

    pub fn queue(&self) -> &Queue {
        &self.queue
    }

    // /// get the underlying pool, bypassing the mutex
    // #[allow(dead_code)]
    // pub unsafe fn pool(&self) -> vk::CommandPool {
    //     self.pool.data_ptr().read()
    // }
}

pub trait HasQueue: DeviceOwned<vk::CommandBuffer> {
    fn queue(&self) -> &Queue;
}

impl HasQueue for SingleUseCommand {
    fn queue(&self) -> &Queue {
        &self.pool.queue
    }
}

pub struct CommandList<T: HasQueue>(pub Vec<T>);

impl<T: HasQueue> CommandList<T> {
    /// all commands in list must be allocated from the same queue.
    pub fn submit<'a>(
        &'a self,
        wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
        signal: Option<vk::Semaphore>,
        fence: Arc<sync::Fence>,
    ) -> VkResult<FenceFuture<'a>> {
        if self.0.is_empty() {
            //exit
        }
        let buffers = self.0.iter().map(|cmd| cmd.handle()).collect::<Vec<_>>();

        let mut info = vk::SubmitInfo::default().command_buffers(&buffers);
        if let Some((sema, stage)) = wait.as_ref() {
            info = info
                .wait_dst_stage_mask(core::slice::from_ref(stage))
                .wait_semaphores(core::slice::from_ref(sema));
        }

        if let Some(signal) = signal.as_ref() {
            info = info.signal_semaphores(core::slice::from_ref(signal));
        }

        self.0[0].queue().with_locked(|queue| unsafe {
            self.0[0]
                .device()
                .dev()
                .queue_submit(queue.raw(), &[info], fence.raw())
        })?;

        Ok(FenceFuture::<'a>::new(fence))
    }
}

#[derive(Debug, PartialEq, Eq, Clone, Copy)]
#[repr(u8)]
pub enum CommandBufferState {
    Initial = 0,
    Recording,
    Executable,
    Pending,
    Invalid,
}

#[derive(Debug)]
#[repr(transparent)]
struct CommandBufferState2(AtomicU8);

#[allow(dead_code)]
impl CommandBufferState2 {
    fn initial() -> Self {
        Self(AtomicU8::new(CommandBufferState::Initial as u8))
    }
    fn recording() -> Self {
        Self(AtomicU8::new(CommandBufferState::Recording as u8))
    }

    fn state(&self) -> CommandBufferState {
        let value = self.0.load(std::sync::atomic::Ordering::Relaxed);
        unsafe { *<*const _>::from(&value).cast::<CommandBufferState>() }
    }

    fn is_initial(&self) -> bool {
        self.0.load(std::sync::atomic::Ordering::Relaxed) == 0
    }
    fn set_executable(&self) {
        self.0.store(
            CommandBufferState::Executable as u8,
            std::sync::atomic::Ordering::Relaxed,
        );
    }
    fn set_recording(&self) {
        self.0.store(
            CommandBufferState::Recording as u8,
            std::sync::atomic::Ordering::Relaxed,
        );
    }
    fn set_pending(&self) {
        self.0.store(
            CommandBufferState::Pending as u8,
            std::sync::atomic::Ordering::Relaxed,
        );
    }
    fn set_invalid(&self) {
        self.0.store(
            CommandBufferState::Invalid as u8,
            std::sync::atomic::Ordering::Relaxed,
        );
    }
}

define_device_owned_handle! {
    #[derive(Debug)]
    pub SingleUseCommand(vk::CommandBuffer) {
        pool: Arc<SingleUseCommandPool>,
        state: CommandBufferState2,
    } => |this|  unsafe {
            this.pool
                .pool
                .with_locked(|&pool| this.device().dev().free_command_buffers(pool, &[this.handle()]))
    }
}

impl !Sync for SingleUseCommand {}

impl SingleUseCommand {
    pub fn new(device: Device, pool: Arc<SingleUseCommandPool>) -> VkResult<Self> {
        let buffer = unsafe {
            let buffer = pool.pool.with_locked(|pool| {
                let alloc_info = vk::CommandBufferAllocateInfo::default()
                    .command_buffer_count(1)
                    .command_pool(*pool)
                    .level(vk::CommandBufferLevel::PRIMARY);
                Ok(device.dev().allocate_command_buffers(&alloc_info)?[0])
            })?;

            device.dev().begin_command_buffer(
                buffer,
                &vk::CommandBufferBeginInfo::default()
                    .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT),
            )?;

            buffer
        };

        Self::construct(device, buffer, None, pool, CommandBufferState2::recording())
    }

    pub fn state(&self) -> CommandBufferState {
        self.state.state()
    }

    /// Safety: commandbuffer must not be accessed from multiple threads at the same time
    pub unsafe fn buffer(&self) -> vk::CommandBuffer {
        self.handle()
    }

    pub fn end(&self) -> VkResult<()> {
        assert_eq!(self.state(), CommandBufferState::Recording);
        unsafe {
            self.inner.dev().dev().end_command_buffer(self.handle())?;
        }
        self.state.set_executable();
        Ok(())
    }

    pub fn submit_fence(
        &self,
        wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
        signal: Option<vk::Semaphore>,
        fence: Option<vk::Fence>,
    ) -> VkResult<()> {
        assert_eq!(self.state(), CommandBufferState::Recording);
        unsafe { self.device().dev().end_command_buffer(self.handle())? };

        let buffers = [self.handle()];
        let mut submit_info = vk::SubmitInfo::default().command_buffers(&buffers);

        if let Some(semaphore) = signal.as_ref() {
            // SAFETY: T and [T;1] have the same layout
            submit_info = submit_info.signal_semaphores(unsafe {
                core::mem::transmute::<&vk::Semaphore, &[vk::Semaphore; 1]>(semaphore)
            });
        }
        if let Some((semaphore, stage)) = wait.as_ref() {
            submit_info = submit_info
                .wait_semaphores(core::slice::from_ref(semaphore))
                .wait_dst_stage_mask(core::slice::from_ref(stage));
        }

        let fence = fence.unwrap_or(vk::Fence::null());
        self.pool.queue().with_locked(|queue| unsafe {
            self.device()
                .dev()
                .queue_submit(queue.raw(), &[submit_info], fence)
        })?;

        self.state.set_pending();

        tracing::trace!(
            "submitted queue {:?} and fence {:?}",
            self.pool.queue(),
            fence
        );

        Ok(())
    }

    pub fn submit_async<'a>(
        &'a self,
        wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
        signal: Option<vk::Semaphore>,
        fence: Arc<sync::Fence>,
    ) -> VkResult<FenceFuture<'a>> {
        self.submit_fence(wait, signal, Some(fence.raw()))?;

        Ok(FenceFuture::new(fence))
    }

    #[allow(dead_code)]
    pub fn submit_blocking(
        self,
        wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
        signal: Option<vk::Semaphore>,
    ) -> crate::Result<()> {
        let fence = Arc::new(sync::Fence::from_pool(&self.device().pools.fences, None)?);
        let future = self.submit_async(wait, signal, fence)?;
        future.block()?;
        Ok(())
    }
}

impl traits::CommandBufferExt for SingleUseCommand {}

pub mod traits {
    use super::*;

    pub trait CommandBufferExt: DeviceOwned<vk::CommandBuffer> {
        fn submit_to_queue(
            self,
            queue: &Queue,
            wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
            signal: Option<vk::Semaphore>,
            fence: Arc<sync::Fence>,
        ) -> VkResult<impl std::future::Future<Output = ()>>
        where
            Self: Sized,
        {
            //assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe { self.device().dev().end_command_buffer(self.handle())? };

            let buffers = [self.handle()];
            let mut submit_info = vk::SubmitInfo::default().command_buffers(&buffers);

            if let Some(semaphore) = signal.as_ref() {
                submit_info = submit_info.signal_semaphores(core::slice::from_ref(semaphore));
            }
            if let Some((semaphore, stage)) = wait.as_ref() {
                submit_info = submit_info
                    .wait_semaphores(core::slice::from_ref(semaphore))
                    .wait_dst_stage_mask(core::slice::from_ref(stage));
            }

            queue.with_locked(|queue| unsafe {
                self.device()
                    .dev()
                    .queue_submit(queue.raw(), &[submit_info], fence.raw())
            })?;

            tracing::trace!("submitted queue {:?} and fence {:?}", queue, fence);

            let future = crate::sync::FenceFuture::new(fence);
            Ok(async move {
                future.await;
                // Keep the command buffer allocated until the submitted work
                // has completed; it is only safe to return it to the pool then.
                drop(self);
            })
        }

        fn image_barrier(
            &self,
            image: vk::Image,
            aspects: vk::ImageAspectFlags,
            src_stage: vk::PipelineStageFlags2,
            src_access: vk::AccessFlags2,
            dst_stage: vk::PipelineStageFlags2,
            dst_access: vk::AccessFlags2,
            old_layout: vk::ImageLayout,
            new_layout: vk::ImageLayout,
            queue_ownership_op: Option<QueueOwnership>,
        ) {
            // assert_eq!(self.state(), CommandBufferState::Recording);

            let (src_family, dst_family) = queue_ownership_op
                .map(|t| (t.src, t.dst))
                .unwrap_or((vk::QUEUE_FAMILY_IGNORED, vk::QUEUE_FAMILY_IGNORED));

            let barrier = vk::ImageMemoryBarrier2::default()
                .image(image)
                .subresource_range(
                    vk::ImageSubresourceRange::default()
                        .aspect_mask(aspects)
                        .base_mip_level(0)
                        .base_array_layer(0)
                        .level_count(vk::REMAINING_MIP_LEVELS)
                        .layer_count(vk::REMAINING_ARRAY_LAYERS),
                )
                .src_stage_mask(src_stage)
                .src_access_mask(src_access)
                .dst_stage_mask(dst_stage)
                .dst_access_mask(dst_access)
                .dst_queue_family_index(dst_family)
                .src_queue_family_index(src_family)
                .old_layout(old_layout)
                .new_layout(new_layout);

            unsafe {
                self.device().dev().cmd_pipeline_barrier2(
                    self.handle(),
                    &vk::DependencyInfo::default()
                        .dependency_flags(vk::DependencyFlags::BY_REGION)
                        .image_memory_barriers(core::slice::from_ref(&barrier)),
                );
            }
        }

        fn blit_images(
            &self,
            src: &Image,
            src_region: util::Rect2D,
            dst: &Image,
            dst_region: util::Rect2D,
        ) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device().dev().cmd_blit_image(
                    self.handle(),
                    src.image(),
                    vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
                    dst.image(),
                    vk::ImageLayout::TRANSFER_DST_OPTIMAL,
                    &[vk::ImageBlit::default()
                        .src_subresource(
                            vk::ImageSubresourceLayers::default()
                                .aspect_mask(vk::ImageAspectFlags::COLOR)
                                .layer_count(1),
                        )
                        .dst_subresource(
                            vk::ImageSubresourceLayers::default()
                                .aspect_mask(vk::ImageAspectFlags::COLOR)
                                .layer_count(1),
                        )
                        .src_offsets(src_region.into_offsets_3d())
                        .dst_offsets(dst_region.into_offsets_3d())],
                    vk::Filter::LINEAR,
                );
            }
        }

        fn copy_buffer_to_image(
            &self,
            buffer: vk::Buffer,
            image: vk::Image,
            layout: vk::ImageLayout,
            regions: &[vk::BufferImageCopy],
        ) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device().dev().cmd_copy_buffer_to_image(
                    self.handle(),
                    buffer,
                    image,
                    layout,
                    regions,
                );
            }
        }

        fn copy_buffers(&self, src: vk::Buffer, dst: vk::Buffer, regions: &[vk::BufferCopy]) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device()
                    .dev()
                    .cmd_copy_buffer(self.handle(), src, dst, regions);
            }
        }

        fn copy_images(
            &self,
            src: vk::Image,
            src_layout: vk::ImageLayout,
            dst: vk::Image,
            dst_layout: vk::ImageLayout,
            regions: &[vk::ImageCopy],
        ) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device().dev().cmd_copy_image(
                    self.handle(),
                    src,
                    src_layout,
                    dst,
                    dst_layout,
                    regions,
                );
            }
        }

        fn clear_color_image(
            &self,
            image: vk::Image,
            format: vk::Format,
            layout: vk::ImageLayout,
            color: crate::Rgba,
            subresources: &[vk::ImageSubresourceRange],
        ) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            let clear_colors = format.clear_value(color.into_f32());

            unsafe {
                self.device().dev().cmd_clear_color_image(
                    self.handle(),
                    image,
                    layout,
                    &clear_colors,
                    subresources,
                );
            }
        }

        fn begin_rendering(&self, rendering_info: vk::RenderingInfo<'_>) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device()
                    .dev()
                    .cmd_begin_rendering(self.handle(), &rendering_info);
            }
        }

        fn set_viewport(&self, viewports: &[vk::Viewport]) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device()
                    .dev()
                    .cmd_set_viewport(self.handle(), 0, viewports);
            }
        }
        fn set_scissors(&self, scissors: &[vk::Rect2D]) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device()
                    .dev()
                    .cmd_set_scissor(self.handle(), 0, scissors);
            }
        }
        fn push_constants(
            &self,
            layout: &PipelineLayout,
            stage: vk::ShaderStageFlags,
            offset: u32,
            bytes: &[u8],
        ) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device().dev().cmd_push_constants(
                    self.handle(),
                    layout.raw(),
                    stage,
                    offset,
                    bytes,
                );
            }
        }
        fn bind_pipeline(&self, pipeline: &Pipeline) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device().dev().cmd_bind_pipeline(
                    self.handle(),
                    pipeline.bind_point(),
                    pipeline.raw(),
                );
            }
        }
        fn bind_vertex_buffers(&self, buffers: &[vk::Buffer], offsets: &[u64]) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device()
                    .dev()
                    .cmd_bind_vertex_buffers(self.handle(), 0, buffers, offsets);
            }
        }
        fn bind_indices(&self, buffer: vk::Buffer, offset: u64, kind: vk::IndexType) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device()
                    .dev()
                    .cmd_bind_index_buffer(self.handle(), buffer, offset, kind);
            }
        }

        fn bind_descriptor_sets(
            &self,
            layout: &PipelineLayout,
            bind_point: vk::PipelineBindPoint,
            descriptor_sets: &[vk::DescriptorSet],
        ) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device().dev().cmd_bind_descriptor_sets(
                    self.handle(),
                    bind_point,
                    layout.raw(),
                    0,
                    descriptor_sets,
                    &[],
                );
            }
        }

        fn draw_indexed(
            &self,
            indices: u32,
            instances: u32,
            index_offset: u32,
            vertex_offset: i32,
            instance_offset: u32,
        ) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device().dev().cmd_draw_indexed(
                    self.handle(),
                    indices,
                    instances,
                    index_offset,
                    vertex_offset,
                    instance_offset,
                );
            }
        }

        fn draw_indexed_indirect(&self, buffer: vk::Buffer, offset: u64, count: u32, stride: u32) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device().dev().cmd_draw_indexed_indirect(
                    self.handle(),
                    buffer,
                    offset,
                    count,
                    stride,
                );
            }
        }

        fn end_rendering(&self) {
            // assert_eq!(self.state(), CommandBufferState::Recording);
            unsafe {
                self.device().dev().cmd_end_rendering(self.handle());
            }
        }
    }
}

pub use command_pools::{
    CommandBuffer, CommandBufferFuture, CommandPool, CommandPools, Error as CommandPoolError,
};

mod command_pools {
    use std::{borrow::Cow, cell::Cell, mem::ManuallyDrop, sync::Arc};

    use arrayvec::ArrayVec;
    use ash::{prelude::VkResult, vk};
    use parking_lot::Mutex;
    use smol::future::FutureExt;
    use thread_local::ThreadLocal;

    use crate::{
        define_device_owned_handle,
        device::{Device, DeviceOwned},
        queue::Queue,
        sync,
        util::MutexExt,
    };

    #[derive(Debug, thiserror::Error)]
    pub enum Error {
        #[error(transparent)]
        Vulkan(#[from] vk::Result),
        #[error("CommandBuffer allocation failed: wrong number of command buffers.")]
        NoCommandBuffersAllocated,
        #[error("Attempted to submit commandbuffer to incompatible queue.")]
        InvalidQueueSubmission,
    }

    define_device_owned_handle! {
        #[derive()]
        pub CommandPool(vk::CommandPool) {
            family: u32,
            free_list: Mutex<Cell<(Vec<vk::CommandBufferLevel>, Vec<vk::CommandBuffer>)>>,
        } => |this|  unsafe {
            this.device().dev().destroy_command_pool(this.handle(), None);
        }
    }

    impl !Sync for CommandPool {}

    impl CommandPool {
        pub fn new(
            dev: Device,
            family: u32,
            flags: vk::CommandPoolCreateFlags,
            name: Option<Cow<'static, str>>,
        ) -> Result<CommandPool, vk::Result> {
            let pool_info = vk::CommandPoolCreateInfo::default()
                .queue_family_index(family)
                .flags(flags);

            let pool = unsafe { dev.dev().create_command_pool(&pool_info, None)? };

            Self::construct(dev, pool, name, family, Mutex::new(Default::default()))
        }

        #[allow(dead_code)]
        pub unsafe fn reset(&self) -> VkResult<()> {
            unsafe {
                self.device()
                    .dev()
                    .reset_command_pool(self.handle(), vk::CommandPoolResetFlags::empty())
            }
        }

        pub fn free_command_buffers(&self) {
            self.free_list_mut(|levels, buffers| {
                if !buffers.is_empty() {
                    unsafe {
                        self.device()
                            .dev()
                            .free_command_buffers(self.handle(), &buffers);
                    }
                }
                levels.clear();
                buffers.clear();
            });
        }

        pub fn free_command_buffer(
            self: &Arc<Self>,
            level: vk::CommandBufferLevel,
            buffer: vk::CommandBuffer,
        ) {
            self.free_list_mut(|levels, buffers| {
                levels.push(level);
                buffers.push(buffer);
            });
        }

        fn free_list_mut<
            T,
            F: FnOnce(&mut Vec<vk::CommandBufferLevel>, &mut Vec<vk::CommandBuffer>) -> T,
        >(
            &self,
            cb: F,
        ) -> T {
            self.free_list.with_locked(|cell| {
                let (mut levels, mut buffers) = cell.take();
                let t = cb(&mut levels, &mut buffers);
                cell.set((levels, buffers));
                t
            })
        }

        pub fn alloc(
            self: &Arc<Self>,
            name: Option<Cow<'static, str>>,
            level: vk::CommandBufferLevel,
            flags: vk::CommandBufferUsageFlags,
        ) -> Result<CommandBuffer, Error> {
            // TODO: is this really the right place?
            self.free_command_buffers();

            let handle = {
                let info = vk::CommandBufferAllocateInfo::default()
                    .command_pool(self.handle())
                    .command_buffer_count(1)
                    .level(level);

                unsafe { self.device().dev().allocate_command_buffers(&info) }?
                    .pop()
                    .ok_or(Error::NoCommandBuffersAllocated)?
            };

            let begin_info = vk::CommandBufferBeginInfo::default().flags(flags);
            unsafe {
                self.device()
                    .dev()
                    .begin_command_buffer(handle, &begin_info)?;
            }

            Ok(CommandBuffer::construct(
                self.device().clone(),
                handle,
                name,
                self.clone(),
                level,
            )?)
        }

        pub fn alloc_primary(
            self: &Arc<Self>,
            name: Option<Cow<'static, str>>,
            flags: vk::CommandBufferUsageFlags,
        ) -> Result<CommandBuffer, Error> {
            self.alloc(name, vk::CommandBufferLevel::PRIMARY, flags)
        }
    }

    impl core::fmt::Debug for CommandPool {
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            f.debug_struct("CommandPool")
                .field("inner", &self.inner)
                .field("family", &self.family)
                .finish_non_exhaustive()
        }
    }

    impl CommandPoolsShard {
        fn new() -> Self {
            Self {
                per_family_pools: ArrayVec::new(),
            }
        }

        fn find_or_default(&mut self, dev: &Device, family: u32) -> Arc<CommandPool> {
            match self
                .per_family_pools
                .binary_search_by_key(&family, |entry| entry.0)
            {
                Ok(i) => self.per_family_pools[i].1.clone(),
                Err(i) => {
                    let pool = Arc::new(
                        CommandPool::new(
                            dev.clone(),
                            family,
                            vk::CommandPoolCreateFlags::empty(),
                            Some(
                                format!("commandpool-{:?}-{}", std::thread::current().id(), family)
                                    .into(),
                            ),
                        )
                        .unwrap(),
                    );
                    self.per_family_pools
                        .try_insert(i, (family, pool.clone()))
                        .expect("too many command pools in thread local");

                    pool
                }
            }
        }
    }

    // SAFETY: this isn't safe lmao. assumes pools are unused when dropped
    unsafe impl Send for CommandPoolsShard {}

    #[derive(Debug)]
    struct CommandPoolsShard {
        per_family_pools: ArrayVec<(u32, Arc<CommandPool>), 8>,
    }

    pub struct CommandPools {
        device: Device,
        shards: ThreadLocal<crossbeam::utils::CachePadded<core::cell::Cell<CommandPoolsShard>>>,
    }

    impl CommandPools {
        pub fn new(device: Device) -> Self {
            Self {
                device,
                shards: ThreadLocal::new(),
            }
        }

        pub fn get_for_family(&self, family: u32) -> Arc<CommandPool> {
            let shard = self
                .shards
                .get_or(|| core::cell::Cell::new(CommandPoolsShard::new()).into());

            let mut inner = shard.replace(CommandPoolsShard::new());
            let pool = inner.find_or_default(&self.device, family);
            _ = shard.replace(inner);

            pool
        }

        pub fn allocate_buffer(
            &self,
            family: u32,
            name: Option<Cow<'static, str>>,
            level: vk::CommandBufferLevel,
            flags: vk::CommandBufferUsageFlags,
        ) -> Result<CommandBuffer, Error> {
            self.get_for_family(family).alloc(name, level, flags)
        }
    }

    define_device_owned_handle! {
        #[derive(Debug)]
        pub CommandBuffer(vk::CommandBuffer) {
            pool: Arc<CommandPool>,
            #[allow(unused)]
            level: vk::CommandBufferLevel,
        } => |this|  unsafe {
            this.device().dev().free_command_buffers(this.pool.handle(), &[this.handle()]);
        }
    }

    impl !Sync for CommandBuffer {}
    impl !Send for CommandBuffer {}

    impl super::traits::CommandBufferExt for CommandBuffer {}

    impl CommandBuffer {
        pub fn submit(
            self,
            queue: &Queue,
            wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
            signal: Option<vk::Semaphore>,
            fence: Arc<sync::Fence>,
        ) -> Result<CommandBufferFuture, Error> {
            let this = ManuallyDrop::new(self);

            if queue.family_index() != this.pool.family {
                tracing::error!("attempted to submit commandbuffer to incompatible queue.");
                return Err(Error::InvalidQueueSubmission);
            }
            unsafe {
                this.device().dev().end_command_buffer(this.handle())?;
            }

            let buffers = [this.handle()];
            let mut submit_info = vk::SubmitInfo::default().command_buffers(&buffers);

            if let Some(semaphore) = signal.as_ref() {
                submit_info = submit_info.signal_semaphores(core::slice::from_ref(semaphore));
            }
            if let Some((semaphore, stage)) = wait.as_ref() {
                submit_info = submit_info
                    .wait_semaphores(core::slice::from_ref(semaphore))
                    .wait_dst_stage_mask(core::slice::from_ref(stage));
            }

            queue.with_locked(|queue| unsafe {
                this.device()
                    .dev()
                    .queue_submit(queue.raw(), &[submit_info], fence.raw())
            })?;

            Ok(CommandBufferFuture {
                inner: sync::FenceFuture::new(fence),
                pool: this.pool.clone(),
                buffer: this.handle(),
                level: this.level,
            })
        }

        pub fn raw(&self) -> vk::CommandBuffer {
            self.handle()
        }
    }

    pub struct CommandBufferFuture {
        inner: sync::FenceFuture<'static>,
        pool: Arc<CommandPool>,
        buffer: vk::CommandBuffer,
        level: vk::CommandBufferLevel,
    }

    impl CommandBufferFuture {
        pub fn block(&self) -> crate::Result<()> {
            self.inner.block()
        }
    }

    impl core::future::Future for CommandBufferFuture {
        type Output = <sync::FenceFuture<'static> as core::future::Future>::Output;

        fn poll(
            mut self: std::pin::Pin<&mut Self>,
            cx: &mut std::task::Context<'_>,
        ) -> std::task::Poll<Self::Output> {
            self.inner.poll(cx)
        }
    }

    impl Drop for CommandBufferFuture {
        fn drop(&mut self) {
            self.pool.free_command_buffer(self.level, self.buffer);
        }
    }
}