use std::{ borrow::Cow, collections::{BTreeSet, HashMap, HashSet}, ffi::CStr, mem::ManuallyDrop, ops::{Deref, DerefMut}, sync::Arc, }; use ash::{ ext, khr, prelude::VkResult, vk::{self, Handle}, }; use parking_lot::Mutex; use raw_window_handle::RawDisplayHandle; use tinyvec::{ArrayVec, array_vec}; use crate::{ Instance, PhysicalDeviceFeatures, PhysicalDeviceInfo, Result, queue::{DeviceQueueInfos, DeviceQueues, Queue}, sync::{self, BinarySemaphore, TimelineSemaphore}, }; #[derive(Debug, Default)] pub struct DeviceQueueFamilies { pub(crate) families: Vec<(u32, u32)>, pub(crate) graphics: (u32, u32), pub(crate) present: (u32, u32), pub(crate) async_compute: (u32, u32), pub(crate) transfer: (u32, u32), #[expect(dead_code)] pub(crate) properties: Box<[vk::QueueFamilyProperties]>, } impl DeviceQueueFamilies { pub fn swapchain_family_indices(&self) -> ArrayVec<[u32; 2]> { let mut indices = array_vec!([u32; 2] => self.graphics.0); if self.present.0 != self.graphics.0 { indices.push(self.present.0); } indices } pub fn graphics_familty(&self) -> u32 { self.graphics.0 } pub fn present_familty(&self) -> u32 { self.present.0 } pub fn async_compute_familty(&self) -> u32 { self.async_compute.0 } pub fn transfer_familty(&self) -> u32 { self.transfer.0 } pub fn family_indices(&self, flags: QueueFlags) -> ArrayVec<[u32; 4]> { let mut indices = array_vec!([u32; 4]); if flags.contains(QueueFlags::GRAPHICS) { indices.push(self.graphics_familty()); } if flags.contains(QueueFlags::PRESENT) { indices.push(self.present_familty()); } if flags.contains(QueueFlags::ASYNC_COMPUTE) { indices.push(self.async_compute_familty()); } if flags.contains(QueueFlags::TRANSFER) { indices.push(self.transfer_familty()); } let unique_len = indices.partition_dedup().0.len(); indices.drain(unique_len..); indices } } bitflags::bitflags! { #[repr(transparent)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct QueueFlags: u32 { const GRAPHICS = 1 << 0; const ASYNC_COMPUTE = 1 << 1; const TRANSFER = 1 << 2; const PRESENT = 1 << 3; const NONE = 0; const PRESENT_GRAPHICS = 1 << 0 | 1 << 2; } } struct DeviceDrop(ash::Device); impl Drop for DeviceDrop { fn drop(&mut self) { unsafe { _ = self.0.device_wait_idle(); self.0.destroy_device(None); } } } struct DeviceExtensions { pub(crate) debug_utils: ext::debug_utils::Device, pub(crate) swapchain: Option, pub(crate) mesh_shader: Option, } type GpuAllocation = gpu_allocator::vulkan::Allocation; impl DeviceHandle for GpuAllocation { unsafe fn destroy(&mut self, device: &Device) { let mut swapped = GpuAllocation::default(); std::mem::swap(self, &mut swapped); _ = device.alloc2.lock().free(swapped); } } #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] pub enum AllocationStrategy { #[default] /// Let gpu_allocator manage the memory for this allocation, sub-allocating /// from larger blocks as needed. AllocatorManaged, /// Allocate a dedicated block of memory for this allocation. This is /// recommended for long-lived resources or resources with specific memory /// requirements. Dedicated, } #[derive(Debug)] pub(crate) enum Allocation { Owned(DeviceObject), Shared(Arc>), Unmanaged, } impl Allocation { pub(crate) fn allocation(&self) -> Option<&GpuAllocation> { match self { Allocation::Owned(obj) => Some(obj), Allocation::Shared(arc) => Some(arc.as_ref()), Allocation::Unmanaged => None, } } pub(crate) fn allocation_mut(&mut self) -> Option<&mut GpuAllocation> { match self { Allocation::Owned(obj) => Some(obj), Allocation::Shared(arc) => Arc::get_mut(arc).map(|alloc| &mut alloc.inner), Allocation::Unmanaged => None, } } } pub struct DeviceInner { pub(crate) alloc2: Mutex, pub(crate) raw: ash::Device, pub(crate) adapter: PhysicalDeviceInfo, pub(crate) instance: Instance, pub(crate) queues: DeviceQueues, pub(crate) sync_threadpool: sync::SyncThreadpool, pub(crate) device_extensions: DeviceExtensions, pub(crate) enabled_extensions: Vec<&'static CStr>, _drop: DeviceDrop, } impl core::fmt::Debug for DeviceInner { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DeviceInner") .field("device", &self.raw.handle()) .finish() } } #[macro_export] macro_rules! make_extension { ($module:path) => {{ use $module::{NAME as EXTENSION_NAME, SPEC_VERSION as EXTENSION_VERSION}; $crate::device::Extension { name: EXTENSION_NAME, version: EXTENSION_VERSION, } }}; ($module:path as $version:expr) => {{ use $module::*; $crate::device::Extension { name: NAME, version: $version, } }}; } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct Extension<'a> { pub name: &'a CStr, pub version: u32, } impl<'a> std::hash::Hash for Extension<'a> { fn hash(&self, state: &mut H) { self.name.hash(state); } } pub(crate) fn get_available_extensions( entry: &ash::Entry, layers: &[&CStr], ) -> Result> { unsafe { let extensions = core::iter::once(entry.enumerate_instance_extension_properties(None)) .chain( layers .iter() .map(|&layer| entry.enumerate_instance_extension_properties(Some(layer))), ) .filter_map(|result| result.ok()) .flatten() .collect::>(); Ok(extensions) } } /// returns a tuple of supported-or-enabled extensions and unsupported-and-requested extensions pub(crate) fn get_extensions<'a>( entry: &ash::Entry, layers: &[&'a CStr], mut extensions: Vec>, display_handle: Option, ) -> Result<(HashSet>, HashSet>)> { let available_extensions = get_available_extensions(entry, layers)?; let available_extension_names = available_extensions .iter() .filter_map(|ext| { Some(( ext.extension_name_as_c_str().ok()?.to_str().ok()?, ext.spec_version, )) }) .collect::>(); tracing::debug!( "Available extensions: {:?}", available_extension_names.iter().collect::>() ); let mut wsi_extensions = Vec::new(); wsi_extensions.push(make_extension!(khr::surface)); // taken from wgpu-hal/src/vulkan/instance.rs: // // we want to enable all the wsi extensions that are applicable to the // platform, even if the user didn't explicitly request them, or // supplied a different/no display handle, because we might later want // to create a surface for a different windowing system, and enabling // all the wsi extensions doesn't have any real downsides. // We don't notify the user if some of these extensions aren't available // (e.g. because wayland isn't supported on some unix system) if cfg!(all( unix, not(target_os = "android"), not(target_os = "macos") )) { wsi_extensions.push(make_extension!(khr::xlib_surface)); wsi_extensions.push(make_extension!(khr::xcb_surface)); wsi_extensions.push(make_extension!(khr::wayland_surface)); } if cfg!(target_os = "windows") { wsi_extensions.push(make_extension!(khr::win32_surface)); } if cfg!(target_os = "android") { wsi_extensions.push(make_extension!(khr::android_surface)); } if cfg!(target_os = "macos") { wsi_extensions.push(make_extension!(ext::metal_surface)); wsi_extensions.push(make_extension!(khr::portability_enumeration)); } if cfg!(all( unix, not(target_vendor = "apple"), not(target_family = "wasm") )) { wsi_extensions.push(make_extension!(ext::acquire_drm_display)); wsi_extensions.push(make_extension!(ext::direct_mode_display)); wsi_extensions.push(make_extension!(khr::display)); } let is_extension_available = |ext: &mut Extension| -> bool { if available_extensions .iter() .any(|inst_ext| inst_ext.extension_name_as_c_str() == Ok(ext.name)) { true } else { tracing::warn!( "Extension {:?} v{} was requested but is not available", ext.name, ext.version ); false } }; let mut enabled_extensions = extensions .extract_if(.., is_extension_available) .collect::>(); enabled_extensions.extend(wsi_extensions.extract_if(.., is_extension_available)); // if a display handle is provided, ensure the required WSI extensions are present if let Some(display_handle) = display_handle { let mut required_extensions = ash_window::enumerate_required_extensions(display_handle)? .iter() .map(|&p| Extension { name: unsafe { CStr::from_ptr(p) }, version: 0, }) // filter out extensions that are already enabled .filter(|ext| { !enabled_extensions .iter() .any(|enabled| enabled.name == ext.name) }) .collect::>(); // filter out extensions that aren't available, and log a warning for them let display_extensions = required_extensions.extract_if(.., is_extension_available); enabled_extensions.extend(display_extensions); extensions.extend(required_extensions); } // all extensions remaining in `extensions` at this point are unsupported, // and were requested by the user or are required by the display handle let unsupported_extensions = HashSet::from_iter(extensions); let out_extensions = HashSet::from_iter(enabled_extensions); Ok((out_extensions, unsupported_extensions)) } /// returns a list of enabled, or a tuple of enabled and unsupported but requested layers. pub(crate) fn get_layers<'a>( entry: &ash::Entry, wants_layers: Vec<&'a CStr>, ) -> core::result::Result, (Vec<&'a CStr>, Vec<&'a CStr>)> { unsafe { let Ok(available_layers) = entry.enumerate_instance_layer_properties() else { return Err((vec![], wants_layers)); }; let Ok(available_layer_names) = available_layers .iter() .map(|layer| layer.layer_name_as_c_str()) .collect::, _>>() else { return Err((vec![], wants_layers)); }; tracing::debug!( "Available layers: {:?}", available_layer_names .iter() .map(|s| s.to_str().unwrap_or("")) .collect::>() ); let mut enabled_layers = Vec::new(); let mut unsupported_layers = Vec::new(); for layer in wants_layers { if available_layer_names.contains(&layer) { enabled_layers.push(layer); } else { unsupported_layers.push(layer); } } if !unsupported_layers.is_empty() { Err((enabled_layers, unsupported_layers)) } else { Ok(enabled_layers) } } } impl PhysicalDeviceInfo { pub fn create_logical_device( self, instance: &Instance, extensions: &[Extension<'static>], mut features: PhysicalDeviceFeatures, display_handle: Option, ) -> Result { let queue_infos = DeviceQueueInfos::select_queue_families(instance, &self, display_handle)?; let queue_create_infos = queue_infos.into_create_infos(); let extensions = Self::required_extensions(&self, extensions); let create_info = vk::DeviceCreateInfo::default() .queue_create_infos(&queue_create_infos) .enabled_extension_names(&extensions); let create_info = features.push_to_device_create_info(create_info); let device = unsafe { instance .inner .raw .create_device(self.pdev, &create_info, None)? }; let device_queues = queue_infos.retrieve_queues(&device); let enabled_extensions = extensions .into_iter() .map(|ptr| unsafe { CStr::from_ptr(ptr) }) .collect::>(); let device_extensions = DeviceExtensions { debug_utils: ext::debug_utils::Device::new(&instance.inner.raw, &device), swapchain: if enabled_extensions.contains(&khr::swapchain::NAME) { Some(khr::swapchain::Device::new(&instance.inner.raw, &device)) } else { None }, mesh_shader: if enabled_extensions.contains(&ext::mesh_shader::NAME) { Some(ext::mesh_shader::Device::new(&instance.inner.raw, &device)) } else { None }, }; let alloc2 = gpu_allocator::vulkan::Allocator::new(&gpu_allocator::vulkan::AllocatorCreateDesc { instance: instance.inner.raw.clone(), device: device.clone(), physical_device: self.pdev, debug_settings: Default::default(), buffer_device_address: false, allocation_sizes: { const MB: u64 = 1024 * 1024; gpu_allocator::AllocationSizes::new(8 * MB, 64 * MB) .with_max_host_memblock_size(256 * MB) .with_max_device_memblock_size(256 * MB) }, })?; let inner = DeviceInner { raw: device.clone(), alloc2: Mutex::new(alloc2), instance: instance.clone(), adapter: self, queues: device_queues, device_extensions, enabled_extensions, sync_threadpool: sync::SyncThreadpool::new(), _drop: DeviceDrop(device), }; let shared = Arc::new(inner); Ok(Device { pools: Arc::new(DevicePools::new(shared.clone())), shared, }) } fn required_extensions(&self, requested_extensions: &[Extension<'static>]) -> Vec<*const i8> { let mut extensions = vec![khr::swapchain::NAME.as_ptr()]; for ext in requested_extensions { if self .properties .supported_extensions .iter() .any(|supported| { supported.extension_name_as_c_str() == Ok(ext.name) && supported.spec_version >= ext.version }) { extensions.push(ext.name.as_ptr()); } else { tracing::warn!( "Physical device {:?} does not support required extension {:?}", self.pdev, ext.name ); } } extensions } } #[derive(Clone, Debug)] pub(crate) struct DevicePools { pub(crate) fences: Pool, pub(crate) binary_semaphores: Pool, pub(crate) timeline_semaphores: Pool, } impl DevicePools { pub fn new(device: Arc) -> Self { Self { fences: Pool::new(device.clone()), binary_semaphores: Pool::new(device.clone()), timeline_semaphores: Pool::new(device), } } } #[derive(Clone, Debug)] pub struct Device { pub(crate) shared: Arc, pub(crate) pools: Arc, } impl PartialEq for Device { fn eq(&self, other: &Self) -> bool { Arc::ptr_eq(&self.shared, &other.shared) } } impl Eq for Device {} impl core::ops::Deref for Device { type Target = DeviceInner; fn deref(&self) -> &Self::Target { &self.shared } } impl DeviceInner { pub fn sync_threadpool(&self) -> &sync::SyncThreadpool { &self.sync_threadpool } pub fn dev(&self) -> &ash::Device { &self.raw } pub fn instance(&self) -> &Instance { &self.instance } pub fn queues(&self) -> &DeviceQueues { &self.queues } pub fn phy(&self) -> vk::PhysicalDevice { self.adapter.pdev } pub fn features(&self) -> &crate::PhysicalDeviceFeatures { &self.adapter.features } pub fn properties(&self) -> &crate::PhysicalDeviceProperties { &self.adapter.properties } pub fn physical_device(&self) -> &PhysicalDeviceInfo { &self.adapter } pub fn main_queue(&self) -> &Queue { self.queues.graphics() } pub fn compute_queue(&self) -> &Queue { self.queues.compute() } pub fn transfer_queue(&self) -> &Queue { self.queues.transfer() } pub unsafe fn lock_queues(&self) { unsafe { self.queues.lock(); } } pub unsafe fn unlock_queues(&self) { unsafe { self.queues.unlock(); } } pub fn wait_queue_idle(&self, queue: &Queue) -> VkResult<()> { tracing::warn!("locking queue {queue:?} and waiting for idle"); queue.with_locked(|q| unsafe { self.raw.queue_wait_idle(q.raw) })?; tracing::warn!("finished waiting: unlocking queue {queue:?}."); Ok(()) } pub fn wait_idle(&self) -> VkResult<()> { tracing::warn!("locking all queues and waiting for device to idle"); unsafe { self.lock_queues(); self.raw.device_wait_idle()?; self.unlock_queues(); } tracing::warn!("finished waiting: unlocking all queues."); Ok(()) } /// # Safety /// /// This method inherits the safety contract from [`vkSetDebugUtilsObjectName`]. In particular: /// /// - `object` must be a valid handle for one of the following: /// - An instance-level object from the same instance as this device. /// - A physical-device-level object that descends from the same physical device as this /// device. /// - A device-level object that descends from this device. /// - `object` must be externally synchronized—only the calling thread should access it during /// this call. /// /// [`vkSetDebugUtilsObjectName`]: https://registry.khronos.org/vulkan/specs/latest/man/html/vkSetDebugUtilsObjectNameEXT.html pub unsafe fn debug_name_object(&self, object: T, name: &str) { // avoid heap allocation for short names let mut buffer = [0u8; 64]; let buffer_vec: Vec; let name_bytes = if name.is_empty() { &[] } else if name.len() < buffer.len() { buffer[..name.len()].copy_from_slice(name.as_bytes()); &buffer[..] } else { buffer_vec = name .as_bytes() .iter() .cloned() .chain(std::iter::once(0)) .collect(); &buffer_vec }; let name = CStr::from_bytes_with_nul(name_bytes) .expect("there is always a nul terminator because we added one"); unsafe { _ = self .device_extensions .debug_utils .set_debug_utils_object_name( &vk::DebugUtilsObjectNameInfoEXT::default() .object_handle(object) .object_name(name), ); } } } #[derive(Clone)] pub struct DeviceOwnedDebugObject { pub(crate) device: Device, pub(crate) object: T, #[cfg(debug_assertions)] name: Option>, } impl Eq for DeviceOwnedDebugObject {} impl PartialEq for DeviceOwnedDebugObject { fn eq(&self, other: &Self) -> bool { std::sync::Arc::ptr_eq(&self.device.shared, &other.device.shared) && self.object == other.object } } impl std::fmt::Debug for DeviceOwnedDebugObject { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut fmt = f.debug_struct(core::any::type_name::()); fmt.field_with("device", |f| { write!(f, "0x{:x}", self.device.raw.handle().as_raw()) }) .field_with("handle", |f| write!(f, "0x{:x}", &self.object.as_raw())); #[cfg(debug_assertions)] { fmt.field("name", &self.name); } fmt.finish() } } impl DeviceOwnedDebugObject { pub fn new( device: crate::Device, object: T, name: Option>, ) -> ash::prelude::VkResult where T: vk::Handle + Copy, { if let Some(name) = name.as_ref() { unsafe { device.debug_name_object(object, name); } } Ok(Self { device, object, #[cfg(debug_assertions)] name, }) } pub fn dev(&self) -> &crate::Device { &self.device } pub fn handle(&self) -> T where T: Copy, { self.object } } #[derive(Debug)] pub struct DeviceObject { inner: T, device: Device, #[cfg(debug_assertions)] name: Option>, } impl Deref for DeviceObject { type Target = T; fn deref(&self) -> &Self::Target { &self.inner } } impl DerefMut for DeviceObject { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner } } impl DeviceObject { pub fn new(inner: T, device: Device, name: Option>) -> Self where T: vk::Handle + Clone, { unsafe { if let Some(name) = name.as_ref() { device.debug_name_object(inner.clone(), &name); } } Self { inner, device, #[cfg(debug_assertions)] name, } } pub fn new_without_name(inner: T, device: Device) -> Self { Self { inner, device, #[cfg(debug_assertions)] name: None, } } pub fn device(&self) -> &Device { &self.device } pub fn name(&self) -> Option<&str> { #[cfg(debug_assertions)] { self.name.as_deref().map(|cow| cow.as_ref()) } #[cfg(not(debug_assertions))] { None } } } impl Drop for DeviceObject { fn drop(&mut self) { unsafe { self.inner.destroy(&self.device); } } } pub trait DeviceHandle { unsafe fn destroy(&mut self, device: &Device); } impl DeviceHandle for vk::Semaphore { unsafe fn destroy(&mut self, device: &Device) { unsafe { device.dev().destroy_semaphore(*self, None); } } } impl DeviceHandle for vk::Fence { unsafe fn destroy(&mut self, device: &Device) { unsafe { device.dev().destroy_fence(*self, None); } } } impl DeviceHandle for vk::Buffer { unsafe fn destroy(&mut self, device: &Device) { unsafe { device.dev().destroy_buffer(*self, None); } } } impl DeviceHandle for vk::SwapchainKHR { unsafe fn destroy(&mut self, device: &Device) { unsafe { device .device_extensions .swapchain .as_ref() .map(|swapchain| swapchain.destroy_swapchain(*self, None)); } } } pub trait DeviceOwned { fn device(&self) -> &Device; fn handle(&self) -> T; } pub trait Pooled: Sized { fn create_from_pool(pool: &Pool) -> Result; } pub struct PoolObject { pub(crate) inner: ManuallyDrop, pub(crate) pool: Pool, #[cfg(debug_assertions)] pub(crate) name: Option>, } impl PoolObject { pub fn name_object(&mut self, name: impl Into>) { #[cfg(debug_assertions)] unsafe { self.name = Some(name.into()); self.pool .device .debug_name_object(T::clone(&self.inner), self.name.as_ref().unwrap()); } } pub fn device(&self) -> &Arc { &self.pool.device } } impl Drop for PoolObject { fn drop(&mut self) { let handle = unsafe { ManuallyDrop::take(&mut self.inner) }; #[cfg(debug_assertions)] if self.name.is_some() { unsafe { self.pool.device.debug_name_object(handle.clone(), "") }; } self.pool.push(handle); } } impl Deref for PoolObject { type Target = T; fn deref(&self) -> &Self::Target { &self.inner } } #[derive(Debug, Clone)] pub struct Pool { pub(crate) pool: Arc>>, pub(crate) device: Arc, } impl Pool { pub fn push(&self, item: T) { self.pool.lock().push(item); } pub fn new(device: Arc) -> Self { Self { pool: Arc::new(Mutex::new(Vec::new())), device, } } pub fn pop(&self) -> Option { self.pool.lock().pop() } } impl Pool { pub fn get(&self) -> Result> { let item = if let Some(item) = self.pool.lock().pop() { item } else { T::create_from_pool(self)? }; Ok(PoolObject { inner: ManuallyDrop::new(item), pool: self.clone(), #[cfg(debug_assertions)] name: None, }) } pub fn get_named(&self, name: Option>>) -> Result> { let mut obj = self.get()?; if let Some(name) = name { obj.name_object(name); } Ok(obj) } } // Macro for helping create and destroy Vulkan objects which are owned by a device. #[macro_export] macro_rules! define_device_owned_handle { ($(#[$attr:meta])* $ty_vis:vis $ty:ident($handle:ty) { $($(#[$field_attr:meta])* $field_vis:vis $field_name:ident : $field_ty:ty),* $(,)? } $(=> |$this:ident| $dtor:stmt)?) => { $(#[$attr])* $ty_vis struct $ty { inner: $crate::device::DeviceOwnedDebugObject<$handle>, $( $(#[$field_attr])* $field_vis $field_name: $field_ty, )* } impl $crate::device::DeviceOwned<$handle> for $ty { fn device(&self) -> &$crate::device::Device { self.inner.dev() } fn handle(&self) -> $handle { self.inner.handle() } } impl $ty { #[allow(clippy::too_many_arguments, reason = "This function is generated by a macro")] fn construct( device: $crate::device::Device, handle: $handle, name: Option<::std::borrow::Cow<'static, str>>, $($field_name: $field_ty,)* ) -> ::ash::prelude::VkResult { Ok(Self { inner: $crate::device::DeviceOwnedDebugObject::new( device, handle, name, )?, $($field_name,)* }) } } $( impl Drop for $ty { fn drop(&mut self) { #[allow(unused_mut)] let mut $this = self; $dtor } } )? }; }