From 81c6b6f4ee5cbbacc7d9675a19ce9da4711c7c7d Mon Sep 17 00:00:00 2001 From: Janis Date: Sun, 19 Jan 2025 00:24:04 +0100 Subject: [PATCH] move all of device and instance creation to Device type --- crates/renderer/Cargo.toml | 2 + crates/renderer/src/device.rs | 634 +++++++++++++++++++++++++++++++++- crates/renderer/src/lib.rs | 2 + crates/renderer/src/util.rs | 26 ++ 4 files changed, 661 insertions(+), 3 deletions(-) diff --git a/crates/renderer/Cargo.toml b/crates/renderer/Cargo.toml index 6d3bc17..b26cf2e 100644 --- a/crates/renderer/Cargo.toml +++ b/crates/renderer/Cargo.toml @@ -6,6 +6,8 @@ edition = "2021" [dependencies] tinyvec = {workspace = true} rand = {workspace = true} + +cfg-if = "1.0.0" # tokio = {workspace = true, features = ["rt", "sync"]} dyn-clone = "1" anyhow = "1.0.89" diff --git a/crates/renderer/src/device.rs b/crates/renderer/src/device.rs index 8487184..45c2528 100644 --- a/crates/renderer/src/device.rs +++ b/crates/renderer/src/device.rs @@ -1,13 +1,23 @@ -use std::{borrow::Cow, collections::BTreeMap, ops::Deref, sync::Arc}; +use std::{ + borrow::Cow, + collections::{BTreeMap, BTreeSet, HashSet}, + ffi::{CStr, CString}, + ops::Deref, + sync::Arc, +}; use ash::{ - khr, + ext, khr, prelude::VkResult, vk::{self, Handle}, }; +use raw_window_handle::RawDisplayHandle; use tinyvec::{array_vec, ArrayVec}; -use crate::{sync, Instance, PhysicalDevice, Queue}; +use crate::{ + make_extention_properties, sync, Error, ExtendsDeviceProperties2Debug, Instance, + PhysicalDevice, PhysicalDeviceFeatures, PhysicalDeviceProperties, Queue, Result, VkNameList, +}; #[derive(Debug, Default)] pub struct DeviceQueueFamilies { @@ -124,11 +134,629 @@ impl core::fmt::Debug for DeviceInner { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Extension<'a> { + pub name: &'a str, + pub version: u32, +} + +impl<'a> std::hash::Hash for Extension<'a> { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} + +#[derive(Debug)] +pub struct DeviceDesc<'a> { + pub app_name: Option<&'a str>, + pub app_version: u32, + pub layers: &'a [&'a CStr], + pub layer_settings: &'a [vk::LayerSettingEXT<'a>], + pub instance_extensions: &'a [Extension<'a>], + pub display_handle: Option, + pub features: crate::PhysicalDeviceFeatures, +} + +const VALIDATION_LAYER_NAME: &'static core::ffi::CStr = c"VK_LAYER_KHRONOS_validation"; +const DEBUG_LAYERS: [&'static core::ffi::CStr; 1] = [VALIDATION_LAYER_NAME]; +impl DeviceDesc<'_> { + fn debug_layer_settings() -> &'static [vk::LayerSettingEXT<'static>; 3] { + static SETTINGS: std::sync::LazyLock<[vk::LayerSettingEXT; 3]> = + std::sync::LazyLock::new(|| { + [ + vk::LayerSettingEXT::default() + .layer_name(VALIDATION_LAYER_NAME) + .setting_name(c"VK_KHRONOS_VALIDATION_VALIDATE_BEST_PRACTICES") + .ty(vk::LayerSettingTypeEXT::BOOL32) + .values(&[1]), + vk::LayerSettingEXT::default() + .layer_name(VALIDATION_LAYER_NAME) + .setting_name(c"VK_KHRONOS_VALIDATION_VALIDATE_BEST_PRACTICES_AMD") + .ty(vk::LayerSettingTypeEXT::BOOL32) + .values(&[1]), + vk::LayerSettingEXT::default() + .layer_name(VALIDATION_LAYER_NAME) + .setting_name(c"VK_KHRONOS_VALIDATION_VALIDATE_SYNC") + .ty(vk::LayerSettingTypeEXT::BOOL32) + .values(&[1]), + ] + }); + &SETTINGS + } +} + +impl<'a> Default for DeviceDesc<'a> { + fn default() -> Self { + Self { + app_name: Default::default(), + app_version: Default::default(), + #[cfg(debug_assertions)] + layers: &DEBUG_LAYERS, + #[cfg(not(debug_assertions))] + layers: &[], + #[cfg(debug_assertions)] + layer_settings: Self::debug_layer_settings(), + #[cfg(not(debug_assertions))] + layer_settings: &[], + instance_extensions: Default::default(), + display_handle: Default::default(), + features: Default::default(), + } + } +} + +struct DeviceBuilder; + +impl DeviceBuilder { + fn queue_family_supports_presentation( + instance: &Instance, + pdev: vk::PhysicalDevice, + queue_family: u32, + display_handle: RawDisplayHandle, + ) -> bool { + unsafe { + match display_handle { + RawDisplayHandle::Xlib(display) => { + let surface = + ash::khr::xlib_surface::Instance::new(&instance.entry, &instance.instance); + surface.get_physical_device_xlib_presentation_support( + pdev, + queue_family, + display.display.unwrap().as_ptr() as _, + display.screen as _, + ) + //todo!("xlib") + } + RawDisplayHandle::Xcb(_xcb_display_handle) => todo!("xcb"), + RawDisplayHandle::Wayland(wayland_display_handle) => { + let surface = ash::khr::wayland_surface::Instance::new( + &instance.entry, + &instance.instance, + ); + surface.get_physical_device_wayland_presentation_support( + pdev, + queue_family, + wayland_display_handle.display.cast().as_mut(), + ) + } + RawDisplayHandle::Drm(_) => { + todo!() + } + RawDisplayHandle::Windows(_) => { + ash::khr::win32_surface::Instance::new(&instance.entry, &instance.instance) + .get_physical_device_win32_presentation_support(pdev, queue_family) + } + _ => panic!("unsupported platform"), + } + } + } + + fn select_pdev_queue_families( + instance: &Instance, + display_handle: Option, + pdev: vk::PhysicalDevice, + ) -> DeviceQueueFamilies { + let queue_families = unsafe { + instance + .instance + .get_physical_device_queue_family_properties(pdev) + }; + + struct QueueFamily { + num_queues: u32, + is_present: bool, + is_compute: bool, + is_graphics: bool, + is_transfer: bool, + } + + impl QueueFamily { + #[allow(dead_code)] + fn is_graphics_and_compute(&self) -> bool { + self.is_compute && self.is_graphics + } + } + + struct QueueFamilies(Vec); + impl QueueFamilies { + fn find_first(&mut self, mut pred: F) -> Option + where + F: FnMut(&QueueFamily) -> bool, + { + if let Some((q, family)) = self + .0 + .iter_mut() + .enumerate() + .filter(|(_, family)| family.num_queues > 0) + .find(|(_, family)| pred(family)) + { + family.num_queues -= 1; + Some(q as u32) + } else { + None + } + } + + fn find_best(&mut self, mut pred: F) -> Option + where + F: FnMut(&QueueFamily) -> Option, + { + let (_, q, family) = self + .0 + .iter_mut() + .enumerate() + .filter_map(|(i, family)| { + if family.num_queues == 0 { + return None; + } + pred(family).map(|score| (score, i, family)) + }) + .max_by_key(|(score, _, _)| *score)?; + family.num_queues -= 1; + Some(q as u32) + } + } + + let mut queue_families = QueueFamilies( + queue_families + .iter() + .enumerate() + .map(|(i, family)| { + let q = i as u32; + let is_graphics = family.queue_flags.contains(vk::QueueFlags::GRAPHICS); + let is_compute = family.queue_flags.contains(vk::QueueFlags::COMPUTE); + let is_transfer = family.queue_flags.contains(vk::QueueFlags::TRANSFER) + || is_compute + || is_graphics; + let is_present = display_handle + .map(|display_handle| { + Self::queue_family_supports_presentation( + instance, + pdev, + q, + display_handle, + ) + }) + .unwrap_or(false); + QueueFamily { + num_queues: family.queue_count, + is_compute, + is_graphics, + is_present, + is_transfer, + } + }) + .collect::>(), + ); + + let graphics = queue_families + .find_best(|family| { + if !family.is_graphics { + return None; + } + // a queue with Graphics+Compute is guaranteed to exist + Some(family.is_compute as u32 * 2 + family.is_present as u32) + }) + .unwrap(); + + // find present queue first because it is rather more important than a secondary compute queue + let present = + if !queue_families.0.get(graphics as usize).unwrap().is_present { + queue_families.find_first(|family| family.is_present) + } else { + None + }.or({ + if display_handle.is_none() { + // in this case the graphics queue will be used by default + tracing::info!("no present queue available, using graphics queue as fallback for headless_surface"); + Some(graphics) + } else { + tracing::warn!("no present queue available, this is unexpected!"); + None} + }); + + let async_compute = queue_families.find_first(|family| family.is_compute); + let transfer = queue_families.find_first(|family| family.is_transfer); + + let mut unique_families = BTreeMap::::new(); + + let mut helper = |family: u32| { + use std::collections::btree_map::Entry; + let index = match unique_families.entry(family) { + Entry::Vacant(vacant_entry) => { + vacant_entry.insert(1); + 0 + } + Entry::Occupied(mut occupied_entry) => { + let idx = occupied_entry.get_mut(); + *idx += 1; + *idx - 1 + } + }; + + (family, index) + }; + + let graphics = helper(graphics); + let async_compute = async_compute.map(|f| helper(f)).unwrap_or(graphics); + let transfer = transfer.map(|f| helper(f)).unwrap_or(async_compute); + let present = present.map(|f| helper(f)).unwrap_or(graphics); + + let families = unique_families + .into_iter() + .filter(|&(_family, count)| count > 0) + .collect::>(); + + // family of each queue, of which one is allocated for each queue, with + // graphics being the fallback queue for compute and transfer, and + // present possibly being `None`, in which case it is Graphics + let queues = DeviceQueueFamilies { + families, + graphics, + async_compute, + transfer, + present, + }; + + queues + } + + fn choose_physical_device( + instance: &Instance, + display_handle: Option, + requirements: &PhysicalDeviceFeatures, + extra_properties: Vec>, + ) -> Result { + let pdevs = unsafe { instance.instance.enumerate_physical_devices()? }; + + let (pdev, properties) = pdevs + .into_iter() + .map(|pdev| { + let mut props = PhysicalDeviceProperties::default().extra_properties( + extra_properties + .iter() + .map(|b| dyn_clone::clone_box(&**b)) + .collect::>(), + ); + props.query(&instance.instance, pdev); + + (pdev, props) + }) + // filter devices which dont support the version of Vulkan we are requesting + .filter(|(_, props)| props.base.api_version >= requirements.version) + // filter devices which don't support the device extensions we + // are requesting + // TODO: figure out a way to fall back to some + // device which doesn't support all of the extensions. + .filter(|(pdev, _)| { + let query_features = + PhysicalDeviceFeatures::query(&instance.instance, *pdev).unwrap(); + + requirements.compatible_with(&query_features) + }) + .max_by_key(|(_, props)| { + let score = match props.base.device_type { + vk::PhysicalDeviceType::DISCRETE_GPU => 5, + vk::PhysicalDeviceType::INTEGRATED_GPU => 4, + vk::PhysicalDeviceType::VIRTUAL_GPU => 3, + vk::PhysicalDeviceType::CPU => 2, + vk::PhysicalDeviceType::OTHER => 1, + _ => unreachable!(), + }; + + // score based on limits or other properties + + score + }) + .ok_or(Error::NoPhysicalDevice)?; + + Ok(PhysicalDevice { + queue_families: Self::select_pdev_queue_families(instance, display_handle, pdev), + pdev, + properties, + }) + } + + fn get_available_extensions( + entry: &ash::Entry, + layers: &[&CStr], + ) -> Result> { + unsafe { + let extensions = core::iter::once(entry.enumerate_instance_extension_properties(None)) + .chain( + layers + .iter() + .map(|&layer| entry.enumerate_instance_extension_properties(Some(layer))), + ) + .filter_map(|result| result.ok()) + .flatten() + .collect::>(); + + Ok(extensions) + } + } + + /// returns a tuple of supported/enabled extensions and unsupported/requested extensions + fn get_extensions<'a>( + entry: &ash::Entry, + layers: &[&'a CStr], + extensions: impl Iterator> + 'a, + display_handle: Option, + ) -> Result<(Vec>, Vec>)> { + unsafe { + let available_extensions = Self::get_available_extensions(entry, layers)?; + + let available_extension_names = available_extensions + .iter() + .filter_map(|ext| { + Some(Extension { + name: ext.extension_name_as_c_str().ok()?.to_str().ok()?, + version: ext.spec_version, + }) + }) + .collect::>(); + + let mut out_extensions = Vec::new(); + let mut unsupported_extensions = Vec::new(); + + for extension in extensions { + if available_extension_names.contains(&extension) { + out_extensions.push(extension); + } else { + unsupported_extensions.push(extension); + } + } + + let required_extension_names = display_handle + .map(|display_handle| ash_window::enumerate_required_extensions(display_handle)) + .unwrap_or(Ok(&[]))?; + + for &extension in required_extension_names { + let extension = core::ffi::CStr::from_ptr(extension); + let extension = Extension { + name: extension.to_str()?, + version: 0, + }; + if available_extension_names.contains(&extension) { + out_extensions.push(extension); + } else { + unsupported_extensions.push(extension); + } + } + + Ok((out_extensions, unsupported_extensions)) + } + } + + fn get_layers<'a>( + entry: &ash::Entry, + wants_layers: impl Iterator + 'a, + ) -> core::result::Result, (Vec<&'a CStr>, Vec<&'a CStr>)> { + unsafe { + let wants_layers = wants_layers.collect::>(); + + let available_layers = entry + .enumerate_instance_layer_properties() + .map_err(|_| (Vec::<&'a CStr>::new(), wants_layers.clone()))?; + let available_layer_names = available_layers + .iter() + .map(|layer| layer.layer_name_as_c_str()) + .collect::, _>>() + .map_err(|_| (Vec::<&'a CStr>::new(), wants_layers.clone()))?; + + let mut out_layers = Vec::new(); + let mut unsupported_layers = Vec::new(); + + for layer in wants_layers { + if available_layer_names.contains(&layer) { + out_layers.push(layer); + } else { + unsupported_layers.push(layer); + } + } + + if !unsupported_layers.is_empty() { + Err((out_layers, unsupported_layers)) + } else { + Ok(out_layers) + } + } + } +} + #[derive(Clone, Debug)] pub struct Device(Arc); pub type WeakDevice = std::sync::Weak; impl Device { + pub fn new_from_default_desc(display_handle: Option) -> crate::Result { + Self::new_from_desc(DeviceDesc { + app_name: Some("Vidya"), + app_version: vk::make_api_version(0, 0, 1, 0), + display_handle, + features: crate::PhysicalDeviceFeatures::default() + .version(vk::make_api_version(0, 1, 3, 0)) + .features10( + vk::PhysicalDeviceFeatures::default() + .sampler_anisotropy(true) + .fill_mode_non_solid(true) + .multi_draw_indirect(true), + ) + .features11( + vk::PhysicalDeviceVulkan11Features::default().shader_draw_parameters(true), + ) + .features12( + vk::PhysicalDeviceVulkan12Features::default() + .shader_int8(true) + .runtime_descriptor_array(true) + .descriptor_binding_partially_bound(true) + .shader_sampled_image_array_non_uniform_indexing(true) + .descriptor_binding_sampled_image_update_after_bind(true) + .storage_buffer8_bit_access(true), + ) + .with_extension( + make_extention_properties( + ash::ext::mesh_shader::NAME, + ash::ext::mesh_shader::SPEC_VERSION, + ), + vk::PhysicalDeviceMeshShaderFeaturesEXT::default() + .mesh_shader(true) + .task_shader(true), + ) + .with_extension( + make_extention_properties( + ash::ext::index_type_uint8::NAME, + ash::ext::index_type_uint8::SPEC_VERSION, + ), + vk::PhysicalDeviceIndexTypeUint8FeaturesEXT::default().index_type_uint8(true), + ) + .with_extensions2([make_extention_properties( + khr::spirv_1_4::NAME, + khr::spirv_1_4::SPEC_VERSION, + )]), + ..Default::default() + }) + } + pub fn new_from_desc(desc: DeviceDesc) -> crate::Result { + let entry = unsafe { ash::Entry::load()? }; + + let app_name = desc + .app_name + .and_then(|name| CString::new(name).ok()) + .unwrap_or(c"ShooterGame".to_owned()); + + let app_info = vk::ApplicationInfo::default() + .api_version(desc.features.version) + .application_name(&app_name) + .engine_name(c"VidyaEngine") + .application_version(desc.app_version) + .engine_version(vk::make_api_version(0, 0, 1, 0)); + + let mut validation_info = + vk::LayerSettingsCreateInfoEXT::default().settings(&desc.layer_settings); + + let extra_layers = (!desc.layer_settings.is_empty()) + .then_some(Extension { + name: "VK_EXT_debug_utils", + version: 1, + }) + .into_iter() + .chain( + cfg_if::cfg_if! {if #[cfg(debug_assertions)] { + [Extension { + name: "VK_EXT_layer_settings", + version: 2, + }]} else {[]}} + .into_iter(), + ); + + let layers = DeviceBuilder::get_layers(&entry, desc.layers.into_iter().cloned()).unwrap(); + + let (extensions, unsupported_extensions) = DeviceBuilder::get_extensions( + &entry, + &layers, + desc.instance_extensions + .into_iter() + .cloned() + .chain(extra_layers), + desc.display_handle, + )?; + + if !unsupported_extensions.is_empty() { + tracing::error!( + "extensions were requested but not supported by instance: {:?}", + unsupported_extensions + ); + } + + let layers = VkNameList::from_strs(&layers); + let extensions = crate::util::CStringList::from_iter(extensions.iter().map(|ext| ext.name)); + + let create_info = vk::InstanceCreateInfo::default() + .application_info(&app_info) + .enabled_extension_names(&extensions.strings) + .enabled_layer_names(&layers.names) + .push_next(&mut validation_info); + + let instance = unsafe { entry.create_instance(&create_info, None)? }; + + let debug_info = vk::DebugUtilsMessengerCreateInfoEXT::default() + .message_severity( + vk::DebugUtilsMessageSeverityFlagsEXT::ERROR + | vk::DebugUtilsMessageSeverityFlagsEXT::WARNING + | vk::DebugUtilsMessageSeverityFlagsEXT::INFO, + ) + .message_type( + vk::DebugUtilsMessageTypeFlagsEXT::GENERAL + | vk::DebugUtilsMessageTypeFlagsEXT::VALIDATION + | vk::DebugUtilsMessageTypeFlagsEXT::PERFORMANCE, + ) + .pfn_user_callback(Some(crate::debug::debug_callback)); + + let debug_utils_instance = ash::ext::debug_utils::Instance::new(&entry, &instance); + let debug_utils_messenger = + unsafe { debug_utils_instance.create_debug_utils_messenger(&debug_info, None)? }; + + let surface_instance = ash::khr::surface::Instance::new(&entry, &instance); + + let instance = Arc::new(Instance { + instance, + debug_utils: debug_utils_instance, + debug_utils_messenger, + surface: surface_instance, + entry, + }); + + let mut features = desc.features.with_extension2(make_extention_properties( + khr::swapchain::NAME, + khr::swapchain::SPEC_VERSION, + )); + + //these are required for the renderpass + let features13 = features.physical_features_13.get_or_insert_default(); + features13.synchronization2 = vk::TRUE; + features13.dynamic_rendering = vk::TRUE; + features13.maintenance4 = vk::TRUE; + + // Consider this: switching physical device in game? + // anything above this point is device agnostic, everything below would have to be recreated + // additionally, pdev would have to be derived from a device and not a scoring function. + + let pdev = DeviceBuilder::choose_physical_device( + &instance, + desc.display_handle, + &features, + vec![Box::new( + vk::PhysicalDeviceMeshShaderPropertiesEXT::default(), + )], + )?; + + tracing::trace!("pdev: {pdev:?}"); + let device = Device::new(instance.clone(), pdev, features)?; + + Ok(device) + } + pub fn new( instance: Arc, physical: PhysicalDevice, diff --git a/crates/renderer/src/lib.rs b/crates/renderer/src/lib.rs index 6e9bc1f..0c274df 100644 --- a/crates/renderer/src/lib.rs +++ b/crates/renderer/src/lib.rs @@ -103,6 +103,8 @@ pub enum Error { #[error(transparent)] CStrError(#[from] core::ffi::c_str::FromBytesUntilNulError), #[error(transparent)] + Utf8Error(#[from] core::str::Utf8Error), + #[error(transparent)] NulError(#[from] std::ffi::NulError), #[error("No Physical Device found.")] NoPhysicalDevice, diff --git a/crates/renderer/src/util.rs b/crates/renderer/src/util.rs index 15553c8..1183bfc 100644 --- a/crates/renderer/src/util.rs +++ b/crates/renderer/src/util.rs @@ -520,3 +520,29 @@ impl<'a> Iterator for ChunkedBitIter<'a> { Some(iter) } } + +pub struct CStringList { + pub strings: Box<[*const i8]>, + bytes: Box<[u8]>, +} + +impl CStringList { + pub fn from_iter(i: I) -> CStringList + where + I: IntoIterator, + S: AsRef, + { + let mut bytes = vec![]; + let mut strings = vec![]; + i.into_iter().for_each(|s| { + strings.push(bytes.as_ptr_range().end as *const i8); + bytes.extend_from_slice(s.as_ref().as_bytes()); + bytes.push(0); + }); + + Self { + strings: strings.into_boxed_slice(), + bytes: bytes.into_boxed_slice(), + } + } +}