more refactoring
This commit is contained in:
parent
05bf7dd61f
commit
4d2dcafb7a
|
|
@ -1,8 +1,7 @@
|
||||||
use std::{
|
use std::{
|
||||||
borrow::Cow,
|
borrow::Cow,
|
||||||
collections::{BTreeMap, BTreeSet, HashMap, HashSet},
|
collections::{BTreeSet, HashMap, HashSet},
|
||||||
ffi::{CStr, CString},
|
ffi::CStr,
|
||||||
ops::Deref,
|
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -15,8 +14,11 @@ use raw_window_handle::RawDisplayHandle;
|
||||||
use tinyvec::{ArrayVec, array_vec};
|
use tinyvec::{ArrayVec, array_vec};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
Error, ExtendsDeviceProperties2Debug, Instance, PhysicalDevice, PhysicalDeviceFeatures,
|
Instance, PhysicalDeviceFeatures, PhysicalDeviceInfo, Result,
|
||||||
PhysicalDeviceInfo, PhysicalDeviceProperties, Queue, Result, instance::InstanceInner, sync,
|
instance::InstanceInner,
|
||||||
|
queue::Queue,
|
||||||
|
queue::{DeviceQueueInfos, DeviceQueues},
|
||||||
|
sync,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
|
|
@ -89,18 +91,8 @@ bitflags::bitflags! {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[repr(transparent)]
|
struct DeviceDrop(ash::Device);
|
||||||
struct DeviceWrapper(ash::Device);
|
impl Drop for DeviceDrop {
|
||||||
|
|
||||||
impl Deref for DeviceWrapper {
|
|
||||||
type Target = ash::Device;
|
|
||||||
|
|
||||||
fn deref(&self) -> &Self::Target {
|
|
||||||
&self.0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for DeviceWrapper {
|
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
unsafe {
|
unsafe {
|
||||||
_ = self.0.device_wait_idle();
|
_ = self.0.device_wait_idle();
|
||||||
|
|
@ -109,30 +101,28 @@ impl Drop for DeviceWrapper {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct DeviceExtensions {
|
||||||
|
debug_utils: ext::debug_utils::Device,
|
||||||
|
mesh_shader: Option<ext::mesh_shader::Device>,
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(unused)]
|
#[allow(unused)]
|
||||||
pub struct DeviceInner {
|
pub struct DeviceInner {
|
||||||
alloc: vk_mem::Allocator,
|
alloc: vk_mem::Allocator,
|
||||||
device: DeviceWrapper,
|
raw: ash::Device,
|
||||||
physical: PhysicalDevice,
|
adapter: PhysicalDeviceInfo,
|
||||||
instance: Arc<InstanceInner>,
|
instance: Arc<InstanceInner>,
|
||||||
swapchain: khr::swapchain::Device,
|
queues: DeviceQueues,
|
||||||
debug_utils: ash::ext::debug_utils::Device,
|
|
||||||
allocated_queues: BTreeMap<(u32, u32), Queue>,
|
|
||||||
// these are resident in allocated_queues, and may in fact be clones of each
|
|
||||||
// other, for ease of access
|
|
||||||
main_queue: Queue,
|
|
||||||
compute_queue: Queue,
|
|
||||||
transfer_queue: Queue,
|
|
||||||
present_queue: Queue,
|
|
||||||
sync_threadpool: sync::SyncThreadpool,
|
sync_threadpool: sync::SyncThreadpool,
|
||||||
features: crate::PhysicalDeviceFeatures,
|
device_extensions: DeviceExtensions,
|
||||||
properties: crate::PhysicalDeviceProperties,
|
enabled_extensions: Vec<&'static CStr>,
|
||||||
|
_drop: DeviceDrop,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl core::fmt::Debug for DeviceInner {
|
impl core::fmt::Debug for DeviceInner {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
f.debug_struct("DeviceInner")
|
f.debug_struct("DeviceInner")
|
||||||
.field("device", &self.device.handle())
|
.field("device", &self.raw.handle())
|
||||||
.finish()
|
.finish()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -224,288 +214,6 @@ impl<'a> Default for DeviceDesc<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct DeviceBuilder;
|
|
||||||
|
|
||||||
impl DeviceBuilder {
|
|
||||||
fn queue_family_supports_presentation(
|
|
||||||
instance: &Arc<InstanceInner>,
|
|
||||||
pdev: vk::PhysicalDevice,
|
|
||||||
queue_family: u32,
|
|
||||||
display_handle: RawDisplayHandle,
|
|
||||||
) -> bool {
|
|
||||||
unsafe {
|
|
||||||
match display_handle {
|
|
||||||
RawDisplayHandle::Xlib(display) => {
|
|
||||||
let surface =
|
|
||||||
ash::khr::xlib_surface::Instance::new(&instance.entry, &instance.raw);
|
|
||||||
surface.get_physical_device_xlib_presentation_support(
|
|
||||||
pdev,
|
|
||||||
queue_family,
|
|
||||||
display.display.unwrap().as_ptr() as _,
|
|
||||||
display.screen as _,
|
|
||||||
)
|
|
||||||
//todo!("xlib")
|
|
||||||
}
|
|
||||||
RawDisplayHandle::Xcb(_xcb_display_handle) => todo!("xcb"),
|
|
||||||
RawDisplayHandle::Wayland(wayland_display_handle) => {
|
|
||||||
let surface =
|
|
||||||
ash::khr::wayland_surface::Instance::new(&instance.entry, &instance.raw);
|
|
||||||
surface.get_physical_device_wayland_presentation_support(
|
|
||||||
pdev,
|
|
||||||
queue_family,
|
|
||||||
wayland_display_handle.display.cast().as_mut(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
RawDisplayHandle::Drm(_) => {
|
|
||||||
todo!()
|
|
||||||
}
|
|
||||||
RawDisplayHandle::Windows(_) => {
|
|
||||||
ash::khr::win32_surface::Instance::new(&instance.entry, &instance.raw)
|
|
||||||
.get_physical_device_win32_presentation_support(pdev, queue_family)
|
|
||||||
}
|
|
||||||
_ => panic!("unsupported platform"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn select_pdev_queue_families(
|
|
||||||
instance: &Arc<InstanceInner>,
|
|
||||||
display_handle: Option<RawDisplayHandle>,
|
|
||||||
pdev: vk::PhysicalDevice,
|
|
||||||
) -> DeviceQueueFamilies {
|
|
||||||
let queue_familiy_properties = unsafe {
|
|
||||||
instance
|
|
||||||
.raw
|
|
||||||
.get_physical_device_queue_family_properties(pdev)
|
|
||||||
};
|
|
||||||
|
|
||||||
struct QueueFamily {
|
|
||||||
num_queues: u32,
|
|
||||||
is_present: bool,
|
|
||||||
is_compute: bool,
|
|
||||||
is_graphics: bool,
|
|
||||||
is_transfer: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl QueueFamily {
|
|
||||||
#[allow(dead_code)]
|
|
||||||
fn is_graphics_and_compute(&self) -> bool {
|
|
||||||
self.is_compute && self.is_graphics
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
struct QueueFamilies(Vec<QueueFamily>);
|
|
||||||
impl QueueFamilies {
|
|
||||||
fn find_first<F>(&mut self, mut pred: F) -> Option<u32>
|
|
||||||
where
|
|
||||||
F: FnMut(&QueueFamily) -> bool,
|
|
||||||
{
|
|
||||||
if let Some((q, family)) = self
|
|
||||||
.0
|
|
||||||
.iter_mut()
|
|
||||||
.enumerate()
|
|
||||||
.filter(|(_, family)| family.num_queues > 0)
|
|
||||||
.find(|(_, family)| pred(family))
|
|
||||||
{
|
|
||||||
family.num_queues -= 1;
|
|
||||||
Some(q as u32)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn find_best<F>(&mut self, mut pred: F) -> Option<u32>
|
|
||||||
where
|
|
||||||
F: FnMut(&QueueFamily) -> Option<u32>,
|
|
||||||
{
|
|
||||||
let (_, q, family) = self
|
|
||||||
.0
|
|
||||||
.iter_mut()
|
|
||||||
.enumerate()
|
|
||||||
.filter_map(|(i, family)| {
|
|
||||||
if family.num_queues == 0 {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
pred(family).map(|score| (score, i, family))
|
|
||||||
})
|
|
||||||
.max_by_key(|(score, _, _)| *score)?;
|
|
||||||
family.num_queues -= 1;
|
|
||||||
Some(q as u32)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut queue_families = QueueFamilies(
|
|
||||||
queue_familiy_properties
|
|
||||||
.iter()
|
|
||||||
.enumerate()
|
|
||||||
.map(|(i, family)| {
|
|
||||||
let q = i as u32;
|
|
||||||
let is_graphics = family.queue_flags.contains(vk::QueueFlags::GRAPHICS);
|
|
||||||
let is_compute = family.queue_flags.contains(vk::QueueFlags::COMPUTE);
|
|
||||||
let is_transfer = family.queue_flags.contains(vk::QueueFlags::TRANSFER)
|
|
||||||
|| is_compute
|
|
||||||
|| is_graphics;
|
|
||||||
let is_present = display_handle
|
|
||||||
.map(|display_handle| {
|
|
||||||
Self::queue_family_supports_presentation(
|
|
||||||
instance,
|
|
||||||
pdev,
|
|
||||||
q,
|
|
||||||
display_handle,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.unwrap_or(false);
|
|
||||||
QueueFamily {
|
|
||||||
num_queues: family.queue_count,
|
|
||||||
is_compute,
|
|
||||||
is_graphics,
|
|
||||||
is_present,
|
|
||||||
is_transfer,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
);
|
|
||||||
|
|
||||||
let graphics = queue_families
|
|
||||||
.find_best(|family| {
|
|
||||||
if !family.is_graphics {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
// a queue with Graphics+Compute is guaranteed to exist
|
|
||||||
Some(family.is_compute as u32 * 2 + family.is_present as u32)
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// find present queue first because it is rather more important than a secondary compute queue
|
|
||||||
let present = if !queue_families.0.get(graphics as usize).unwrap().is_present {
|
|
||||||
queue_families.find_first(|family| family.is_present)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
.or({
|
|
||||||
if display_handle.is_none() {
|
|
||||||
// in this case the graphics queue will be used by default
|
|
||||||
tracing::info!("no display handle, using graphics queue family as fallback");
|
|
||||||
Some(graphics)
|
|
||||||
} else {
|
|
||||||
tracing::warn!("no present queue available, this is unexpected!");
|
|
||||||
None
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let async_compute = queue_families.find_first(|family| family.is_compute);
|
|
||||||
let transfer = queue_families.find_first(|family| family.is_transfer);
|
|
||||||
|
|
||||||
let mut unique_families = BTreeMap::<u32, u32>::new();
|
|
||||||
|
|
||||||
let mut helper = |family: u32| {
|
|
||||||
use std::collections::btree_map::Entry;
|
|
||||||
let index = match unique_families.entry(family) {
|
|
||||||
Entry::Vacant(vacant_entry) => {
|
|
||||||
vacant_entry.insert(0);
|
|
||||||
0
|
|
||||||
}
|
|
||||||
Entry::Occupied(mut occupied_entry) => {
|
|
||||||
let max = queue_families.0[family as usize].num_queues;
|
|
||||||
let idx = occupied_entry.get_mut();
|
|
||||||
if *idx + 1 >= max {
|
|
||||||
tracing::warn!("ran out of queues in family {family}, reusing queue {idx}");
|
|
||||||
*idx
|
|
||||||
} else {
|
|
||||||
*idx += 1;
|
|
||||||
*idx
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
(family, index)
|
|
||||||
};
|
|
||||||
|
|
||||||
let graphics = helper(graphics);
|
|
||||||
let async_compute = async_compute.map(&mut helper).unwrap_or(graphics);
|
|
||||||
let transfer = transfer.map(&mut helper).unwrap_or(async_compute);
|
|
||||||
let present = present.map(&mut helper).unwrap_or(graphics);
|
|
||||||
|
|
||||||
tracing::debug!(
|
|
||||||
"selected queue families: graphics={:?}, async_compute={:?}, transfer={:?}, present={:?}",
|
|
||||||
graphics,
|
|
||||||
async_compute,
|
|
||||||
transfer,
|
|
||||||
present
|
|
||||||
);
|
|
||||||
|
|
||||||
let families = unique_families
|
|
||||||
.into_iter()
|
|
||||||
.map(|(family, count)| (family, count + 1))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
// family of each queue, of which one is allocated for each queue, with
|
|
||||||
// graphics being the fallback queue for compute and transfer, and
|
|
||||||
// present possibly being `None`, in which case it is Graphics
|
|
||||||
DeviceQueueFamilies {
|
|
||||||
families,
|
|
||||||
graphics,
|
|
||||||
async_compute,
|
|
||||||
transfer,
|
|
||||||
present,
|
|
||||||
properties: queue_familiy_properties.into_boxed_slice(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn choose_physical_device(
|
|
||||||
instance: &Arc<InstanceInner>,
|
|
||||||
display_handle: Option<RawDisplayHandle>,
|
|
||||||
requirements: &PhysicalDeviceFeatures,
|
|
||||||
extra_properties: Vec<Box<dyn ExtendsDeviceProperties2Debug>>,
|
|
||||||
) -> Result<PhysicalDevice> {
|
|
||||||
let pdevs = unsafe { instance.raw.enumerate_physical_devices()? };
|
|
||||||
|
|
||||||
let (pdev, properties) = pdevs
|
|
||||||
.into_iter()
|
|
||||||
.map(|pdev| {
|
|
||||||
let mut props = PhysicalDeviceProperties::default().extra_properties(
|
|
||||||
extra_properties
|
|
||||||
.iter()
|
|
||||||
.map(|b| dyn_clone::clone_box(&**b))
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
);
|
|
||||||
props.query(&instance.raw, pdev);
|
|
||||||
|
|
||||||
(pdev, props)
|
|
||||||
})
|
|
||||||
// filter devices which dont support the version of Vulkan we are requesting
|
|
||||||
.filter(|(_, props)| props.base.api_version >= requirements.version)
|
|
||||||
// filter devices which don't support the device extensions we
|
|
||||||
// are requesting
|
|
||||||
// TODO: figure out a way to fall back to some
|
|
||||||
// device which doesn't support all of the extensions.
|
|
||||||
.filter(|(pdev, _)| {
|
|
||||||
let query_features = PhysicalDeviceFeatures::query(&instance.raw, *pdev).unwrap();
|
|
||||||
|
|
||||||
requirements.compatible_with(&query_features)
|
|
||||||
})
|
|
||||||
.max_by_key(|(_, props)| {
|
|
||||||
match props.base.device_type {
|
|
||||||
vk::PhysicalDeviceType::DISCRETE_GPU => 5,
|
|
||||||
vk::PhysicalDeviceType::INTEGRATED_GPU => 4,
|
|
||||||
vk::PhysicalDeviceType::VIRTUAL_GPU => 3,
|
|
||||||
vk::PhysicalDeviceType::CPU => 2,
|
|
||||||
vk::PhysicalDeviceType::OTHER => 1,
|
|
||||||
_ => unreachable!(),
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: score based on limits or other properties
|
|
||||||
})
|
|
||||||
.ok_or(Error::NoPhysicalDevice)?;
|
|
||||||
|
|
||||||
Ok(PhysicalDevice {
|
|
||||||
queue_families: Self::select_pdev_queue_families(instance, display_handle, pdev),
|
|
||||||
pdev,
|
|
||||||
properties,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn get_available_extensions(
|
pub(crate) fn get_available_extensions(
|
||||||
entry: &ash::Entry,
|
entry: &ash::Entry,
|
||||||
layers: &[&CStr],
|
layers: &[&CStr],
|
||||||
|
|
@ -688,8 +396,103 @@ pub(crate) fn get_layers<'a>(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl PhysicalDeviceInfo {
|
||||||
|
pub fn create_logical_device(
|
||||||
|
self,
|
||||||
|
instance: &Instance,
|
||||||
|
extensions: &[Extension<'static>],
|
||||||
|
mut features: PhysicalDeviceFeatures,
|
||||||
|
display_handle: Option<RawDisplayHandle>,
|
||||||
|
) -> Result<Device> {
|
||||||
|
let queue_infos = DeviceQueueInfos::select_queue_families(instance, &self, display_handle)?;
|
||||||
|
|
||||||
|
let queue_create_infos = queue_infos.into_create_infos();
|
||||||
|
let extensions = Self::required_extensions(&self, extensions);
|
||||||
|
|
||||||
|
let create_info = vk::DeviceCreateInfo::default()
|
||||||
|
.queue_create_infos(&queue_create_infos)
|
||||||
|
.enabled_extension_names(&extensions);
|
||||||
|
let create_info = features.push_to_device_create_info(create_info);
|
||||||
|
|
||||||
|
let device = unsafe {
|
||||||
|
instance
|
||||||
|
.inner
|
||||||
|
.raw
|
||||||
|
.create_device(self.pdev, &create_info, None)?
|
||||||
|
};
|
||||||
|
|
||||||
|
let device_queues = queue_infos.retrieve_queues(&device);
|
||||||
|
|
||||||
|
let enabled_extensions = extensions
|
||||||
|
.into_iter()
|
||||||
|
.map(|ptr| unsafe { CStr::from_ptr(ptr) })
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
let device_extensions = DeviceExtensions {
|
||||||
|
debug_utils: ext::debug_utils::Device::new(&instance.inner.raw, &device),
|
||||||
|
mesh_shader: if enabled_extensions.contains(&ext::mesh_shader::NAME) {
|
||||||
|
Some(ext::mesh_shader::Device::new(&instance.inner.raw, &device))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
let inner = DeviceInner {
|
||||||
|
raw: device.clone(),
|
||||||
|
alloc: unsafe {
|
||||||
|
vk_mem::Allocator::new(vk_mem::AllocatorCreateInfo::new(
|
||||||
|
&instance.inner.raw,
|
||||||
|
&device,
|
||||||
|
self.pdev,
|
||||||
|
))?
|
||||||
|
},
|
||||||
|
instance: instance.inner.clone(),
|
||||||
|
adapter: self,
|
||||||
|
queues: device_queues,
|
||||||
|
device_extensions,
|
||||||
|
enabled_extensions,
|
||||||
|
sync_threadpool: sync::SyncThreadpool::new(),
|
||||||
|
_drop: DeviceDrop(device),
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(Device(Arc::new(inner)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn required_extensions(&self, requested_extensions: &[Extension<'static>]) -> Vec<*const i8> {
|
||||||
|
let mut extensions = vec![khr::swapchain::NAME.as_ptr()];
|
||||||
|
for ext in requested_extensions {
|
||||||
|
if self
|
||||||
|
.properties
|
||||||
|
.supported_extensions
|
||||||
|
.iter()
|
||||||
|
.any(|supported| {
|
||||||
|
supported.extension_name_as_c_str() == Ok(ext.name)
|
||||||
|
&& supported.spec_version >= ext.version
|
||||||
|
})
|
||||||
|
{
|
||||||
|
extensions.push(ext.name.as_ptr());
|
||||||
|
} else {
|
||||||
|
tracing::warn!(
|
||||||
|
"Physical device {:?} does not support required extension {:?}",
|
||||||
|
self.pdev,
|
||||||
|
ext.name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
extensions
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct Device(Arc<DeviceInner>);
|
pub struct Device(Arc<DeviceInner>);
|
||||||
|
|
||||||
|
impl core::ops::Deref for Device {
|
||||||
|
type Target = DeviceInner;
|
||||||
|
|
||||||
|
fn deref(&self) -> &Self::Target {
|
||||||
|
&self.0
|
||||||
|
}
|
||||||
|
}
|
||||||
pub type WeakDevice = std::sync::Weak<DeviceInner>;
|
pub type WeakDevice = std::sync::Weak<DeviceInner>;
|
||||||
|
|
||||||
impl Device {
|
impl Device {
|
||||||
|
|
@ -722,113 +525,9 @@ impl Device {
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
// Consider this: switching physical device in game?
|
todo!()
|
||||||
// anything above this point is device agnostic, everything below would have to be recreated
|
|
||||||
// additionally, pdev would have to be derived from a device and not a scoring function.
|
|
||||||
|
|
||||||
let pdev = instance.choose_adapter_default(
|
|
||||||
None,
|
|
||||||
&[make_extension!(ext::mesh_shader)],
|
|
||||||
Some(&features),
|
|
||||||
)?;
|
|
||||||
|
|
||||||
tracing::trace!("pdev: {pdev:?}");
|
|
||||||
let device = Device::new(instance.inner.clone(), pdev, features)?;
|
|
||||||
|
|
||||||
Ok(device)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new(
|
|
||||||
instance: Arc<InstanceInner>,
|
|
||||||
physical: PhysicalDeviceInfo,
|
|
||||||
mut features: crate::PhysicalDeviceFeatures,
|
|
||||||
) -> VkResult<Self> {
|
|
||||||
// we have 4 queues at most: graphics, compute, transfer, present
|
|
||||||
let priorities = [1.0f32; 4];
|
|
||||||
|
|
||||||
let queue_infos = physical
|
|
||||||
.queue_families
|
|
||||||
.families
|
|
||||||
.iter()
|
|
||||||
.map(|&(family, queues)| {
|
|
||||||
vk::DeviceQueueCreateInfo::default()
|
|
||||||
.queue_family_index(family)
|
|
||||||
.queue_priorities(&priorities[..queues as usize])
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
let extensions = features
|
|
||||||
.device_extensions
|
|
||||||
.iter()
|
|
||||||
.map(|ext| ext.extension_name.as_ptr())
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
let mut features2 = features.features2();
|
|
||||||
let device_info = vk::DeviceCreateInfo::default()
|
|
||||||
.queue_create_infos(&queue_infos)
|
|
||||||
.enabled_extension_names(&extensions)
|
|
||||||
.push_next(&mut features2);
|
|
||||||
|
|
||||||
tracing::debug!("creating device: {:#?}", device_info);
|
|
||||||
let device = unsafe {
|
|
||||||
let device = instance
|
|
||||||
.raw
|
|
||||||
.create_device(physical.pdev, &device_info, None)?;
|
|
||||||
|
|
||||||
tracing::debug!("allocating queues: {queue_infos:#?}");
|
|
||||||
let allocated_queues = queue_infos
|
|
||||||
.iter()
|
|
||||||
.flat_map(|info| {
|
|
||||||
(0..info.queue_count).map(|i| {
|
|
||||||
(
|
|
||||||
(info.queue_family_index, i),
|
|
||||||
Queue::new(&device, info.queue_family_index, i),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect::<BTreeMap<_, _>>();
|
|
||||||
|
|
||||||
let get_queue = |(family, index)| {
|
|
||||||
allocated_queues
|
|
||||||
.get(&(family, index))
|
|
||||||
.cloned()
|
|
||||||
.unwrap_or_else(|| {
|
|
||||||
panic!(
|
|
||||||
"
|
|
||||||
queue family {family} index {index} failed to allocate"
|
|
||||||
)
|
|
||||||
})
|
|
||||||
};
|
|
||||||
|
|
||||||
let main_queue = get_queue(physical.queue_families.graphics);
|
|
||||||
let present_queue = get_queue(physical.queue_families.present);
|
|
||||||
let compute_queue = get_queue(physical.queue_families.async_compute);
|
|
||||||
let transfer_queue = get_queue(physical.queue_families.transfer);
|
|
||||||
|
|
||||||
let alloc_info =
|
|
||||||
vk_mem::AllocatorCreateInfo::new(&instance.raw, &device, physical.pdev);
|
|
||||||
|
|
||||||
let alloc = vk_mem::Allocator::new(alloc_info)?;
|
|
||||||
|
|
||||||
DeviceInner {
|
|
||||||
device: DeviceWrapper(device.clone()),
|
|
||||||
physical,
|
|
||||||
swapchain: khr::swapchain::Device::new(&instance.raw, &device),
|
|
||||||
debug_utils: ash::ext::debug_utils::Device::new(&instance.raw, &device),
|
|
||||||
instance,
|
|
||||||
alloc,
|
|
||||||
allocated_queues,
|
|
||||||
main_queue,
|
|
||||||
present_queue,
|
|
||||||
compute_queue,
|
|
||||||
transfer_queue,
|
|
||||||
features,
|
|
||||||
sync_threadpool: sync::SyncThreadpool::new(),
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(Self(Arc::new(device)))
|
|
||||||
}
|
|
||||||
pub fn sync_threadpool(&self) -> &sync::SyncThreadpool {
|
pub fn sync_threadpool(&self) -> &sync::SyncThreadpool {
|
||||||
&self.0.sync_threadpool
|
&self.0.sync_threadpool
|
||||||
}
|
}
|
||||||
|
|
@ -839,61 +538,52 @@ impl Device {
|
||||||
&self.0.alloc
|
&self.0.alloc
|
||||||
}
|
}
|
||||||
pub fn dev(&self) -> &ash::Device {
|
pub fn dev(&self) -> &ash::Device {
|
||||||
&self.0.device
|
&self.0.raw
|
||||||
}
|
}
|
||||||
pub fn instance(&self) -> &Arc<InstanceInner> {
|
pub fn instance(&self) -> &Arc<InstanceInner> {
|
||||||
&self.0.instance
|
&self.0.instance
|
||||||
}
|
}
|
||||||
pub fn swapchain(&self) -> &khr::swapchain::Device {
|
pub fn queues(&self) -> &DeviceQueues {
|
||||||
&self.0.swapchain
|
&self.0.queues
|
||||||
}
|
|
||||||
pub fn debug_utils(&self) -> &ash::ext::debug_utils::Device {
|
|
||||||
&self.0.debug_utils
|
|
||||||
}
|
|
||||||
pub fn queue_families(&self) -> &DeviceQueueFamilies {
|
|
||||||
&self.0.physical.queue_families
|
|
||||||
}
|
}
|
||||||
pub fn phy(&self) -> vk::PhysicalDevice {
|
pub fn phy(&self) -> vk::PhysicalDevice {
|
||||||
self.0.physical.pdev
|
self.0.adapter.pdev
|
||||||
}
|
}
|
||||||
pub fn features(&self) -> &crate::PhysicalDeviceFeatures {
|
pub fn features(&self) -> &crate::PhysicalDeviceFeatures {
|
||||||
&self.0.features
|
&self.0.adapter.features
|
||||||
}
|
}
|
||||||
pub fn properties(&self) -> &crate::PhysicalDeviceProperties {
|
pub fn properties(&self) -> &crate::PhysicalDeviceProperties {
|
||||||
&self.0.properties
|
&self.0.adapter.properties
|
||||||
}
|
}
|
||||||
pub fn physical_device(&self) -> &PhysicalDevice {
|
pub fn physical_device(&self) -> &PhysicalDeviceInfo {
|
||||||
&self.0.physical
|
&self.0.adapter
|
||||||
}
|
}
|
||||||
pub fn main_queue(&self) -> &Queue {
|
pub fn main_queue(&self) -> &Queue {
|
||||||
&self.0.main_queue
|
self.0.queues.graphics()
|
||||||
|
}
|
||||||
|
pub fn compute_queue(&self) -> &Queue {
|
||||||
|
self.0.queues.compute()
|
||||||
}
|
}
|
||||||
pub fn transfer_queue(&self) -> &Queue {
|
pub fn transfer_queue(&self) -> &Queue {
|
||||||
&self.0.transfer_queue
|
self.0.queues.transfer()
|
||||||
}
|
|
||||||
pub fn present_queue(&self) -> &Queue {
|
|
||||||
&self.0.present_queue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub unsafe fn lock_queues(&self) {
|
pub unsafe fn lock_queues(&self) {
|
||||||
// this is obviously awful, allocating for this
|
unsafe {
|
||||||
self.0
|
self.0.queues.lock();
|
||||||
.allocated_queues
|
}
|
||||||
.values()
|
|
||||||
.for_each(|q| core::mem::forget(q.lock()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub unsafe fn unlock_queues(&self) {
|
pub unsafe fn unlock_queues(&self) {
|
||||||
self.0
|
unsafe {
|
||||||
.allocated_queues
|
self.0.queues.unlock();
|
||||||
.values()
|
}
|
||||||
.for_each(|q| unsafe { q.0.force_unlock() });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn wait_queue_idle(&self, queue: &Queue) -> VkResult<()> {
|
pub fn wait_queue_idle(&self, queue: &Queue) -> VkResult<()> {
|
||||||
tracing::warn!("locking queue {queue:?} and waiting for idle");
|
tracing::warn!("locking queue {queue:?} and waiting for idle");
|
||||||
|
|
||||||
queue.with_locked(|q| unsafe { self.dev().queue_wait_idle(q) })?;
|
queue.with_locked(|q| unsafe { self.raw.queue_wait_idle(q.raw) })?;
|
||||||
|
|
||||||
tracing::warn!("finished waiting: unlocking queue {queue:?}.");
|
tracing::warn!("finished waiting: unlocking queue {queue:?}.");
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
@ -913,7 +603,9 @@ impl Device {
|
||||||
pub fn debug_name_object<T: vk::Handle>(&self, handle: T, name: &str) -> VkResult<()> {
|
pub fn debug_name_object<T: vk::Handle>(&self, handle: T, name: &str) -> VkResult<()> {
|
||||||
let name = std::ffi::CString::new(name.as_bytes()).unwrap_or(c"invalid name".to_owned());
|
let name = std::ffi::CString::new(name.as_bytes()).unwrap_or(c"invalid name".to_owned());
|
||||||
unsafe {
|
unsafe {
|
||||||
self.debug_utils().set_debug_utils_object_name(
|
self.device_extensions
|
||||||
|
.debug_utils
|
||||||
|
.set_debug_utils_object_name(
|
||||||
&vk::DebugUtilsObjectNameInfoEXT::default()
|
&vk::DebugUtilsObjectNameInfoEXT::default()
|
||||||
.object_handle(handle)
|
.object_handle(handle)
|
||||||
.object_name(&name),
|
.object_name(&name),
|
||||||
|
|
@ -923,38 +615,6 @@ impl Device {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AsRef<khr::swapchain::Device> for Device {
|
|
||||||
fn as_ref(&self) -> &khr::swapchain::Device {
|
|
||||||
&self.0.swapchain
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl AsRef<ash::Device> for Device {
|
|
||||||
fn as_ref(&self) -> &ash::Device {
|
|
||||||
&self.0.device
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub struct DeviceAndQueues {
|
|
||||||
pub(crate) device: Device,
|
|
||||||
pub(crate) main_queue: Queue,
|
|
||||||
pub(crate) compute_queue: Queue,
|
|
||||||
pub(crate) transfer_queue: Queue,
|
|
||||||
pub(crate) present_queue: Queue,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl AsRef<ash::Device> for DeviceAndQueues {
|
|
||||||
fn as_ref(&self) -> &ash::Device {
|
|
||||||
&self.device.as_ref()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
impl AsRef<ash::khr::swapchain::Device> for DeviceAndQueues {
|
|
||||||
fn as_ref(&self) -> &ash::khr::swapchain::Device {
|
|
||||||
&self.device.as_ref()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct DeviceOwnedDebugObject<T> {
|
pub struct DeviceOwnedDebugObject<T> {
|
||||||
pub(crate) device: Device,
|
pub(crate) device: Device,
|
||||||
|
|
@ -975,7 +635,7 @@ impl<T: std::fmt::Debug + vk::Handle + Copy> std::fmt::Debug for DeviceOwnedDebu
|
||||||
let mut fmt = f.debug_struct(core::any::type_name::<T>());
|
let mut fmt = f.debug_struct(core::any::type_name::<T>());
|
||||||
|
|
||||||
fmt.field_with("device", |f| {
|
fmt.field_with("device", |f| {
|
||||||
write!(f, "0x{:x}", self.device.0.device.handle().as_raw())
|
write!(f, "0x{:x}", self.device.raw.handle().as_raw())
|
||||||
})
|
})
|
||||||
.field_with("handle", |f| write!(f, "0x{:x}", &self.object.as_raw()));
|
.field_with("handle", |f| write!(f, "0x{:x}", &self.object.as_raw()));
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
use std::{
|
use std::{
|
||||||
cmp::Ordering,
|
cmp::Ordering,
|
||||||
ffi::{CStr, CString},
|
ffi::{CStr, CString},
|
||||||
|
ops::Deref,
|
||||||
sync::Arc,
|
sync::Arc,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -30,6 +31,14 @@ pub struct Instance {
|
||||||
pub(crate) inner: Arc<InstanceInner>,
|
pub(crate) inner: Arc<InstanceInner>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Deref for Instance {
|
||||||
|
type Target = InstanceInner;
|
||||||
|
|
||||||
|
fn deref(&self) -> &Self::Target {
|
||||||
|
&self.inner
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl core::fmt::Debug for Instance {
|
impl core::fmt::Debug for Instance {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
f.debug_struct("Instance")
|
f.debug_struct("Instance")
|
||||||
|
|
|
||||||
|
|
@ -158,7 +158,7 @@ fn compatible_extension_properties(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
mod queue;
|
pub mod queue;
|
||||||
|
|
||||||
// Queues must be externally synchronised for calls to `vkQueueSubmit` and `vkQueuePresentKHR`.
|
// Queues must be externally synchronised for calls to `vkQueueSubmit` and `vkQueuePresentKHR`.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
|
|
@ -224,6 +224,20 @@ pub struct PhysicalDeviceFeatures {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PhysicalDeviceFeatures {
|
impl PhysicalDeviceFeatures {
|
||||||
|
pub fn push_to_device_create_info<'a>(
|
||||||
|
&'a mut self,
|
||||||
|
mut create_info: vk::DeviceCreateInfo<'a>,
|
||||||
|
) -> vk::DeviceCreateInfo<'a> {
|
||||||
|
create_info = create_info
|
||||||
|
.push_next(&mut self.core11)
|
||||||
|
.push_next(&mut self.core12)
|
||||||
|
.push_next(&mut self.core13);
|
||||||
|
|
||||||
|
if let Some(mesh_shader) = &mut self.mesh_shader {
|
||||||
|
create_info = create_info.push_next(mesh_shader);
|
||||||
|
}
|
||||||
|
create_info
|
||||||
|
}
|
||||||
pub fn superset_of(&self, other: &PhysicalDeviceFeatures) -> bool {
|
pub fn superset_of(&self, other: &PhysicalDeviceFeatures) -> bool {
|
||||||
fn core_superset_of(
|
fn core_superset_of(
|
||||||
a: &vk::PhysicalDeviceFeatures,
|
a: &vk::PhysicalDeviceFeatures,
|
||||||
|
|
@ -629,6 +643,21 @@ fn get_physical_device_properties(
|
||||||
Ok(props)
|
Ok(props)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn extension_intersection<'a>(
|
||||||
|
supported: &'a [vk::ExtensionProperties],
|
||||||
|
required: &[Extension<'a>],
|
||||||
|
) -> Vec<vk::ExtensionProperties> {
|
||||||
|
supported
|
||||||
|
.iter()
|
||||||
|
.filter(|ext| {
|
||||||
|
required.iter().any(|req| {
|
||||||
|
ext.extension_name_as_c_str() == Ok(req.name) && ext.spec_version >= req.version
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.cloned()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
impl PhysicalDeviceProperties {
|
impl PhysicalDeviceProperties {
|
||||||
pub(crate) fn supports_extension(&self, e: Extension) -> bool {
|
pub(crate) fn supports_extension(&self, e: Extension) -> bool {
|
||||||
self.supported_extensions
|
self.supported_extensions
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,49 @@
|
||||||
use bitflags::bitflags;
|
use bitflags::bitflags;
|
||||||
|
use parking_lot::Mutex;
|
||||||
use raw_window_handle::RawDisplayHandle;
|
use raw_window_handle::RawDisplayHandle;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::{collections::HashMap, ops::Deref, sync::Arc};
|
||||||
|
|
||||||
use ash::vk;
|
use ash::vk;
|
||||||
|
|
||||||
use crate::{Instance, PhysicalDeviceInfo, Result, instance::InstanceInner};
|
use crate::{Instance, PhysicalDeviceInfo, Result};
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct Queue {
|
pub struct Queue {
|
||||||
inner: Arc<QueueInner>,
|
inner: Arc<QueueInner>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Deref for Queue {
|
||||||
|
type Target = QueueInner;
|
||||||
|
|
||||||
|
fn deref(&self) -> &Self::Target {
|
||||||
|
&self.inner
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Queue {
|
||||||
|
pub fn from_vk_queue(raw: vk::Queue, family: QueueFamily) -> Self {
|
||||||
|
Self {
|
||||||
|
inner: Arc::new(QueueInner {
|
||||||
|
raw,
|
||||||
|
family,
|
||||||
|
lock: Mutex::new(()),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn with_locked<F, R>(&self, f: F) -> R
|
||||||
|
where
|
||||||
|
F: FnOnce(&Queue) -> R,
|
||||||
|
{
|
||||||
|
let _lock = self.inner.lock.lock();
|
||||||
|
f(&self)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct QueueInner {
|
pub struct QueueInner {
|
||||||
queue: vk::Queue,
|
pub(crate) raw: vk::Queue,
|
||||||
family: QueueFamily,
|
pub(crate) family: QueueFamily,
|
||||||
lock: Mutex<()>,
|
pub(crate) lock: Mutex<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
|
@ -94,6 +122,34 @@ pub struct DeviceQueues {
|
||||||
transfer: Queue,
|
transfer: Queue,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl DeviceQueues {
|
||||||
|
pub fn graphics(&self) -> &Queue {
|
||||||
|
&self.graphics
|
||||||
|
}
|
||||||
|
pub fn compute(&self) -> &Queue {
|
||||||
|
&self.compute
|
||||||
|
}
|
||||||
|
pub fn transfer(&self) -> &Queue {
|
||||||
|
&self.transfer
|
||||||
|
}
|
||||||
|
|
||||||
|
pub unsafe fn lock(&self) {
|
||||||
|
core::mem::forget((
|
||||||
|
self.graphics.inner.lock.lock(),
|
||||||
|
self.compute.inner.lock.lock(),
|
||||||
|
self.transfer.inner.lock.lock(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub unsafe fn unlock(&self) {
|
||||||
|
unsafe {
|
||||||
|
self.graphics.inner.lock.force_unlock();
|
||||||
|
self.compute.inner.lock.force_unlock();
|
||||||
|
self.transfer.inner.lock.force_unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct DeviceQueueInfos {
|
pub struct DeviceQueueInfos {
|
||||||
graphics: QueueFamily,
|
graphics: QueueFamily,
|
||||||
|
|
@ -102,6 +158,81 @@ pub struct DeviceQueueInfos {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DeviceQueueInfos {
|
impl DeviceQueueInfos {
|
||||||
|
const PRIORITIES: [f32; 4] = [1.0, 1.0, 1.0, 1.0];
|
||||||
|
pub fn into_create_infos(&self) -> Vec<vk::DeviceQueueCreateInfo> {
|
||||||
|
let families = self.queue_family_indices();
|
||||||
|
|
||||||
|
let create_infos = families
|
||||||
|
.into_iter()
|
||||||
|
.map(|(index, count)| {
|
||||||
|
vk::DeviceQueueCreateInfo::default()
|
||||||
|
.queue_family_index(index)
|
||||||
|
.queue_priorities(&Self::PRIORITIES[..count as usize])
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
create_infos
|
||||||
|
}
|
||||||
|
|
||||||
|
fn queue_family_indices(&self) -> HashMap<u32, u32> {
|
||||||
|
let mut families = HashMap::new();
|
||||||
|
|
||||||
|
families.insert(self.graphics.index, 1);
|
||||||
|
self.compute.map(|compute| {
|
||||||
|
*families.entry(compute.index).or_insert(0) += 1;
|
||||||
|
});
|
||||||
|
self.transfer.map(|transfer| {
|
||||||
|
*families.entry(transfer.index).or_insert(0) += 1;
|
||||||
|
});
|
||||||
|
|
||||||
|
families
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn retrieve_queues(self, dev: &ash::Device) -> DeviceQueues {
|
||||||
|
let families = self.queue_family_indices();
|
||||||
|
|
||||||
|
let mut queues = families
|
||||||
|
.into_iter()
|
||||||
|
.map(|(queue_family_index, count)| {
|
||||||
|
let queues = (0..count)
|
||||||
|
.map(|queue_index| unsafe {
|
||||||
|
dev.get_device_queue(queue_family_index, queue_index)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
(queue_family_index, queues)
|
||||||
|
})
|
||||||
|
.collect::<HashMap<u32, Vec<vk::Queue>>>();
|
||||||
|
|
||||||
|
let graphics = Queue::from_vk_queue(
|
||||||
|
queues.get_mut(&self.graphics.index).unwrap().pop().unwrap(),
|
||||||
|
self.graphics,
|
||||||
|
);
|
||||||
|
let compute = self
|
||||||
|
.compute
|
||||||
|
.map(|compute| {
|
||||||
|
Queue::from_vk_queue(
|
||||||
|
queues.get_mut(&compute.index).unwrap().pop().unwrap(),
|
||||||
|
compute,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| graphics.clone());
|
||||||
|
let transfer = self
|
||||||
|
.transfer
|
||||||
|
.map(|transfer| {
|
||||||
|
Queue::from_vk_queue(
|
||||||
|
queues.get_mut(&transfer.index).unwrap().pop().unwrap(),
|
||||||
|
transfer,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| graphics.clone());
|
||||||
|
|
||||||
|
DeviceQueues {
|
||||||
|
graphics,
|
||||||
|
compute,
|
||||||
|
transfer,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn select_queue_families(
|
pub fn select_queue_families(
|
||||||
instance: &Instance,
|
instance: &Instance,
|
||||||
pdev: &PhysicalDeviceInfo,
|
pdev: &PhysicalDeviceInfo,
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue