async acquire image

crates/game/src/main.rs

@@ -14,7 +14,6 @@ use winit::{
 struct WindowState {
     last_resize_events: BTreeMap<WindowId, PhysicalSize<u32>>,
     window_attrs: WindowAttributes,
-    window: Option<Window>,
     windows: BTreeMap<WindowId, Window>,
     renderer: Renderer,
 }
@@ -22,7 +21,6 @@ struct WindowState {
 impl WindowState {
     fn new(window_title: String, display: DisplayHandle) -> WindowState {
         Self {
-            window: None,
             windows: BTreeMap::new(),
             last_resize_events: BTreeMap::new(),
             window_attrs: WindowAttributes::default()
@@ -95,7 +93,7 @@ impl WindowState {
 
 impl ApplicationHandler for WindowState {
     fn resumed(&mut self, event_loop: &winit::event_loop::ActiveEventLoop) {
-        tracing::debug!("winit::resumed");
+        tracing::info!("winit::resumed");
 
         self.create_window(event_loop);
     }
@@ -104,6 +102,7 @@ impl ApplicationHandler for WindowState {
         &mut self,
         _event_loop: &winit::event_loop::ActiveEventLoop,
     ) {
+        tracing::info!("winit::about_to_wait");
         for (&window, &resize) in self.last_resize_events.clone().iter() {
             self.handle_final_resize(window, resize);
         }
@@ -121,6 +120,7 @@ impl ApplicationHandler for WindowState {
                 self.handle_final_resize(window_id, resize);
             }
         }
+
         match event {
             winit::event::WindowEvent::Resized(physical_size) => {
                 _ = self.last_resize_events.insert(window_id, physical_size);
@@ -153,6 +153,30 @@ impl ApplicationHandler for WindowState {
             } => {
                 self.create_window(event_loop);
             }
+            winit::event::WindowEvent::KeyboardInput {
+                event:
+                    winit::event::KeyEvent {
+                        physical_key:
+                            winit::keyboard::PhysicalKey::Code(
+                                winit::keyboard::KeyCode::KeyQ,
+                            ),
+                        state: ElementState::Pressed,
+                        repeat: false,
+                        ..
+                    },
+                ..
+            } => {
+                tracing::info!(
+                    window = u64::from(window_id),
+                    "window close requested"
+                );
+
+                self.windows.remove(&window_id);
+                self.renderer.window_contexts.remove(&window_id);
+                if self.windows.is_empty() {
+                    event_loop.exit();
+                }
+            }
             winit::event::WindowEvent::KeyboardInput {
                 device_id,
                 event,
@@ -187,6 +211,7 @@ impl ApplicationHandler for WindowState {
 fn main() {
     tracing_subscriber::fmt().init();
     let ev = EventLoop::new().unwrap();
+    ev.set_control_flow(winit::event_loop::ControlFlow::Poll);
 
     let display = ev.display_handle().expect("display handle");
     let mut game = WindowState::new("Vidya".to_owned(), display);

crates/renderer/Cargo.toml

@@ -18,3 +18,6 @@ tracing-subscriber = "0.3.18"
 vk-mem = "0.4.0"
 vk-sync = "0.1.6"
 winit = "0.30.5"
+crossbeam = "0.8.4"
+parking_lot = "0.12.3"
+smol.workspace = true

crates/renderer/src/commands.rs

@@ -1,61 +1,10 @@
-use std::{future::Future, marker::PhantomData};
+use std::{future::Future, marker::PhantomData, sync::Arc};
 
-use super::{Device2 as Device, Queue};
+use crate::sync::{self, FenceFuture};
+
+use super::{Device, Queue};
 use ash::{prelude::*, vk};
 
-pub struct FenceFuture<'a> {
-    device: Device,
-    fence: vk::Fence,
-    _pd: PhantomData<&'a ()>,
-}
-
-impl Drop for FenceFuture<'_> {
-    fn drop(&mut self) {
-        unsafe { self.device.dev().destroy_fence(self.fence, None) };
-    }
-}
-
-impl FenceFuture<'_> {
-    /// Unsafe because `fence` must not be destroyed while this future is live.
-    pub unsafe fn from_fence(device: Device, fence: vk::Fence) -> Self {
-        Self {
-            device,
-            fence,
-            _pd: PhantomData,
-        }
-    }
-    pub fn block_until(&self) -> VkResult<()> {
-        unsafe {
-            self.device
-                .dev()
-                .wait_for_fences(&[self.fence], true, u64::MAX)
-        }
-    }
-}
-
-impl Future for FenceFuture<'_> {
-    type Output = ();
-
-    fn poll(
-        self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Self::Output> {
-        let signaled = unsafe {
-            self.device
-                .dev()
-                .get_fence_status(self.fence)
-                .unwrap_or(false)
-        };
-
-        if signaled {
-            std::task::Poll::Ready(())
-        } else {
-            tokio::task::spawn_blocking(move || {});
-            std::task::Poll::Pending
-        }
-    }
-}
-
 pub struct SingleUseCommandPool {
     device: Device,
     pool: vk::CommandPool,
@@ -128,12 +77,13 @@ impl SingleUseCommand {
         self.buffer
     }
 
-    fn submit_fence(
+    pub fn submit_fence(
         &self,
         queue: Queue,
         wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
         signal: Option<vk::Semaphore>,
-    ) -> VkResult<vk::Fence> {
+        fence: Option<vk::Fence>,
+    ) -> VkResult<()> {
         unsafe { self.device.dev().end_command_buffer(self.buffer)? };
 
         let buffers = [self.buffer];
@@ -154,16 +104,13 @@ impl SingleUseCommand {
                 .wait_dst_stage_mask(core::slice::from_ref(stage));
         }
 
-        let fence = {
+        let fence = fence.unwrap_or(vk::Fence::null());
-            let create_info = vk::FenceCreateInfo::default();
-            unsafe { self.device.dev().create_fence(&create_info, None)? }
-        };
-
         queue.with_locked(|queue| unsafe {
             self.device.dev().queue_submit(queue, &[submit_info], fence)
         })?;
+        tracing::info!("submitted queue {:?} and fence {:?}", queue, fence);
 
-        Ok(fence)
+        Ok(())
     }
 
     pub fn submit_async<'a>(
@@ -171,11 +118,12 @@ impl SingleUseCommand {
         queue: Queue,
         wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
         signal: Option<vk::Semaphore>,
+        fence: Arc<sync::Fence>,
     ) -> VkResult<FenceFuture<'a>> {
         let device = self.device.clone();
-        let fence = self.submit_fence(queue, wait, signal)?;
+        self.submit_fence(queue, wait, signal, Some(fence.fence()))?;
 
-        Ok(unsafe { FenceFuture::from_fence(device, fence) })
+        Ok(unsafe { FenceFuture::new(fence) })
     }
 
     pub fn submit_blocking(
@@ -184,13 +132,9 @@ impl SingleUseCommand {
         wait: Option<(vk::Semaphore, vk::PipelineStageFlags)>,
         signal: Option<vk::Semaphore>,
     ) -> VkResult<()> {
-        let device = self.device.clone();
+        let fence = Arc::new(sync::Fence::create(self.device.clone())?);
-        let fence = self.submit_fence(queue, wait, signal)?;
+        let future = self.submit_async(queue, wait, signal, fence)?;
-
+        future.block();
-        unsafe {
-            device.dev().wait_for_fences(&[fence], true, u64::MAX)?;
-            device.dev().destroy_fence(fence, None);
-        }
         Ok(())
     }
 }
@@ -200,6 +144,13 @@ mod tests {
     use super::*;
 
     async fn async_submit(cmd: SingleUseCommand, queue: Queue) {
-        cmd.submit_async(queue, None, None).unwrap().await;
+        cmd.submit_async(
+            queue,
+            None,
+            None,
+            Arc::new(sync::Fence::create(cmd.device.clone()).unwrap()),
+        )
+        .unwrap()
+        .await;
     }
 }

crates/renderer/src/images.rs

@@ -1,4 +1,4 @@
-use super::{Device2 as Device, Queue, VkAllocator};
+use super::{Device, Queue, VkAllocator};
 use ash::{prelude::*, vk};
 use vk_mem::Alloc;
 

crates/renderer/src/lib.rs

@@ -1,15 +1,20 @@
-#![feature(c_str_module, closure_lifetime_binder, let_chains)]
+#![feature(c_str_module, closure_lifetime_binder, let_chains, negative_impls)]
 #![allow(unused)]
 use std::{
     collections::{BTreeMap, BTreeSet, HashMap},
     ffi::{CStr, CString},
     fmt::Debug,
     marker::PhantomData,
-    sync::{atomic::AtomicU64, Arc, Mutex},
+    ops::Deref,
+    sync::{
+        atomic::{AtomicU32, AtomicU64},
+        Arc, Mutex,
+    },
 };
 
 use ash::{
     khr,
+    prelude::VkResult,
     vk::{self, Handle},
     Entry,
 };
@@ -27,6 +32,8 @@ type VkAllocator = Arc<vk_mem::Allocator>;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
+    #[error("Swapchain suboptimal.")]
+    SuboptimalSwapchain,
     #[error(transparent)]
     LoadingError(#[from] ash::LoadingError),
     #[error(transparent)]
@@ -118,7 +125,7 @@ trait ExtendsDeviceFeatures2Debug:
 {
 }
 trait ExtendsDeviceProperties2Debug:
-    vk::ExtendsPhysicalDeviceProperties2 + Debug + DynClone
+    vk::ExtendsPhysicalDeviceProperties2 + Debug + DynClone + Send + Sync
 {
 }
 
@@ -126,8 +133,9 @@ impl<T: vk::ExtendsPhysicalDeviceFeatures2 + Debug> ExtendsDeviceFeatures2Debug
     for T
 {
 }
-impl<T: vk::ExtendsPhysicalDeviceProperties2 + Debug + DynClone>
+impl<
-    ExtendsDeviceProperties2Debug for T
+        T: vk::ExtendsPhysicalDeviceProperties2 + Debug + DynClone + Send + Sync,
+    > ExtendsDeviceProperties2Debug for T
 {
 }
 
@@ -389,20 +397,6 @@ struct PhysicalDevice {
     properties: PhysicalDeviceProperties,
 }
 
-impl PhysicalDevice {
-    fn swapchain_family_indices(&self) -> ArrayVec<[u32; 2]> {
-        let mut indices = array_vec!([u32; 2] => self.queue_families.graphics);
-
-        if let Some(present) = self.queue_families.present
-            && present != self.queue_families.graphics
-        {
-            indices.push(present);
-        }
-
-        indices
-    }
-}
-
 struct Instance {
     entry: Entry,
     instance: ash::Instance,
@@ -441,7 +435,22 @@ struct DeviceQueueFamilies {
     transfer: Option<u32>,
 }
 
-struct Device {
+impl DeviceQueueFamilies {
+    fn swapchain_family_indices(&self) -> ArrayVec<[u32; 2]> {
+        let mut indices = array_vec!([u32; 2] => self.graphics);
+
+        if let Some(present) = self.present
+            && present != self.graphics
+        {
+            indices.push(present);
+        }
+
+        indices
+    }
+}
+
+struct DeviceInner {
+    instance: Arc<Instance>,
     physical: PhysicalDevice,
     device: ash::Device,
     swapchain: khr::swapchain::Device,
@@ -449,12 +458,23 @@ struct Device {
     compute_queue: Queue,
     transfer_queue: Queue,
     present_queue: Queue,
+    sync_threadpool: sync::SyncThreadpool,
 }
 
 #[derive(Clone)]
-struct Device2(Arc<Device>);
+pub struct Device(Arc<DeviceInner>);
+pub type WeakDevice = std::sync::Weak<DeviceInner>;
 
-impl Device2 {
+impl Device {
+    fn new(inner: DeviceInner) -> Self {
+        Self(Arc::new(inner))
+    }
+    fn sync_threadpool(&self) -> &sync::SyncThreadpool {
+        &self.0.sync_threadpool
+    }
+    fn weak(&self) -> WeakDevice {
+        Arc::downgrade(&self.0)
+    }
     fn dev(&self) -> &ash::Device {
         &self.0.device
     }
@@ -464,8 +484,8 @@ impl Device2 {
     fn queue_families(&self) -> &DeviceQueueFamilies {
         &self.0.physical.queue_families
     }
-    fn phy(&self) -> &vk::PhysicalDevice {
+    fn phy(&self) -> vk::PhysicalDevice {
-        &self.0.physical.pdev
+        self.0.physical.pdev
     }
     fn graphics_queue(&self) -> &Queue {
         &self.0.main_queue
@@ -475,19 +495,19 @@ impl Device2 {
     }
 }
 
-impl AsRef<khr::swapchain::Device> for Device2 {
+impl AsRef<khr::swapchain::Device> for Device {
     fn as_ref(&self) -> &khr::swapchain::Device {
         &self.0.swapchain
     }
 }
 
-impl AsRef<ash::Device> for Device2 {
+impl AsRef<ash::Device> for Device {
     fn as_ref(&self) -> &ash::Device {
         &self.0.device
     }
 }
 
-impl Drop for Device {
+impl Drop for DeviceInner {
     fn drop(&mut self) {
         unsafe {
             _ = self.device.device_wait_idle();
@@ -497,7 +517,7 @@ impl Drop for Device {
 }
 
 struct DeviceAndQueues {
-    device: Arc<Device>,
+    device: Device,
     main_queue: Queue,
     compute_queue: Queue,
     transfer_queue: Queue,
@@ -506,27 +526,67 @@ struct DeviceAndQueues {
 
 impl AsRef<ash::Device> for DeviceAndQueues {
     fn as_ref(&self) -> &ash::Device {
-        &self.device.device
+        &self.device.as_ref()
     }
 }
 impl AsRef<ash::khr::swapchain::Device> for DeviceAndQueues {
     fn as_ref(&self) -> &ash::khr::swapchain::Device {
-        &self.device.swapchain
+        &self.device.as_ref()
     }
 }
 
-struct Swapchain {
+struct RawSwapchain(vk::SwapchainKHR);
+
+impl !Sync for RawSwapchain {}
+
+#[derive(Debug)]
+struct SwapchainHandle(parking_lot::Mutex<vk::SwapchainKHR>);
+
+impl SwapchainHandle {
+    unsafe fn from_handle(swapchain: vk::SwapchainKHR) -> SwapchainHandle {
+        Self(parking_lot::Mutex::new(swapchain))
+    }
+
+    fn lock(
+        &self,
+    ) -> parking_lot::lock_api::MutexGuard<
+        '_,
+        parking_lot::RawMutex,
+        vk::SwapchainKHR,
+    > {
+        self.0.lock()
+    }
+
+    fn with_locked<T, F: FnOnce(vk::SwapchainKHR) -> T>(&self, f: F) -> T {
+        let lock = self.0.lock();
+        f(*lock)
+    }
+}
+
+pub struct Swapchain {
+    device: Device,
     instance: Arc<Instance>,
-    device: Arc<Device>,
     // has a strong ref to the surface because the surface may not outlive the swapchain
     surface: Arc<Surface>,
-    swapchain: vk::SwapchainKHR,
+    swapchain: SwapchainHandle,
     present_mode: vk::PresentModeKHR,
     color_space: vk::ColorSpaceKHR,
     format: vk::Format,
     images: Vec<vk::Image>,
     image_views: Vec<vk::ImageView>,
     extent: vk::Extent2D,
+    min_image_count: u32,
+
+    // sync objects:
+    // we need two semaphores per each image, one acquire-semaphore and one release-semaphore.
+    // semaphores must be unique to each frame and cannot be reused per swapchain.
+    acquire_semaphores: Vec<vk::Semaphore>,
+    release_semaphores: Vec<vk::Semaphore>,
+
+    // one fence per in-flight frame, to synchronize image acquisition
+    fences: Vec<Arc<sync::Fence>>,
+
+    current_frame: AtomicU32,
 }
 
 impl Drop for Swapchain {
@@ -534,11 +594,22 @@ impl Drop for Swapchain {
         unsafe {
             info!("dropping swapchain {:?}", self.swapchain);
             for view in &self.image_views {
-                self.device.device.destroy_image_view(*view, None);
+                self.device.dev().destroy_image_view(*view, None);
+            }
+
+            self.swapchain.with_locked(|swapchain| {
+                self.device.swapchain().destroy_swapchain(swapchain, None)
+            });
+
+            for &semaphore in self
+                .acquire_semaphores
+                .iter()
+                .chain(&self.release_semaphores)
+            {
+                unsafe {
+                    self.device.dev().destroy_semaphore(semaphore, None);
+                }
             }
-            self.device
-                .swapchain
-                .destroy_swapchain(self.swapchain, None);
         }
     }
 }
@@ -563,19 +634,35 @@ fn current_extent_or_clamped(
     }
 }
 
+struct SwapchainParams {
+    present_mode: vk::PresentModeKHR,
+    format: vk::Format,
+    color_space: vk::ColorSpaceKHR,
+    /// the number of images to request from the device
+    image_count: u32,
+    /// the minimum number of images the surface permits
+    min_image_count: u32,
+    extent: vk::Extent2D,
+}
+
+pub struct SwapchainFrame {
+    pub swapchain: Arc<Swapchain>,
+    pub index: u32,
+    pub image: vk::Image,
+    pub view: vk::ImageView,
+    pub acquire: vk::Semaphore,
+    pub release: vk::Semaphore,
+}
+
 impl Swapchain {
+    const PREFERRED_IMAGES_IN_FLIGHT: u32 = 3;
+
     fn get_swapchain_params_from_surface(
         instance: &Arc<Instance>,
         surface: vk::SurfaceKHR,
         pdev: vk::PhysicalDevice,
         requested_extent: vk::Extent2D,
-    ) -> Result<(
+    ) -> Result<SwapchainParams> {
-        vk::PresentModeKHR,
-        vk::Format,
-        vk::ColorSpaceKHR,
-        u32,
-        vk::Extent2D,
-    )> {
         let caps = unsafe {
             instance
                 .surface
@@ -611,52 +698,76 @@ impl Swapchain {
             .cloned()
             .expect("no surface format available!");
 
-        let image_count = 3u32.clamp(
+        // 0 here means no limit
-            caps.min_image_count,
+        let max_image_count = core::num::NonZero::new(caps.max_image_count)
-            if caps.max_image_count == 0 {
+            .map(|n| n.get())
-                u32::MAX
+            .unwrap_or(u32::MAX);
-            } else {
+
-                caps.max_image_count
+        // we want PREFERRED_IMAGES_IN_FLIGHT images acquired at the same time,
-            },
+        let image_count = (caps.min_image_count
-        );
+            + Self::PREFERRED_IMAGES_IN_FLIGHT)
+            .min(max_image_count);
 
         let extent = current_extent_or_clamped(&caps, requested_extent);
 
-        Ok((
+        Ok(SwapchainParams {
             present_mode,
-            format.format,
+            format: format.format,
-            format.color_space,
+            color_space: format.color_space,
             image_count,
             extent,
-        ))
+            min_image_count: caps.min_image_count,
+        })
     }
-    fn create_new(
+
+    pub fn new(
         instance: Arc<Instance>,
-        device: Arc<Device>,
+        device: Device,
         surface: Arc<Surface>,
         pdev: vk::PhysicalDevice,
         extent: vk::Extent2D,
     ) -> Result<Self> {
-        let (present_mode, format, color_space, image_count, extent) =
+        Self::create(instance, device, surface, pdev, extent, None)
-            Self::get_swapchain_params_from_surface(
+    }
-                &instance,
-                surface.surface,
-                pdev,
-                extent,
-            )?;
 
-        let (swapchain, images) = Self::create_vkswapchainkhr(
+    fn create(
-            &device,
+        instance: Arc<Instance>,
-            surface.surface,
+        device: Device,
-            &device.physical.swapchain_family_indices(),
+        surface: Arc<Surface>,
-            extent,
+        pdev: vk::PhysicalDevice,
-            None,
+        extent: vk::Extent2D,
+        old_swapchain: Option<&SwapchainHandle>,
+    ) -> Result<Self> {
+        let SwapchainParams {
             present_mode,
             format,
             color_space,
             image_count,
+            min_image_count,
+            extent,
+        } = Self::get_swapchain_params_from_surface(
+            &instance,
+            surface.surface,
+            pdev,
+            extent,
         )?;
 
+        let (swapchain, images) = {
+            let lock = old_swapchain.as_ref().map(|handle| handle.lock());
+
+            Self::create_vkswapchainkhr(
+                &device,
+                surface.surface,
+                &device.queue_families().swapchain_family_indices(),
+                extent,
+                lock.as_ref().map(|lock| **lock),
+                present_mode,
+                format,
+                color_space,
+                image_count,
+            )
+        }?;
+
         let image_views = images
             .iter()
             .map(|&image| {
@@ -674,10 +785,42 @@ impl Swapchain {
                             .layer_count(1),
                     );
 
-                unsafe { device.device.create_image_view(&info, None) }
+                unsafe { device.dev().create_image_view(&info, None) }
             })
             .collect::<core::result::Result<Vec<vk::ImageView>, _>>()?;
 
+        let num_images = images.len() as u32;
+        let inflight_frames = num_images - min_image_count;
+
+        let acquire_semaphores = {
+            (0..inflight_frames)
+                .map(|_| unsafe {
+                    device.dev().create_semaphore(
+                        &vk::SemaphoreCreateInfo::default(),
+                        None,
+                    )
+                })
+                .collect::<VkResult<Vec<_>>>()?
+        };
+        let release_semaphores = {
+            (0..inflight_frames)
+                .map(|_| unsafe {
+                    device.dev().create_semaphore(
+                        &vk::SemaphoreCreateInfo::default(),
+                        None,
+                    )
+                })
+                .collect::<VkResult<Vec<_>>>()?
+        };
+
+        let fences = {
+            (0..inflight_frames)
+                .map(|_| Ok(Arc::new(sync::Fence::create(device.clone())?)))
+                .collect::<VkResult<Vec<_>>>()?
+        };
+
+        tracing::info!("fences: {fences:?}");
+
         Ok(Self {
             instance,
             device,
@@ -688,68 +831,96 @@ impl Swapchain {
             format,
             images,
             image_views,
+            min_image_count,
             extent,
+            acquire_semaphores,
+            release_semaphores,
+            fences,
+            current_frame: AtomicU32::new(0),
         })
     }
 
+    pub fn max_in_flight_images(&self) -> u32 {
+        self.num_images() - self.min_image_count
+    }
+
+    pub fn num_images(&self) -> u32 {
+        self.images.len() as u32
+    }
+
     fn recreate(&self, extent: vk::Extent2D) -> Result<Self> {
-        let (present_mode, format, color_space, image_count, extent) =
+        Self::create(
-            Self::get_swapchain_params_from_surface(
+            self.instance.clone(),
-                &self.instance,
+            self.device.clone(),
-                self.surface.surface,
+            self.surface.clone(),
-                self.device.physical.pdev,
+            self.device.phy(),
-                extent,
-            )?;
-
-        let (swapchain, images) = Self::create_vkswapchainkhr(
-            &self.device,
-            self.surface.surface,
-            &self.device.physical.swapchain_family_indices(),
             extent,
-            Some(self.swapchain),
+            Some(&self.swapchain),
-            present_mode,
+        )
-            format,
+    }
-            color_space,
-            image_count,
-        )?;
 
-        let image_views = images
+    /// returns a future yielding the frame, and true if the swapchain is
-            .iter()
+    /// suboptimal and should be recreated.
-            .map(|&image| {
+    fn acquire_image(
-                let info = vk::ImageViewCreateInfo::default()
+        self: Arc<Self>,
-                    .image(image)
+    ) -> impl std::future::Future<Output = VkResult<(SwapchainFrame, bool)>>
-                    .view_type(vk::ImageViewType::TYPE_2D)
+    {
-                    .format(format)
+        let frame = self
-                    .components(vk::ComponentMapping::default())
+            .current_frame
-                    .subresource_range(
+            .fetch_update(
-                        vk::ImageSubresourceRange::default()
+                std::sync::atomic::Ordering::Release,
-                            .aspect_mask(vk::ImageAspectFlags::COLOR)
+                std::sync::atomic::Ordering::Relaxed,
-                            .base_mip_level(0)
+                |i| Some((i + 1) % self.max_in_flight_images()),
-                            .level_count(1)
+            )
-                            .base_array_layer(0)
+            .unwrap() as usize;
-                            .layer_count(1),
-                    );
 
-                unsafe { self.device.device.create_image_view(&info, None) }
+        tracing::info!(frame, "acquiring image for frame {frame}");
+
+        async move {
+            let fence = self.fences[frame].clone();
+            let acquire = self.acquire_semaphores[frame];
+            let release = self.release_semaphores[frame];
+
+            // spawn on threadpool because it might block.
+            let (idx, suboptimal) = smol::unblock({
+                let this = self.clone();
+                let fence = fence.clone();
+                move || unsafe {
+                    this.swapchain.with_locked(|swapchain| {
+                        this.device.swapchain().acquire_next_image(
+                            swapchain,
+                            u64::MAX,
+                            acquire,
+                            fence.fence(),
+                        )
+                    })
+                }
             })
-            .collect::<core::result::Result<Vec<vk::ImageView>, _>>()?;
+            .await?;
 
-        Ok(Self {
+            // wait for image to become available.
-            instance: self.instance.clone(),
+            sync::FenceFuture::new(fence.clone()).await;
-            device: self.device.clone(),
+
-            swapchain,
+            let idx = idx as usize;
-            surface: self.surface.clone(),
+            let image = self.images[idx];
-            present_mode,
+            let view = self.image_views[idx];
-            color_space,
+
-            format,
+            Ok((
-            images,
+                SwapchainFrame {
-            image_views,
+                    index: idx as u32,
-            extent,
+                    swapchain: self.clone(),
-        })
+                    image,
+                    view,
+                    acquire,
+                    release,
+                },
+                suboptimal,
+            ))
+        }
     }
 
     fn create_vkswapchainkhr(
-        device: &Arc<Device>,
+        device: &Device,
         surface: vk::SurfaceKHR,
         queue_families: &[u32],
         image_extent: vk::Extent2D,
@@ -758,7 +929,7 @@ impl Swapchain {
         image_format: vk::Format,
         image_color_space: vk::ColorSpaceKHR,
         image_count: u32,
-    ) -> Result<(vk::SwapchainKHR, Vec<vk::Image>)> {
+    ) -> Result<(SwapchainHandle, Vec<vk::Image>)> {
         let create_info = vk::SwapchainCreateInfoKHR::default()
             .surface(surface)
             .present_mode(present_mode)
@@ -784,10 +955,10 @@ impl Swapchain {
 
         let (swapchain, images) = unsafe {
             let swapchain =
-                device.swapchain.create_swapchain(&create_info, None)?;
+                device.swapchain().create_swapchain(&create_info, None)?;
-            let images = device.swapchain.get_swapchain_images(swapchain)?;
+            let images = device.swapchain().get_swapchain_images(swapchain)?;
 
-            (swapchain, images)
+            (SwapchainHandle::from_handle(swapchain), images)
         };
 
         Ok((swapchain, images))
@@ -829,7 +1000,7 @@ impl Drop for Surface {
 
 pub struct Vulkan {
     instance: Arc<Instance>,
-    device: Arc<Device>,
+    device: Device,
     alloc: VkAllocator,
 }
 
@@ -1017,12 +1188,12 @@ impl Vulkan {
 
         tracing::debug!("pdev: {pdev:?}");
         let device =
-            Arc::new(Self::create_device(&instance, pdev, &mut features)?);
+            Self::create_device(instance.clone(), pdev, &mut features)?;
 
         let alloc_info = vk_mem::AllocatorCreateInfo::new(
             &instance.instance,
-            &device.device,
+            device.dev(),
-            device.physical.pdev,
+            device.phy(),
         );
 
         let alloc = Arc::new(unsafe { vk_mem::Allocator::new(alloc_info)? });
@@ -1221,7 +1392,7 @@ impl Vulkan {
     }
 
     fn create_device(
-        instance: &Instance,
+        instance: Arc<Instance>,
         pdev: PhysicalDevice,
         features: &mut PhysicalDeviceFeatures,
     ) -> Result<Device> {
@@ -1305,18 +1476,20 @@ impl Vulkan {
                 .map(|(f, i)| Queue::new(&device, f, i))
                 .unwrap_or(compute_queue.clone());
 
-            Device {
+            Device::new(DeviceInner {
                 device: device.clone(),
                 physical: pdev,
                 swapchain: khr::swapchain::Device::new(
                     &instance.instance,
                     &device,
                 ),
+                instance,
                 main_queue,
                 present_queue,
                 compute_queue,
                 transfer_queue,
-            }
+                sync_threadpool: sync::SyncThreadpool::new(),
+            })
         };
 
         Ok(device)
@@ -1500,7 +1673,7 @@ pub struct WindowContext {
 impl WindowContext {
     fn new(
         instance: Arc<Instance>,
-        device: Arc<Device>,
+        device: Device,
         extent: vk::Extent2D,
         window: winit::raw_window_handle::RawWindowHandle,
         display: winit::raw_window_handle::RawDisplayHandle,
@@ -1508,11 +1681,11 @@ impl WindowContext {
         let surface =
             Arc::new(Surface::create(instance.clone(), display, window)?);
 
-        let swapchain = Arc::new(Swapchain::create_new(
+        let swapchain = Arc::new(Swapchain::new(
             instance,
             device.clone(),
             surface.clone(),
-            device.physical.pdev,
+            device.phy(),
             extent,
         )?);
 
@@ -1549,7 +1722,7 @@ impl Renderer {
     }
 
     pub fn debug_draw(&mut self) -> Result<()> {
-        let dev = Device2(self.vulkan.device.clone());
+        let dev = self.vulkan.device.clone();
 
         unsafe { dev.dev().device_wait_idle()? };
 
@@ -1565,20 +1738,15 @@ impl Renderer {
 
             let extent = ctx.current_swapchain.extent;
 
-            let ready_semaphore = sync::Semaphore::new(dev.clone())?;
+            let (frame, suboptimal) =
-            let (swapchain_index, suboptimal) = unsafe {
+                smol::block_on(ctx.current_swapchain.clone().acquire_image())?;
-                dev.swapchain().acquire_next_image(
-                    ctx.current_swapchain.swapchain,
-                    u64::MAX,
-                    ready_semaphore.semaphore(),
-                    vk::Fence::null(),
-                )?
-            };
-            if suboptimal {
-                continue;
-            }
 
-            let image = ctx.current_swapchain.images[swapchain_index as usize];
+            if suboptimal {
+                tracing::warn!(
+                    "swapchain ({:?}) is suboptimal!",
+                    ctx.current_swapchain.swapchain
+                );
+            }
 
             // let image = images::Image2D::new_exclusive(
             //     self.vulkan.alloc.clone(),
@@ -1604,7 +1772,7 @@ impl Renderer {
 
             unsafe {
                 let barriers = [images::image_barrier(
-                    image,
+                    frame.image,
                     vk::ImageAspectFlags::COLOR,
                     vk::PipelineStageFlags2::TRANSFER,
                     vk::AccessFlags2::empty(),
@@ -1622,7 +1790,7 @@ impl Renderer {
                 dev.dev().cmd_pipeline_barrier2(buffer, &dependency_info);
                 dev.dev().cmd_clear_color_image(
                     buffer,
-                    image,
+                    frame.image,
                     vk::ImageLayout::TRANSFER_DST_OPTIMAL,
                     &clear_values,
                     &[vk::ImageSubresourceRange::default()
@@ -1634,7 +1802,7 @@ impl Renderer {
                 );
 
                 let barriers = [images::image_barrier(
-                    image,
+                    frame.image,
                     vk::ImageAspectFlags::COLOR,
                     vk::PipelineStageFlags2::TRANSFER,
                     vk::AccessFlags2::TRANSFER_WRITE,
@@ -1651,52 +1819,52 @@ impl Renderer {
 
                 dev.dev().cmd_pipeline_barrier2(buffer, &dependency_info);
 
-                let signal_semaphore = sync::Semaphore::new(dev.clone())?;
-
                 let future = cmd.submit_async(
                     dev.graphics_queue().clone(),
-                    Some((
+                    Some((frame.acquire, vk::PipelineStageFlags::ALL_COMMANDS)),
-                        ready_semaphore.semaphore(),
+                    Some(frame.release),
-                        vk::PipelineStageFlags::ALL_COMMANDS,
+                    Arc::new(sync::Fence::create(dev.clone())?),
-                    )),
-                    Some(signal_semaphore.semaphore()),
                 )?;
 
-                let wait_semaphores = [signal_semaphore.semaphore()];
+                let present_id = {
-                let swapchains = [ctx.current_swapchain.swapchain];
+                    let swapchain = ctx.current_swapchain.swapchain.lock();
-                let indices = [swapchain_index];
+                    let queue = dev.present_queue().lock();
-                static PRESENT_ID: AtomicU64 = AtomicU64::new(1);
+
-                let mut present_ids = [PRESENT_ID
+                    let wait_semaphores = [frame.release];
-                    .fetch_add(1, std::sync::atomic::Ordering::Release)];
+                    let swapchains = [*swapchain];
-                let mut present_id =
+                    let indices = [frame.index];
-                    vk::PresentIdKHR::default().present_ids(&present_ids);
+                    static PRESENT_ID: AtomicU64 = AtomicU64::new(1);
-                let present_info = vk::PresentInfoKHR::default()
+                    let mut present_ids = [PRESENT_ID
-                    .image_indices(&indices)
+                        .fetch_add(1, std::sync::atomic::Ordering::Release)];
-                    .swapchains(&swapchains)
+                    let mut present_id =
-                    .wait_semaphores(&wait_semaphores)
+                        vk::PresentIdKHR::default().present_ids(&present_ids);
-                    .push_next(&mut present_id);
+                    let present_info = vk::PresentInfoKHR::default()
-                dev.present_queue().with_locked(|queue| {
+                        .image_indices(core::slice::from_ref(&frame.index))
-                    dev.swapchain().queue_present(queue, &present_info)
+                        .swapchains(core::slice::from_ref(&swapchain))
+                        .wait_semaphores(&wait_semaphores)
+                        .push_next(&mut present_id);
+                    dev.swapchain().queue_present(*queue, &present_info)?;
+
+                    present_ids[0]
+                };
+
+                future.block()?;
+                ctx.current_swapchain.swapchain.with_locked(|swapchain| {
+                    khr::present_wait::Device::new(
+                        &self.vulkan.instance.instance,
+                        self.vulkan.device.dev(),
+                    )
+                    .wait_for_present(
+                        swapchain,
+                        present_id,
+                        u64::MAX,
+                    )
                 })?;
 
-                future.block_until()?;
+                dev.dev().device_wait_idle();
-                khr::present_wait::Device::new(
-                    &self.vulkan.instance.instance,
-                    &self.vulkan.device.device,
-                )
-                .wait_for_present(
-                    ctx.current_swapchain.swapchain,
-                    present_ids[0],
-                    u64::MAX,
-                )?;
-                // dev.dev().device_wait_idle();
-                drop(ready_semaphore);
-                drop(signal_semaphore);
             }
         }
 
-        //unsafe {dev.dev().}
-
         Ok(())
     }
 

crates/renderer/src/sync.rs

@@ -1,11 +1,220 @@
-use super::Device2 as Device;
+use std::{
+    future::Future,
+    marker::PhantomData,
+    sync::{atomic::AtomicU32, Arc},
+    time::Duration,
+};
+
+use super::Device;
 use ash::{prelude::*, vk};
+use crossbeam::channel::{Receiver, Sender};
+
+type Message = (Arc<Fence>, std::task::Waker);
+
+pub struct SyncThreadpool {
+    channel: (Sender<Message>, Receiver<Message>),
+    timeout: u64,
+    thread_dies_after: Duration,
+    max_threads: u32,
+    num_threads: Arc<AtomicU32>,
+}
+
+impl SyncThreadpool {
+    pub fn new() -> SyncThreadpool {
+        Self::with_max_threads(512)
+    }
+
+    pub fn with_max_threads(max_threads: u32) -> SyncThreadpool {
+        Self {
+            // 0-capacity channel to ensure immediate consumption of fences
+            channel: crossbeam::channel::bounded(0),
+            max_threads,
+            num_threads: Arc::new(AtomicU32::new(0)),
+            timeout: u64::MAX,
+            thread_dies_after: Duration::from_secs(5),
+        }
+    }
+
+    fn try_spawn_thread(&self) -> Option<()> {
+        use std::sync::atomic::Ordering;
+        match self.num_threads.fetch_update(
+            Ordering::Release,
+            Ordering::Acquire,
+            |i| {
+                if i < self.max_threads {
+                    Some(i + 1)
+                } else {
+                    None
+                }
+            },
+        ) {
+            Ok(tid) => {
+                struct SyncThread {
+                    timeout: u64,
+                    thread_dies_after: Duration,
+                    num_threads: Arc<AtomicU32>,
+                    rx: Receiver<Message>,
+                }
+
+                impl SyncThread {
+                    fn run(self, barrier: Arc<std::sync::Barrier>) {
+                        tracing::info!("spawned new sync thread");
+                        barrier.wait();
+                        while let Ok((fence, waker)) =
+                            self.rx.recv_timeout(self.thread_dies_after)
+                        {
+                            tracing::info!(
+                                "received ({:?}, {:?})",
+                                fence,
+                                waker
+                            );
+                            loop {
+                                match fence.wait_on(Some(self.timeout)) {
+                                    Ok(_) => {
+                                        waker.wake();
+                                        break;
+                                    }
+                                    Err(vk::Result::TIMEOUT) => {}
+                                    Err(err) => {
+                                        tracing::error!(
+                        "failed to wait on fence in waiter thread: {err}"
+                    );
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                        self.num_threads.fetch_sub(1, Ordering::AcqRel);
+                    }
+                }
+
+                let thread = SyncThread {
+                    timeout: self.timeout,
+                    thread_dies_after: self.thread_dies_after,
+                    num_threads: self.num_threads.clone(),
+                    rx: self.channel.1.clone(),
+                };
+
+                let barrier = Arc::new(std::sync::Barrier::new(2));
+                std::thread::Builder::new()
+                    .name(format!("fence-waiter-{tid}"))
+                    .spawn({
+                        let barrier = barrier.clone();
+                        move || {
+                            thread.run(barrier);
+                        }
+                    });
+                barrier.wait();
+                Some(())
+            }
+            Err(_) => {
+                tracing::error!(
+                    "sync-threadpool exceeded local thread limit ({})",
+                    self.max_threads
+                );
+                None
+            }
+        }
+    }
+
+    fn spawn_waiter(&self, fence: Arc<Fence>, waker: std::task::Waker) {
+        use std::sync::atomic::Ordering;
+
+        let mut msg = (fence, waker);
+        while let Err(err) = self.channel.0.try_send(msg) {
+            match err {
+                crossbeam::channel::TrySendError::Full(msg2) => {
+                    msg = msg2;
+                    self.try_spawn_thread();
+                }
+                crossbeam::channel::TrySendError::Disconnected(_) => {
+                    //tracing::error!("sync-threadpool channel disconnected?");
+                    unreachable!()
+                }
+            }
+        }
+    }
+}
 
 pub struct Semaphore {
     device: Device,
     inner: vk::Semaphore,
 }
 
+pub struct Fence {
+    dev: Device,
+    fence: vk::Fence,
+}
+
+impl std::fmt::Debug for Fence {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Fence").field("fence", &self.fence).finish()
+    }
+}
+
+impl Drop for Fence {
+    fn drop(&mut self) {
+        unsafe {
+            self.dev.dev().destroy_fence(self.fence, None);
+        }
+    }
+}
+
+impl Fence {
+    unsafe fn new(dev: Device, fence: vk::Fence) -> Fence {
+        Self { dev, fence }
+    }
+    pub fn create(dev: Device) -> VkResult<Fence> {
+        unsafe {
+            Ok(Self::new(
+                dev.clone(),
+                dev.dev()
+                    .create_fence(&vk::FenceCreateInfo::default(), None)?,
+            ))
+        }
+    }
+    pub fn create_signaled(dev: Device) -> VkResult<Fence> {
+        unsafe {
+            Ok(Self::new(
+                dev.clone(),
+                dev.dev().create_fence(
+                    &vk::FenceCreateInfo::default()
+                        .flags(vk::FenceCreateFlags::SIGNALED),
+                    None,
+                )?,
+            ))
+        }
+    }
+    pub fn wait_on(&self, timeout: Option<u64>) -> Result<(), vk::Result> {
+        use core::slice::from_ref;
+        unsafe {
+            self.dev.dev().wait_for_fences(
+                from_ref(&self.fence),
+                true,
+                timeout.unwrap_or(u64::MAX),
+            )
+        }
+    }
+    pub fn fence(&self) -> vk::Fence {
+        self.fence
+    }
+    pub fn is_signaled(&self) -> bool {
+        unsafe { self.dev.dev().get_fence_status(self.fence).unwrap_or(false) }
+    }
+    pub fn reset(&self) -> Result<(), vk::Result> {
+        unsafe {
+            self.dev
+                .dev()
+                .reset_fences(core::slice::from_ref(&self.fence))
+        }
+    }
+}
+impl AsRef<vk::Fence> for Fence {
+    fn as_ref(&self) -> &vk::Fence {
+        todo!()
+    }
+}
+
 impl Semaphore {
     pub fn new(device: Device) -> VkResult<Self> {
         let mut type_info = vk::SemaphoreTypeCreateInfo::default()
@@ -40,3 +249,50 @@ impl Drop for Semaphore {
         }
     }
 }
+
+pub struct FenceFuture<'a> {
+    fence: Arc<Fence>,
+    // lifetime parameter to prevent release of resources while future is pendign
+    _pd: PhantomData<&'a ()>,
+}
+
+impl FenceFuture<'_> {
+    /// Unsafe because `fence` must not be destroyed while this future is live.
+    pub unsafe fn from_fence(device: Device, fence: vk::Fence) -> Self {
+        Self {
+            fence: Arc::new(Fence::new(device, fence)),
+            _pd: PhantomData,
+        }
+    }
+    pub fn new(fence: Arc<Fence>) -> Self {
+        Self {
+            fence,
+            _pd: PhantomData,
+        }
+    }
+    pub fn block(&self) -> VkResult<()> {
+        self.fence.wait_on(None)?;
+        self.fence.reset()
+    }
+}
+
+impl Future for FenceFuture<'_> {
+    type Output = ();
+
+    fn poll(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Self::Output> {
+        if self.fence.is_signaled() {
+            tracing::info!("fence ({:?}) is signaled", self.fence);
+            _ = self.fence.reset();
+            std::task::Poll::Ready(())
+        } else {
+            self.fence
+                .dev
+                .sync_threadpool()
+                .spawn_waiter(self.fence.clone(), cx.waker().clone());
+            std::task::Poll::Pending
+        }
+    }
+}