Skip to content

Commit

Permalink
vk: grow descriptor pools progressively instead of pre-allocating
Browse files Browse the repository at this point in the history
  • Loading branch information
kvark committed May 27, 2024
1 parent e35b2d4 commit 45bf7c4
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 46 deletions.
31 changes: 9 additions & 22 deletions blade-graphics/src/vulkan/command.rs
Original file line number Diff line number Diff line change
Expand Up @@ -219,19 +219,14 @@ impl super::CommandEncoder {

pub fn start(&mut self) {
self.buffers.rotate_left(1);
self.device
.reset_descriptor_pool(&mut self.buffers[0].descriptor_pool);

let vk_info = vk::CommandBufferBeginInfo {
flags: vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT,
..Default::default()
};
unsafe {
self.device
.core
.reset_descriptor_pool(
self.buffers[0].descriptor_pool,
vk::DescriptorPoolResetFlags::empty(),
)
.unwrap();
self.device
.core
.begin_command_buffer(self.buffers[0].raw, &vk_info)
Expand Down Expand Up @@ -357,7 +352,7 @@ impl super::CommandEncoder {
self.barrier();
self.mark("pass/compute");
super::ComputeCommandEncoder {
cmd_buf: self.buffers[0],
cmd_buf: self.buffers.first_mut().unwrap(),
device: &self.device,
update_data: &mut self.update_data,
}
Expand Down Expand Up @@ -407,7 +402,7 @@ impl super::CommandEncoder {
};
rendering_info.render_area = render_area;

let cmd_buf = self.buffers[0];
let cmd_buf = self.buffers.first_mut().unwrap();
unsafe {
self.device
.core
Expand Down Expand Up @@ -708,20 +703,12 @@ impl crate::traits::PipelineEncoder for super::PipelineEncoder<'_, '_> {
template_offsets: &dsl.template_offsets,
});

let descriptor_set_layouts = [dsl.raw];
let descriptor_set_info = vk::DescriptorSetAllocateInfo {
descriptor_pool: self.cmd_buf.descriptor_pool,
..Default::default()
}
.set_layouts(&descriptor_set_layouts);
let vk_set = self
.device
.allocate_descriptor_set(&mut self.cmd_buf.descriptor_pool, dsl);
unsafe {
let sets = self
.device
.core
.allocate_descriptor_sets(&descriptor_set_info)
.unwrap();
self.device.core.update_descriptor_set_with_template(
sets[0],
vk_set,
dsl.update_template,
self.update_data.as_ptr() as *const _,
);
Expand All @@ -730,7 +717,7 @@ impl crate::traits::PipelineEncoder for super::PipelineEncoder<'_, '_> {
self.bind_point,
self.layout.raw,
group,
&sets,
&[vk_set],
&[],
);
}
Expand Down
111 changes: 111 additions & 0 deletions blade-graphics/src/vulkan/descriptor.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
use ash::vk;

//TODO: replace by an abstraction in `gpu-descriptor`
// https://github.com/zakarumych/gpu-descriptor/issues/42
const COUNT_BASE: u32 = 16;

#[derive(Debug)]
pub struct DescriptorPool {
sub_pools: Vec<vk::DescriptorPool>,
}

impl super::Device {
fn create_descriptor_sub_pool(&self, max_sets: u32) -> vk::DescriptorPool {
log::info!("Creating a descriptor pool for at most {} sets", max_sets);
let mut descriptor_sizes = vec![
vk::DescriptorPoolSize {
ty: vk::DescriptorType::INLINE_UNIFORM_BLOCK_EXT,
descriptor_count: max_sets * crate::limits::PLAIN_DATA_SIZE,
},
vk::DescriptorPoolSize {
ty: vk::DescriptorType::STORAGE_BUFFER,
descriptor_count: max_sets,
},
vk::DescriptorPoolSize {
ty: vk::DescriptorType::SAMPLED_IMAGE,
descriptor_count: 2 * max_sets,
},
vk::DescriptorPoolSize {
ty: vk::DescriptorType::SAMPLER,
descriptor_count: max_sets,
},
vk::DescriptorPoolSize {
ty: vk::DescriptorType::STORAGE_IMAGE,
descriptor_count: max_sets,
},
];
if self.ray_tracing.is_some() {
descriptor_sizes.push(vk::DescriptorPoolSize {
ty: vk::DescriptorType::ACCELERATION_STRUCTURE_KHR,
descriptor_count: max_sets,
});
}

let mut inline_uniform_block_info = vk::DescriptorPoolInlineUniformBlockCreateInfoEXT {
max_inline_uniform_block_bindings: max_sets,
..Default::default()
};
let descriptor_pool_info = vk::DescriptorPoolCreateInfo::default()
.max_sets(max_sets)
.pool_sizes(&descriptor_sizes)
.push_next(&mut inline_uniform_block_info);
unsafe {
self.core
.create_descriptor_pool(&descriptor_pool_info, None)
.unwrap()
}
}

pub(super) fn create_descriptor_pool(&self) -> DescriptorPool {
let vk_pool = self.create_descriptor_sub_pool(COUNT_BASE);
DescriptorPool {
sub_pools: vec![vk_pool],
}
}

pub(super) fn destroy_descriptor_pool(&self, pool: &mut DescriptorPool) {
for sub_pool in pool.sub_pools.drain(..) {
unsafe { self.core.destroy_descriptor_pool(sub_pool, None) };
}
}

pub(super) fn allocate_descriptor_set(
&self,
pool: &mut DescriptorPool,
layout: &super::DescriptorSetLayout,
) -> vk::DescriptorSet {
let descriptor_set_layouts = [layout.raw];
let mut descriptor_set_info = vk::DescriptorSetAllocateInfo::default()
.descriptor_pool(pool.sub_pools[0])
.set_layouts(&descriptor_set_layouts);
let result = unsafe { self.core.allocate_descriptor_sets(&descriptor_set_info) };
match result {
Ok(vk_sets) => return vk_sets[0],
Err(vk::Result::ERROR_OUT_OF_POOL_MEMORY) => {}
Err(other) => panic!("Unexpected descriptor allocation error: {:?}", other),
};

let next_max_sets = COUNT_BASE.pow(pool.sub_pools.len() as u32 + 1);
let vk_pool = self.create_descriptor_sub_pool(next_max_sets);
// Always insert in front to avoid expecting any overflows down the road
pool.sub_pools.insert(0, vk_pool);

descriptor_set_info.descriptor_pool = vk_pool;
let vk_sets = unsafe {
self.core
.allocate_descriptor_sets(&descriptor_set_info)
.unwrap()
};
vk_sets[0]
}

pub(super) fn reset_descriptor_pool(&self, pool: &mut DescriptorPool) {
for &vk_pool in pool.sub_pools.iter() {
unsafe {
self.core
.reset_descriptor_pool(vk_pool, vk::DescriptorPoolResetFlags::empty())
.unwrap();
}
}
}
}
34 changes: 10 additions & 24 deletions blade-graphics/src/vulkan/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use ash::{khr, vk};
use std::{num::NonZeroU32, ptr, sync::Mutex};

mod command;
mod descriptor;
mod init;
mod pipeline;
mod resource;
Expand Down Expand Up @@ -208,10 +209,10 @@ pub struct RenderPipeline {
layout: PipelineLayout,
}

#[derive(Clone, Copy, Debug)]
#[derive(Debug)]
struct CommandBuffer {
raw: vk::CommandBuffer,
descriptor_pool: vk::DescriptorPool,
descriptor_pool: descriptor::DescriptorPool,
}

#[derive(Debug, PartialEq)]
Expand Down Expand Up @@ -244,17 +245,17 @@ pub struct AccelerationStructureCommandEncoder<'a> {
device: &'a Device,
}
pub struct ComputeCommandEncoder<'a> {
cmd_buf: CommandBuffer,
cmd_buf: &'a mut CommandBuffer,
device: &'a Device,
update_data: &'a mut Vec<u8>,
}
pub struct RenderCommandEncoder<'a> {
cmd_buf: CommandBuffer,
cmd_buf: &'a mut CommandBuffer,
device: &'a Device,
update_data: &'a mut Vec<u8>,
}
pub struct PipelineEncoder<'a, 'p> {
cmd_buf: CommandBuffer,
cmd_buf: &'a mut CommandBuffer,
layout: &'p PipelineLayout,
bind_point: vk::PipelineBindPoint,
device: &'a Device,
Expand Down Expand Up @@ -332,21 +333,7 @@ impl crate::traits::CommandDevice for Context {
if !desc.name.is_empty() {
self.set_object_name(raw, desc.name);
};
let mut inline_uniform_block_info =
vk::DescriptorPoolInlineUniformBlockCreateInfoEXT {
max_inline_uniform_block_bindings: ROUGH_SET_COUNT,
..Default::default()
};
let descriptor_pool_info = vk::DescriptorPoolCreateInfo::default()
.max_sets(ROUGH_SET_COUNT)
.pool_sizes(&descriptor_sizes)
.push_next(&mut inline_uniform_block_info);
let descriptor_pool = unsafe {
self.device
.core
.create_descriptor_pool(&descriptor_pool_info, None)
.unwrap()
};
let descriptor_pool = self.device.create_descriptor_pool();
CommandBuffer {
raw,
descriptor_pool,
Expand Down Expand Up @@ -380,16 +367,15 @@ impl crate::traits::CommandDevice for Context {
}

fn destroy_command_encoder(&self, command_encoder: &mut CommandEncoder) {
for cmd_buf in command_encoder.buffers.iter() {
for cmd_buf in command_encoder.buffers.iter_mut() {
let raw_cmd_buffers = [cmd_buf.raw];
unsafe {
self.device
.core
.free_command_buffers(command_encoder.pool, &raw_cmd_buffers);
self.device
.core
.destroy_descriptor_pool(cmd_buf.descriptor_pool, None);
}
self.device
.destroy_descriptor_pool(&mut cmd_buf.descriptor_pool);
}
unsafe {
self.device
Expand Down

0 comments on commit 45bf7c4

Please sign in to comment.