Revision control
Copy as Markdown
Other Tools
use {↩
alloc::{collections::VecDeque, vec::Vec},↩
core::{↩
convert::TryFrom as _,↩
fmt::{self, Debug, Display},↩
},↩
gpu_descriptor_types::{↩
CreatePoolError, DescriptorDevice, DescriptorPoolCreateFlags, DescriptorTotalCount,↩
DeviceAllocationError,↩
},↩
hashbrown::HashMap,↩
};↩
↩
bitflags::bitflags! {↩
/// Flags to augment descriptor set allocation.↩
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]↩
pub struct DescriptorSetLayoutCreateFlags: u32 {↩
/// Specified that descriptor set must be allocated from\↩
/// pool with `DescriptorPoolCreateFlags::UPDATE_AFTER_BIND`.↩
///↩
/// This flag must be specified when and only when layout was created with matching backend-specific flag,↩
/// that allows layout to have UpdateAfterBind bindings.↩
const UPDATE_AFTER_BIND = 0x2;↩
}↩
}↩
↩
/// Descriptor set from allocator.↩
#[derive(Debug)]↩
pub struct DescriptorSet<S> {↩
raw: S,↩
pool_id: u64,↩
size: DescriptorTotalCount,↩
update_after_bind: bool,↩
}↩
↩
impl<S> DescriptorSet<S> {↩
/// Returns reference to raw descriptor set.↩
pub fn raw(&self) -> &S {↩
&self.raw↩
}↩
↩
/// Returns mutable reference to raw descriptor set.↩
///↩
/// # Safety↩
///↩
/// Object must not be replaced.↩
pub unsafe fn raw_mut(&mut self) -> &mut S {↩
&mut self.raw↩
}↩
}↩
↩
/// AllocationError that may occur during descriptor sets allocation.↩
#[derive(Debug)]↩
pub enum AllocationError {↩
/// Backend reported that device memory has been exhausted.\↩
/// Deallocating device memory or other resources may increase chance↩
/// that another allocation would succeed.↩
OutOfDeviceMemory,↩
↩
/// Backend reported that host memory has been exhausted.\↩
/// Deallocating host memory may increase chance that another allocation would succeed.↩
OutOfHostMemory,↩
↩
/// The total number of descriptors across all pools created\↩
/// with flag `CREATE_UPDATE_AFTER_BIND_BIT` set exceeds `max_update_after_bind_descriptors_in_all_pools`↩
/// Or fragmentation of the underlying hardware resources occurs.↩
Fragmentation,↩
}↩
↩
impl Display for AllocationError {↩
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {↩
match self {↩
AllocationError::OutOfDeviceMemory => fmt.write_str("Device memory exhausted"),↩
AllocationError::OutOfHostMemory => fmt.write_str("Host memory exhausted"),↩
AllocationError::Fragmentation => fmt.write_str("Fragmentation"),↩
}↩
}↩
}↩
↩
#[cfg(feature = "std")]↩
impl std::error::Error for AllocationError {}↩
↩
impl From<CreatePoolError> for AllocationError {↩
fn from(err: CreatePoolError) -> Self {↩
match err {↩
CreatePoolError::OutOfDeviceMemory => AllocationError::OutOfDeviceMemory,↩
CreatePoolError::OutOfHostMemory => AllocationError::OutOfHostMemory,↩
CreatePoolError::Fragmentation => AllocationError::Fragmentation,↩
}↩
}↩
}↩
↩
const MIN_SETS: u32 = 64;↩
const MAX_SETS: u32 = 512;↩
↩
#[derive(Debug)]↩
struct DescriptorPool<P> {↩
raw: P,↩
↩
/// Number of sets allocated from pool.↩
allocated: u32,↩
↩
/// Expected number of sets available.↩
available: u32,↩
}↩
↩
#[derive(Debug)]↩
struct DescriptorBucket<P> {↩
offset: u64,↩
pools: VecDeque<DescriptorPool<P>>,↩
total: u32,↩
update_after_bind: bool,↩
size: DescriptorTotalCount,↩
}↩
↩
impl<P> Drop for DescriptorBucket<P> {↩
#[cfg(feature = "tracing")]↩
fn drop(&mut self) {↩
#[cfg(feature = "std")]↩
{↩
if std::thread::panicking() {↩
return;↩
}↩
}↩
if self.total > 0 {↩
tracing::error!("Descriptor sets were not deallocated");↩
}↩
}↩
↩
#[cfg(all(not(feature = "tracing"), feature = "std"))]↩
fn drop(&mut self) {↩
if std::thread::panicking() {↩
return;↩
}↩
if self.total > 0 {↩
eprintln!("Descriptor sets were not deallocated")↩
}↩
}↩
↩
#[cfg(all(not(feature = "tracing"), not(feature = "std")))]↩
fn drop(&mut self) {↩
if self.total > 0 {↩
panic!("Descriptor sets were not deallocated")↩
}↩
}↩
}↩
↩
impl<P> DescriptorBucket<P> {↩
fn new(update_after_bind: bool, size: DescriptorTotalCount) -> Self {↩
DescriptorBucket {↩
offset: 0,↩
pools: VecDeque::new(),↩
total: 0,↩
update_after_bind,↩
size,↩
}↩
}↩
↩
fn new_pool_size(&self, minimal_set_count: u32) -> (DescriptorTotalCount, u32) {↩
let mut max_sets = MIN_SETS // at least MIN_SETS↩
.max(minimal_set_count) // at least enough for allocation↩
.max(self.total.min(MAX_SETS)) // at least as much as was allocated so far capped to MAX_SETS↩
.checked_next_power_of_two() // rounded up to nearest 2^N↩
.unwrap_or(i32::MAX as u32);↩
↩
max_sets = (u32::MAX / self.size.sampler.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.combined_image_sampler.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.sampled_image.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.storage_image.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.uniform_texel_buffer.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.storage_texel_buffer.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.uniform_buffer.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.storage_buffer.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.uniform_buffer_dynamic.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.storage_buffer_dynamic.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.input_attachment.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.acceleration_structure.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.inline_uniform_block_bytes.max(1)).min(max_sets);↩
max_sets = (u32::MAX / self.size.inline_uniform_block_bindings.max(1)).min(max_sets);↩
↩
let mut pool_size = DescriptorTotalCount {↩
sampler: self.size.sampler * max_sets,↩
combined_image_sampler: self.size.combined_image_sampler * max_sets,↩
sampled_image: self.size.sampled_image * max_sets,↩
storage_image: self.size.storage_image * max_sets,↩
uniform_texel_buffer: self.size.uniform_texel_buffer * max_sets,↩
storage_texel_buffer: self.size.storage_texel_buffer * max_sets,↩
uniform_buffer: self.size.uniform_buffer * max_sets,↩
storage_buffer: self.size.storage_buffer * max_sets,↩
uniform_buffer_dynamic: self.size.uniform_buffer_dynamic * max_sets,↩
storage_buffer_dynamic: self.size.storage_buffer_dynamic * max_sets,↩
input_attachment: self.size.input_attachment * max_sets,↩
acceleration_structure: self.size.acceleration_structure * max_sets,↩
inline_uniform_block_bytes: self.size.inline_uniform_block_bytes * max_sets,↩
inline_uniform_block_bindings: self.size.inline_uniform_block_bindings * max_sets,↩
};↩
↩
if pool_size == Default::default() {↩
pool_size.sampler = 1;↩
}↩
↩
(pool_size, max_sets)↩
}↩
↩
unsafe fn allocate<L, S>(↩
&mut self,↩
device: &impl DescriptorDevice<L, P, S>,↩
layout: &L,↩
mut count: u32,↩
allocated_sets: &mut Vec<DescriptorSet<S>>,↩
) -> Result<(), AllocationError> {↩
debug_assert!(usize::try_from(count).is_ok(), "Must be ensured by caller");↩
↩
if count == 0 {↩
return Ok(());↩
}↩
↩
for (index, pool) in self.pools.iter_mut().enumerate().rev() {↩
if pool.available == 0 {↩
continue;↩
}↩
↩
let allocate = pool.available.min(count);↩
↩
#[cfg(feature = "tracing")]↩
tracing::trace!("Allocate `{}` sets from exising pool", allocate);↩
↩
let result = device.alloc_descriptor_sets(↩
&mut pool.raw,↩
(0..allocate).map(|_| layout),↩
&mut Allocation {↩
size: self.size,↩
update_after_bind: self.update_after_bind,↩
pool_id: index as u64 + self.offset,↩
sets: allocated_sets,↩
},↩
);↩
↩
match result {↩
Ok(()) => {}↩
Err(DeviceAllocationError::OutOfDeviceMemory) => {↩
return Err(AllocationError::OutOfDeviceMemory)↩
}↩
Err(DeviceAllocationError::OutOfHostMemory) => {↩
return Err(AllocationError::OutOfHostMemory)↩
}↩
Err(DeviceAllocationError::FragmentedPool) => {↩
// Should not happen, but better this than panicing.↩
#[cfg(feature = "tracing")]↩
tracing::error!("Unexpectedly failed to allocated descriptor sets due to pool fragmentation");↩
pool.available = 0;↩
continue;↩
}↩
Err(DeviceAllocationError::OutOfPoolMemory) => {↩
pool.available = 0;↩
continue;↩
}↩
}↩
↩
count -= allocate;↩
pool.available -= allocate;↩
pool.allocated += allocate;↩
self.total += allocate;↩
↩
if count == 0 {↩
return Ok(());↩
}↩
}↩
↩
while count > 0 {↩
let (pool_size, max_sets) = self.new_pool_size(count);↩
#[cfg(feature = "tracing")]↩
tracing::trace!(↩
"Create new pool with {} sets and {:?} descriptors",↩
max_sets,↩
pool_size,↩
);↩
↩
let mut raw = device.create_descriptor_pool(↩
&pool_size,↩
max_sets,↩
if self.update_after_bind {↩
DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET↩
| DescriptorPoolCreateFlags::UPDATE_AFTER_BIND↩
} else {↩
DescriptorPoolCreateFlags::FREE_DESCRIPTOR_SET↩
},↩
)?;↩
↩
let pool_id = self.pools.len() as u64 + self.offset;↩
↩
let allocate = max_sets.min(count);↩
let result = device.alloc_descriptor_sets(↩
&mut raw,↩
(0..allocate).map(|_| layout),↩
&mut Allocation {↩
pool_id,↩
size: self.size,↩
update_after_bind: self.update_after_bind,↩
sets: allocated_sets,↩
},↩
);↩
↩
match result {↩
Ok(()) => {}↩
Err(err) => {↩
device.destroy_descriptor_pool(raw);↩
match err {↩
DeviceAllocationError::OutOfDeviceMemory => {↩
return Err(AllocationError::OutOfDeviceMemory)↩
}↩
DeviceAllocationError::OutOfHostMemory => {↩
return Err(AllocationError::OutOfHostMemory)↩
}↩
DeviceAllocationError::FragmentedPool => {↩
// Should not happen, but better this than panicing.↩
#[cfg(feature = "trace")]↩
trace::error!("Unexpectedly failed to allocated descriptor sets due to pool fragmentation");↩
}↩
DeviceAllocationError::OutOfPoolMemory => {}↩
}↩
panic!("Failed to allocate descriptor sets from fresh pool");↩
}↩
}↩
↩
count -= allocate;↩
self.pools.push_back(DescriptorPool {↩
raw,↩
allocated: allocate,↩
available: max_sets - allocate,↩
});↩
self.total += allocate;↩
}↩
↩
Ok(())↩
}↩
↩
unsafe fn free<L, S>(↩
&mut self,↩
device: &impl DescriptorDevice<L, P, S>,↩
raw_sets: impl IntoIterator<Item = S>,↩
pool_id: u64,↩
) {↩
let pool = usize::try_from(pool_id - self.offset)↩
.ok()↩
.and_then(|index| self.pools.get_mut(index))↩
.expect("Invalid pool id");↩
↩
let mut raw_sets = raw_sets.into_iter();↩
let mut count = 0;↩
device.dealloc_descriptor_sets(&mut pool.raw, raw_sets.by_ref().inspect(|_| count += 1));↩
↩
debug_assert!(↩
raw_sets.next().is_none(),↩
"Device must deallocated all sets from iterator"↩
);↩
↩
pool.available += count;↩
pool.allocated -= count;↩
self.total -= count;↩
#[cfg(feature = "tracing")]↩
tracing::trace!("Freed {} from descriptor bucket", count);↩
↩
while let Some(pool) = self.pools.pop_front() {↩
if self.pools.is_empty() || pool.allocated != 0 {↩
self.pools.push_front(pool);↩
break;↩
}↩
↩
#[cfg(feature = "tracing")]↩
tracing::trace!("Destroying old descriptor pool");↩
↩
device.destroy_descriptor_pool(pool.raw);↩
self.offset += 1;↩
}↩
}↩
↩
unsafe fn cleanup<L, S>(&mut self, device: &impl DescriptorDevice<L, P, S>) {↩
while let Some(pool) = self.pools.pop_front() {↩
if pool.allocated != 0 {↩
self.pools.push_front(pool);↩
break;↩
}↩
↩
#[cfg(feature = "tracing")]↩
tracing::trace!("Destroying old descriptor pool");↩
↩
device.destroy_descriptor_pool(pool.raw);↩
self.offset += 1;↩
}↩
}↩
}↩
↩
/// Descriptor allocator.↩
/// Can be used to allocate descriptor sets for any layout.↩
#[derive(Debug)]↩
pub struct DescriptorAllocator<P, S> {↩
buckets: HashMap<(DescriptorTotalCount, bool), DescriptorBucket<P>>,↩
sets_cache: Vec<DescriptorSet<S>>,↩
raw_sets_cache: Vec<S>,↩
max_update_after_bind_descriptors_in_all_pools: u32,↩
current_update_after_bind_descriptors_in_all_pools: u32,↩
total: u32,↩
}↩
↩
impl<P, S> Drop for DescriptorAllocator<P, S> {↩
fn drop(&mut self) {↩
if self.buckets.drain().any(|(_, bucket)| bucket.total != 0) {↩
#[cfg(feature = "tracing")]↩
tracing::error!(↩
"`DescriptorAllocator` is dropped while some descriptor sets were not deallocated"↩
);↩
}↩
}↩
}↩
↩
impl<P, S> DescriptorAllocator<P, S> {↩
/// Create new allocator instance.↩
pub fn new(max_update_after_bind_descriptors_in_all_pools: u32) -> Self {↩
DescriptorAllocator {↩
buckets: HashMap::default(),↩
total: 0,↩
sets_cache: Vec::new(),↩
raw_sets_cache: Vec::new(),↩
max_update_after_bind_descriptors_in_all_pools,↩
current_update_after_bind_descriptors_in_all_pools: 0,↩
}↩
}↩
↩
/// Allocate descriptor set with specified layout.↩
///↩
/// # Safety↩
///↩
/// * Same `device` instance must be passed to all method calls of↩
/// one `DescriptorAllocator` instance.↩
/// * `flags` must match flags that were used to create the layout.↩
/// * `layout_descriptor_count` must match descriptor numbers in the layout.↩
pub unsafe fn allocate<L, D>(↩
&mut self,↩
device: &D,↩
layout: &L,↩
flags: DescriptorSetLayoutCreateFlags,↩
layout_descriptor_count: &DescriptorTotalCount,↩
count: u32,↩
) -> Result<Vec<DescriptorSet<S>>, AllocationError>↩
where↩
S: Debug,↩
L: Debug,↩
D: DescriptorDevice<L, P, S>,↩
{↩
if count == 0 {↩
return Ok(Vec::new());↩
}↩
↩
let descriptor_count = count * layout_descriptor_count.total();↩
↩
let update_after_bind = flags.contains(DescriptorSetLayoutCreateFlags::UPDATE_AFTER_BIND);↩
↩
if update_after_bind↩
&& self.max_update_after_bind_descriptors_in_all_pools↩
- self.current_update_after_bind_descriptors_in_all_pools↩
< descriptor_count↩
{↩
return Err(AllocationError::Fragmentation);↩
}↩
↩
#[cfg(feature = "tracing")]↩
tracing::trace!(↩
"Allocating {} sets with layout {:?} @ {:?}",↩
count,↩
layout,↩
layout_descriptor_count↩
);↩
↩
let bucket = self↩
.buckets↩
.entry((*layout_descriptor_count, update_after_bind))↩
.or_insert_with(|| DescriptorBucket::new(update_after_bind, *layout_descriptor_count));↩
match bucket.allocate(device, layout, count, &mut self.sets_cache) {↩
Ok(()) => {↩
self.total += descriptor_count;↩
if update_after_bind {↩
self.current_update_after_bind_descriptors_in_all_pools += descriptor_count;↩
}↩
↩
Ok(core::mem::take(&mut self.sets_cache))↩
}↩
Err(err) => {↩
debug_assert!(self.raw_sets_cache.is_empty());↩
↩
// Free sets allocated so far.↩
let mut last = None;↩
↩
for set in self.sets_cache.drain(..) {↩
if Some(set.pool_id) != last {↩
if let Some(last_id) = last {↩
// Free contiguous range of sets from one pool in one go.↩
bucket.free(device, self.raw_sets_cache.drain(..), last_id);↩
}↩
}↩
last = Some(set.pool_id);↩
self.raw_sets_cache.push(set.raw);↩
}↩
↩
if let Some(last_id) = last {↩
bucket.free(device, self.raw_sets_cache.drain(..), last_id);↩
}↩
↩
Err(err)↩
}↩
}↩
}↩
↩
/// Free descriptor sets.↩
///↩
/// # Safety↩
///↩
/// * Same `device` instance must be passed to all method calls of↩
/// one `DescriptorAllocator` instance.↩
/// * None of descriptor sets can be referenced in any pending command buffers.↩
/// * All command buffers where at least one of descriptor sets referenced↩
/// move to invalid state.↩
pub unsafe fn free<L, D, I>(&mut self, device: &D, sets: I)↩
where↩
D: DescriptorDevice<L, P, S>,↩
I: IntoIterator<Item = DescriptorSet<S>>,↩
{↩
debug_assert!(self.raw_sets_cache.is_empty());↩
↩
let mut last_key = (EMPTY_COUNT, false);↩
let mut last_pool_id = None;↩
↩
for set in sets {↩
if last_key != (set.size, set.update_after_bind) || last_pool_id != Some(set.pool_id) {↩
if let Some(pool_id) = last_pool_id {↩
let bucket = self↩
.buckets↩
.get_mut(&last_key)↩
.expect("Set must be allocated from this allocator");↩
↩
debug_assert!(u32::try_from(self.raw_sets_cache.len())↩
.ok()↩
.map_or(false, |count| count <= bucket.total));↩
↩
bucket.free(device, self.raw_sets_cache.drain(..), pool_id);↩
}↩
last_key = (set.size, set.update_after_bind);↩
last_pool_id = Some(set.pool_id);↩
}↩
self.raw_sets_cache.push(set.raw);↩
}↩
↩
if let Some(pool_id) = last_pool_id {↩
let bucket = self↩
.buckets↩
.get_mut(&last_key)↩
.expect("Set must be allocated from this allocator");↩
↩
debug_assert!(u32::try_from(self.raw_sets_cache.len())↩
.ok()↩
.map_or(false, |count| count <= bucket.total));↩
↩
bucket.free(device, self.raw_sets_cache.drain(..), pool_id);↩
}↩
}↩
↩
/// Perform cleanup to allow resources reuse.↩
///↩
/// # Safety↩
///↩
/// * Same `device` instance must be passed to all method calls of↩
/// one `DescriptorAllocator` instance.↩
pub unsafe fn cleanup<L>(&mut self, device: &impl DescriptorDevice<L, P, S>) {↩
for bucket in self.buckets.values_mut() {↩
bucket.cleanup(device)↩
}↩
self.buckets.retain(|_, bucket| !bucket.pools.is_empty());↩
}↩
}↩
↩
/// Empty descriptor per_type.↩
const EMPTY_COUNT: DescriptorTotalCount = DescriptorTotalCount {↩
sampler: 0,↩
combined_image_sampler: 0,↩
sampled_image: 0,↩
storage_image: 0,↩
uniform_texel_buffer: 0,↩
storage_texel_buffer: 0,↩
uniform_buffer: 0,↩
storage_buffer: 0,↩
uniform_buffer_dynamic: 0,↩
storage_buffer_dynamic: 0,↩
input_attachment: 0,↩
acceleration_structure: 0,↩
inline_uniform_block_bytes: 0,↩
inline_uniform_block_bindings: 0,↩
};↩
↩
struct Allocation<'a, S> {↩
update_after_bind: bool,↩
size: DescriptorTotalCount,↩
pool_id: u64,↩
sets: &'a mut Vec<DescriptorSet<S>>,↩
}↩
↩
impl<S> Extend<S> for Allocation<'_, S> {↩
fn extend<T: IntoIterator<Item = S>>(&mut self, iter: T) {↩
let update_after_bind = self.update_after_bind;↩
let size = self.size;↩
let pool_id = self.pool_id;↩
self.sets.extend(iter.into_iter().map(|raw| DescriptorSet {↩
raw,↩
pool_id,↩
update_after_bind,↩
size,↩
}))↩
}↩
}↩