From 0c11b12574b0ab8fb81506217025149e0a23f9f0 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Fri, 2 Mar 2018 13:05:39 +0100 Subject: [PATCH] More documentation. Especially added new section "Finding out if memory is mappable". --- docs/html/custom_memory_pools.html | 8 +- docs/html/memory_mapping.html | 8 +- .../struct_vma_allocation_create_info.html | 4 +- docs/html/vk__mem__alloc_8h.html | 5 +- docs/html/vk__mem__alloc_8h_source.html | 218 +++++++++--------- src/vk_mem_alloc.h | 78 +++++-- 6 files changed, 190 insertions(+), 131 deletions(-) diff --git a/docs/html/custom_memory_pools.html b/docs/html/custom_memory_pools.html index 017fcf6..c40e397 100644 --- a/docs/html/custom_memory_pools.html +++ b/docs/html/custom_memory_pools.html @@ -66,11 +66,13 @@ $(function() {
Custom memory pools
-

The library automatically creates and manages default memory pool for each memory type available on the device. A pool contains a number of VkDeviceMemory blocks. You can create custom pool and allocate memory out of it. It can be useful if you want to:

+

A memory pool contains a number of VkDeviceMemory blocks. The library automatically creates and manages default pool for each memory type available on the device. Default memory pool automatically grows in size. Size of allocated blocks is also variable and managed automatically.

+

You can create custom pool and allocate memory out of it. It can be useful if you want to:

  • Keep certain kind of allocations separate from others.
    • -
  • Enforce particular size of Vulkan memory blocks.
    • +
  • Enforce particular, fixed size of Vulkan memory blocks.
  • Limit maximum amount of Vulkan memory allocated for that pool.
    • +
  • Reserve minimum or fixed amount of Vulkan memory always preallocated for that pool.

To use custom memory pools:

    @@ -79,7 +81,7 @@ $(function() {
  1. When making an allocation, set VmaAllocationCreateInfo::pool to this handle. You don't need to specify any other parameters of this structure, like usage.

Example:

-
// Create a pool that could have at most 2 blocks, 128 MiB each.
VmaPoolCreateInfo poolCreateInfo = {};
poolCreateInfo.memoryTypeIndex = ...
poolCreateInfo.blockSize = 128ull * 1024 * 1024;
poolCreateInfo.maxBlockCount = 2;
VmaPool pool;
vmaCreatePool(allocator, &poolCreateInfo, &pool);
// Allocate a buffer out of it.
VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
bufCreateInfo.size = 1024;
bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VmaAllocationCreateInfo allocCreateInfo = {};
allocCreateInfo.pool = pool;
VkBuffer buf;
VmaAllocation alloc;
VmaAllocationInfo allocInfo;
vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);

You have to free all allocations made from this pool before destroying it.

+
// Create a pool that can have at most 2 blocks, 128 MiB each.
VmaPoolCreateInfo poolCreateInfo = {};
poolCreateInfo.memoryTypeIndex = ...
poolCreateInfo.blockSize = 128ull * 1024 * 1024;
poolCreateInfo.maxBlockCount = 2;
VmaPool pool;
vmaCreatePool(allocator, &poolCreateInfo, &pool);
// Allocate a buffer out of it.
VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
bufCreateInfo.size = 1024;
bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VmaAllocationCreateInfo allocCreateInfo = {};
allocCreateInfo.pool = pool;
VkBuffer buf;
VmaAllocation alloc;
VmaAllocationInfo allocInfo;
vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);

You have to free all allocations made from this pool before destroying it.

vmaDestroyBuffer(allocator, buf, alloc);
vmaDestroyPool(allocator, pool);

Choosing memory type index

When creating a pool, you must explicitly specify memory type index. To find the one suitable for your buffers or images, you can use code similar to the following:

diff --git a/docs/html/memory_mapping.html b/docs/html/memory_mapping.html index 8df6d86..58da591 100644 --- a/docs/html/memory_mapping.html +++ b/docs/html/memory_mapping.html @@ -82,8 +82,12 @@ Persistently mapped memory Cache control

Memory in Vulkan doesn't need to be unmapped before using it on GPU, but unless a memory types has VK_MEMORY_PROPERTY_HOST_COHERENT_BIT flag set, you need to manually invalidate cache before reading of mapped pointer using function vkvkInvalidateMappedMemoryRanges() and flush cache after writing to mapped pointer using function vkFlushMappedMemoryRanges(). Example:

memcpy(allocInfo.pMappedData, &constantBufferData, sizeof(constantBufferData));
VkMemoryPropertyFlags memFlags;
vmaGetMemoryTypeProperties(allocator, allocInfo.memoryType, &memFlags);
if((memFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) == 0)
{
VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
memRange.memory = allocInfo.deviceMemory;
memRange.offset = allocInfo.offset;
memRange.size = allocInfo.size;
vkFlushMappedMemoryRanges(device, 1, &memRange);
}

Please note that memory allocated with VMA_MEMORY_USAGE_CPU_ONLY is guaranteed to be host coherent.

-

Also, Windows drivers from all 3 PC GPU vendors (AMD, Intel, NVIDIA) currently provide VK_MEMORY_PROPERTY_HOST_COHERENT_BIT flag on all memory types that are VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, so on this platform you may not need to bother.

-
+

Also, Windows drivers from all 3 PC GPU vendors (AMD, Intel, NVIDIA) currently provide VK_MEMORY_PROPERTY_HOST_COHERENT_BIT flag on all memory types that are VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, so on this platform you may not need to bother.

+

+Finding out if memory is mappable

+

It may happen that your allocation ends up in memory that is HOST_VISIBLE (available for mapping) despite it wasn't explicitly requested. For example, application may work on integrated graphics with unified memory (like Intel) or allocation from video memory might have failed, so the library chose system memory as fallback.

+

You can detect this case and map such allocation to access its memory on CPU directly, instead of launching a transfer operation. You can even use VMA_ALLOCATION_CREATE_MAPPED_BIT flag while creating allocations that are not necessarily HOST_VISIBLE (e.g. using VMA_MEMORY_USAGE_GPU_ONLY). If the allocation ends up in memory type that is HOST_VISIBLE, it will be persistently mapped and you can use it directly. If not, the flag is just ignored. Example:

+
VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
bufCreateInfo.size = sizeof(ConstantBuffer);
bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VmaAllocationCreateInfo allocCreateInfo = {};
allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT;
VkBuffer buf;
VmaAllocation alloc;
VmaAllocationInfo allocInfo;
vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
if(allocInfo.pUserData != nullptr)
{
// Allocation ended up in mappable memory.
// It's persistently mapped. You can access it directly.
memcpy(allocInfo.pMappedData, &constantBufferData, sizeof(constantBufferData));
}
else
{
// Allocation ended up in non-mappable memory.
// You need to create CPU-side copy in VMA_MEMORY_USAGE_CPU_ONLY and make a transfer.
}
Generated by   diff --git a/docs/html/struct_vma_allocation_create_info.html b/docs/html/struct_vma_allocation_create_info.html index 43f87f0..30623d1 100644 --- a/docs/html/struct_vma_allocation_create_info.html +++ b/docs/html/struct_vma_allocation_create_info.html @@ -71,7 +71,7 @@ $(function() {

Public Attributes

VmaAllocationCreateFlags flags - Use VmaAllocationCreateFlagBits enum. More...
+ Use VmaAllocationCreateFlagBits enum. More...
  VmaMemoryUsage usage  Intended usage of memory. More...
@@ -105,7 +105,7 @@ Public Attributes
-

Use VmaAllocationCreateFlagBits enum.

+

Use VmaAllocationCreateFlagBits enum.

diff --git a/docs/html/vk__mem__alloc_8h.html b/docs/html/vk__mem__alloc_8h.html index b60f43d..f8b4d11 100644 --- a/docs/html/vk__mem__alloc_8h.html +++ b/docs/html/vk__mem__alloc_8h.html @@ -1295,8 +1295,9 @@ Functions

The function also frees empty VkDeviceMemory blocks.

After allocation has been moved, its VmaAllocationInfo::deviceMemory and/or VmaAllocationInfo::offset changes. You must query them again using vmaGetAllocationInfo() if you need them.

If an allocation has been moved, data in memory is copied to new place automatically, but if it was bound to a buffer or an image, you must destroy that object yourself, create new one and bind it to the new memory pointed by the allocation. You must use vkDestroyBuffer(), vkDestroyImage(), vkCreateBuffer(), vkCreateImage() for that purpose and NOT vmaDestroyBuffer(), vmaDestroyImage(), vmaCreateBuffer(), vmaCreateImage()! Example:

-
VkDevice device = ...;
VmaAllocator allocator = ...;
std::vector<VkBuffer> buffers = ...;
std::vector<VmaAllocation> allocations = ...;
std::vector<VkBool32> allocationsChanged(allocations.size());
vmaDefragment(allocator, allocations.data(), allocations.size(), allocationsChanged.data(), nullptr, nullptr);
for(size_t i = 0; i < allocations.size(); ++i)
{
if(allocationsChanged[i])
{
VmaAllocationInfo allocInfo;
vmaGetAllocationInfo(allocator, allocations[i], &allocInfo);
vkDestroyBuffer(device, buffers[i], nullptr);
VkBufferCreateInfo bufferInfo = ...;
vkCreateBuffer(device, &bufferInfo, nullptr, &buffers[i]);
// You can make dummy call to vkGetBufferMemoryRequirements here to silence validation layer warning.
vkBindBufferMemory(device, buffers[i], allocInfo.deviceMemory, allocInfo.offset);
}
}

Warning! This function is not correct according to Vulkan specification. Use it at your own risk. That's becuase Vulkan doesn't guarantee that memory requirements (size and alignment) for a new buffer or image are consistent. They may be different even for subsequent calls with the same parameters. It really does happen on some platforms, especially with images.

-

This function may be time-consuming, so you shouldn't call it too often (like every frame or after every resource creation/destruction), but rater you can call it on special occasions (like when reloading a game level, when you just destroyed a lot of objects).

+
VkDevice device = ...;
VmaAllocator allocator = ...;
std::vector<VkBuffer> buffers = ...;
std::vector<VmaAllocation> allocations = ...;
std::vector<VkBool32> allocationsChanged(allocations.size());
vmaDefragment(allocator, allocations.data(), allocations.size(), allocationsChanged.data(), nullptr, nullptr);
for(size_t i = 0; i < allocations.size(); ++i)
{
if(allocationsChanged[i])
{
VmaAllocationInfo allocInfo;
vmaGetAllocationInfo(allocator, allocations[i], &allocInfo);
vkDestroyBuffer(device, buffers[i], nullptr);
VkBufferCreateInfo bufferInfo = ...;
vkCreateBuffer(device, &bufferInfo, nullptr, &buffers[i]);
// You can make dummy call to vkGetBufferMemoryRequirements here to silence validation layer warning.
vkBindBufferMemory(device, buffers[i], allocInfo.deviceMemory, allocInfo.offset);
}
}

Note: Please don't expect memory to be fully compacted after this call. Algorithms inside are based on some heuristics that try to maximize number of Vulkan memory blocks to make totally empty to release them, as well as to maximimze continuous empty space inside remaining blocks, while minimizing the number and size of data that needs to be moved. Some fragmentation still remains after this call. This is normal.

+

Warning: This function is not 100% correct according to Vulkan specification. Use it at your own risk. That's because Vulkan doesn't guarantee that memory requirements (size and alignment) for a new buffer or image are consistent. They may be different even for subsequent calls with the same parameters. It really does happen on some platforms, especially with images.

+

Warning: This function may be time-consuming, so you shouldn't call it too often (like every frame or after every resource creation/destruction). You can call it on special occasions (like when reloading a game level or when you just destroyed a lot of objects).

diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index 8442361..959443f 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,154 +62,154 @@ $(function() {
vk_mem_alloc.h
-Go to the documentation of this file.
1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
777 #include <vulkan/vulkan.h>
778 
779 VK_DEFINE_HANDLE(VmaAllocator)
780 
781 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
783  VmaAllocator allocator,
784  uint32_t memoryType,
785  VkDeviceMemory memory,
786  VkDeviceSize size);
788 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
789  VmaAllocator allocator,
790  uint32_t memoryType,
791  VkDeviceMemory memory,
792  VkDeviceSize size);
793 
801 typedef struct VmaDeviceMemoryCallbacks {
803  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
805  PFN_vmaFreeDeviceMemoryFunction pfnFree;
806 } VmaDeviceMemoryCallbacks;
807 
809 typedef enum VmaAllocatorCreateFlagBits {
814  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
836  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
837 
838  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
839 } VmaAllocatorCreateFlagBits;
840 typedef VkFlags VmaAllocatorCreateFlags;
841 
846 typedef struct VmaVulkanFunctions {
847  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
848  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
849  PFN_vkAllocateMemory vkAllocateMemory;
850  PFN_vkFreeMemory vkFreeMemory;
851  PFN_vkMapMemory vkMapMemory;
852  PFN_vkUnmapMemory vkUnmapMemory;
853  PFN_vkBindBufferMemory vkBindBufferMemory;
854  PFN_vkBindImageMemory vkBindImageMemory;
855  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
856  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
857  PFN_vkCreateBuffer vkCreateBuffer;
858  PFN_vkDestroyBuffer vkDestroyBuffer;
859  PFN_vkCreateImage vkCreateImage;
860  PFN_vkDestroyImage vkDestroyImage;
861  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
862  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
863 } VmaVulkanFunctions;
864 
866 typedef struct VmaAllocatorCreateInfo
867 {
869  VmaAllocatorCreateFlags flags;
871 
872  VkPhysicalDevice physicalDevice;
874 
875  VkDevice device;
877 
878  VkDeviceSize preferredLargeHeapBlockSize;
880 
881  const VkAllocationCallbacks* pAllocationCallbacks;
883 
884  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
898  uint32_t frameInUseCount;
922  const VkDeviceSize* pHeapSizeLimit;
934  const VmaVulkanFunctions* pVulkanFunctions;
935 } VmaAllocatorCreateInfo;
936 
938 VkResult vmaCreateAllocator(
939  const VmaAllocatorCreateInfo* pCreateInfo,
940  VmaAllocator* pAllocator);
941 
943 void vmaDestroyAllocator(
944  VmaAllocator allocator);
945 
950 void vmaGetPhysicalDeviceProperties(
951  VmaAllocator allocator,
952  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
953 
958 void vmaGetMemoryProperties(
959  VmaAllocator allocator,
960  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
961 
968 void vmaGetMemoryTypeProperties(
969  VmaAllocator allocator,
970  uint32_t memoryTypeIndex,
971  VkMemoryPropertyFlags* pFlags);
972 
981 void vmaSetCurrentFrameIndex(
982  VmaAllocator allocator,
983  uint32_t frameIndex);
984 
987 typedef struct VmaStatInfo
988 {
990  uint32_t blockCount;
992  uint32_t allocationCount;
994  uint32_t unusedRangeCount;
996  VkDeviceSize usedBytes;
998  VkDeviceSize unusedBytes;
999  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1000  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1001 } VmaStatInfo;
1002 
1004 typedef struct VmaStats
1005 {
1006  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1007  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1008  VmaStatInfo total;
1009 } VmaStats;
1010 
1012 void vmaCalculateStats(
1013  VmaAllocator allocator,
1014  VmaStats* pStats);
1015 
1016 #define VMA_STATS_STRING_ENABLED 1
1017 
1018 #if VMA_STATS_STRING_ENABLED
1019 
1021 
1023 void vmaBuildStatsString(
1024  VmaAllocator allocator,
1025  char** ppStatsString,
1026  VkBool32 detailedMap);
1027 
1028 void vmaFreeStatsString(
1029  VmaAllocator allocator,
1030  char* pStatsString);
1031 
1032 #endif // #if VMA_STATS_STRING_ENABLED
1033 
1034 VK_DEFINE_HANDLE(VmaPool)
1035 
1036 typedef enum VmaMemoryUsage
1037 {
1041  VMA_MEMORY_USAGE_UNKNOWN = 0,
1058  VMA_MEMORY_USAGE_GPU_ONLY = 1,
1068  VMA_MEMORY_USAGE_CPU_ONLY = 2,
1075  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1084  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1085  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1086 } VmaMemoryUsage;
1087 
1089 typedef enum VmaAllocationCreateFlagBits {
1101  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1102 
1112  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1125  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1138  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1145  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1151  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1152 
1153  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1154 } VmaAllocationCreateFlagBits;
1155 typedef VkFlags VmaAllocationCreateFlags;
1156 
1157 typedef struct VmaAllocationCreateInfo
1158 {
1160  VmaAllocationCreateFlags flags;
1166  VmaMemoryUsage usage;
1171  VkMemoryPropertyFlags requiredFlags;
1176  VkMemoryPropertyFlags preferredFlags;
1184  uint32_t memoryTypeBits;
1190  VmaPool pool;
1197  void* pUserData;
1198 } VmaAllocationCreateInfo;
1199 
1214 VkResult vmaFindMemoryTypeIndex(
1215  VmaAllocator allocator,
1216  uint32_t memoryTypeBits,
1217  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1218  uint32_t* pMemoryTypeIndex);
1219 
1221 typedef enum VmaPoolCreateFlagBits {
1239  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1240 
1241  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1242 } VmaPoolCreateFlagBits;
1243 typedef VkFlags VmaPoolCreateFlags;
1244 
1247 typedef struct VmaPoolCreateInfo {
1250  uint32_t memoryTypeIndex;
1253  VmaPoolCreateFlags flags;
1258  VkDeviceSize blockSize;
1263  size_t minBlockCount;
1271  size_t maxBlockCount;
1285  uint32_t frameInUseCount;
1286 } VmaPoolCreateInfo;
1287 
1290 typedef struct VmaPoolStats {
1293  VkDeviceSize size;
1296  VkDeviceSize unusedSize;
1299  size_t allocationCount;
1302  size_t unusedRangeCount;
1309  VkDeviceSize unusedRangeSizeMax;
1310 } VmaPoolStats;
1311 
1318 VkResult vmaCreatePool(
1319  VmaAllocator allocator,
1320  const VmaPoolCreateInfo* pCreateInfo,
1321  VmaPool* pPool);
1322 
1325 void vmaDestroyPool(
1326  VmaAllocator allocator,
1327  VmaPool pool);
1328 
1335 void vmaGetPoolStats(
1336  VmaAllocator allocator,
1337  VmaPool pool,
1338  VmaPoolStats* pPoolStats);
1339 
1346 void vmaMakePoolAllocationsLost(
1347  VmaAllocator allocator,
1348  VmaPool pool,
1349  size_t* pLostAllocationCount);
1350 
1351 VK_DEFINE_HANDLE(VmaAllocation)
1352 
1353 
1355 typedef struct VmaAllocationInfo {
1360  uint32_t memoryType;
1369  VkDeviceMemory deviceMemory;
1374  VkDeviceSize offset;
1379  VkDeviceSize size;
1388  void* pMappedData;
1393  void* pUserData;
1394 } VmaAllocationInfo;
1395 
1406 VkResult vmaAllocateMemory(
1407  VmaAllocator allocator,
1408  const VkMemoryRequirements* pVkMemoryRequirements,
1409  const VmaAllocationCreateInfo* pCreateInfo,
1410  VmaAllocation* pAllocation,
1411  VmaAllocationInfo* pAllocationInfo);
1412 
1419 VkResult vmaAllocateMemoryForBuffer(
1420  VmaAllocator allocator,
1421  VkBuffer buffer,
1422  const VmaAllocationCreateInfo* pCreateInfo,
1423  VmaAllocation* pAllocation,
1424  VmaAllocationInfo* pAllocationInfo);
1425 
1427 VkResult vmaAllocateMemoryForImage(
1428  VmaAllocator allocator,
1429  VkImage image,
1430  const VmaAllocationCreateInfo* pCreateInfo,
1431  VmaAllocation* pAllocation,
1432  VmaAllocationInfo* pAllocationInfo);
1433 
1435 void vmaFreeMemory(
1436  VmaAllocator allocator,
1437  VmaAllocation allocation);
1438 
1440 void vmaGetAllocationInfo(
1441  VmaAllocator allocator,
1442  VmaAllocation allocation,
1443  VmaAllocationInfo* pAllocationInfo);
1444 
1458 void vmaSetAllocationUserData(
1459  VmaAllocator allocator,
1460  VmaAllocation allocation,
1461  void* pUserData);
1462 
1473 void vmaCreateLostAllocation(
1474  VmaAllocator allocator,
1475  VmaAllocation* pAllocation);
1476 
1511 VkResult vmaMapMemory(
1512  VmaAllocator allocator,
1513  VmaAllocation allocation,
1514  void** ppData);
1515 
1520 void vmaUnmapMemory(
1521  VmaAllocator allocator,
1522  VmaAllocation allocation);
1523 
1525 typedef struct VmaDefragmentationInfo {
1530  VkDeviceSize maxBytesToMove;
1535  uint32_t maxAllocationsToMove;
1536 } VmaDefragmentationInfo;
1537 
1539 typedef struct VmaDefragmentationStats {
1541  VkDeviceSize bytesMoved;
1543  VkDeviceSize bytesFreed;
1545  uint32_t allocationsMoved;
1547  uint32_t deviceMemoryBlocksFreed;
1548 } VmaDefragmentationStats;
1549 
1626 VkResult vmaDefragment(
1627  VmaAllocator allocator,
1628  VmaAllocation* pAllocations,
1629  size_t allocationCount,
1630  VkBool32* pAllocationsChanged,
1631  const VmaDefragmentationInfo *pDefragmentationInfo,
1632  VmaDefragmentationStats* pDefragmentationStats);
1633 
1660 VkResult vmaCreateBuffer(
1661  VmaAllocator allocator,
1662  const VkBufferCreateInfo* pBufferCreateInfo,
1663  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1664  VkBuffer* pBuffer,
1665  VmaAllocation* pAllocation,
1666  VmaAllocationInfo* pAllocationInfo);
1667 
1679 void vmaDestroyBuffer(
1680  VmaAllocator allocator,
1681  VkBuffer buffer,
1682  VmaAllocation allocation);
1683 
1685 VkResult vmaCreateImage(
1686  VmaAllocator allocator,
1687  const VkImageCreateInfo* pImageCreateInfo,
1688  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1689  VkImage* pImage,
1690  VmaAllocation* pAllocation,
1691  VmaAllocationInfo* pAllocationInfo);
1692 
1704 void vmaDestroyImage(
1705  VmaAllocator allocator,
1706  VkImage image,
1707  VmaAllocation allocation);
1708 
1709 #ifdef __cplusplus
1710 }
1711 #endif
1712 
1713 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
1714 
1715 // For Visual Studio IntelliSense.
1716 #ifdef __INTELLISENSE__
1717 #define VMA_IMPLEMENTATION
1718 #endif
1719 
1720 #ifdef VMA_IMPLEMENTATION
1721 #undef VMA_IMPLEMENTATION
1722 
1723 #include <cstdint>
1724 #include <cstdlib>
1725 #include <cstring>
1726 
1727 /*******************************************************************************
1728 CONFIGURATION SECTION
1729 
1730 Define some of these macros before each #include of this header or change them
1731 here if you need other then default behavior depending on your environment.
1732 */
1733 
1734 /*
1735 Define this macro to 1 to make the library fetch pointers to Vulkan functions
1736 internally, like:
1737 
1738  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1739 
1740 Define to 0 if you are going to provide you own pointers to Vulkan functions via
1741 VmaAllocatorCreateInfo::pVulkanFunctions.
1742 */
1743 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
1744 #define VMA_STATIC_VULKAN_FUNCTIONS 1
1745 #endif
1746 
1747 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
1748 //#define VMA_USE_STL_CONTAINERS 1
1749 
1750 /* Set this macro to 1 to make the library including and using STL containers:
1751 std::pair, std::vector, std::list, std::unordered_map.
1752 
1753 Set it to 0 or undefined to make the library using its own implementation of
1754 the containers.
1755 */
1756 #if VMA_USE_STL_CONTAINERS
1757  #define VMA_USE_STL_VECTOR 1
1758  #define VMA_USE_STL_UNORDERED_MAP 1
1759  #define VMA_USE_STL_LIST 1
1760 #endif
1761 
1762 #if VMA_USE_STL_VECTOR
1763  #include <vector>
1764 #endif
1765 
1766 #if VMA_USE_STL_UNORDERED_MAP
1767  #include <unordered_map>
1768 #endif
1769 
1770 #if VMA_USE_STL_LIST
1771  #include <list>
1772 #endif
1773 
1774 /*
1775 Following headers are used in this CONFIGURATION section only, so feel free to
1776 remove them if not needed.
1777 */
1778 #include <cassert> // for assert
1779 #include <algorithm> // for min, max
1780 #include <mutex> // for std::mutex
1781 #include <atomic> // for std::atomic
1782 
1783 #if !defined(_WIN32) && !defined(__APPLE__)
1784  #include <malloc.h> // for aligned_alloc()
1785 #endif
1786 
1787 #if defined(__APPLE__)
1788 #include <cstdlib>
1789 void *aligned_alloc(size_t alignment, size_t size)
1790 {
1791  // alignment must be >= sizeof(void*)
1792  if(alignment < sizeof(void*))
1793  {
1794  alignment = sizeof(void*);
1795  }
1796 
1797  void *pointer;
1798  if(posix_memalign(&pointer, alignment, size) == 0)
1799  return pointer;
1800  return VMA_NULL;
1801 }
1802 #endif
1803 
1804 // Normal assert to check for programmer's errors, especially in Debug configuration.
1805 #ifndef VMA_ASSERT
1806  #ifdef _DEBUG
1807  #define VMA_ASSERT(expr) assert(expr)
1808  #else
1809  #define VMA_ASSERT(expr)
1810  #endif
1811 #endif
1812 
1813 // Assert that will be called very often, like inside data structures e.g. operator[].
1814 // Making it non-empty can make program slow.
1815 #ifndef VMA_HEAVY_ASSERT
1816  #ifdef _DEBUG
1817  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
1818  #else
1819  #define VMA_HEAVY_ASSERT(expr)
1820  #endif
1821 #endif
1822 
1823 #ifndef VMA_NULL
1824  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
1825  #define VMA_NULL nullptr
1826 #endif
1827 
1828 #ifndef VMA_ALIGN_OF
1829  #define VMA_ALIGN_OF(type) (__alignof(type))
1830 #endif
1831 
1832 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
1833  #if defined(_WIN32)
1834  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
1835  #else
1836  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
1837  #endif
1838 #endif
1839 
1840 #ifndef VMA_SYSTEM_FREE
1841  #if defined(_WIN32)
1842  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
1843  #else
1844  #define VMA_SYSTEM_FREE(ptr) free(ptr)
1845  #endif
1846 #endif
1847 
1848 #ifndef VMA_MIN
1849  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
1850 #endif
1851 
1852 #ifndef VMA_MAX
1853  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
1854 #endif
1855 
1856 #ifndef VMA_SWAP
1857  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
1858 #endif
1859 
1860 #ifndef VMA_SORT
1861  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
1862 #endif
1863 
1864 #ifndef VMA_DEBUG_LOG
1865  #define VMA_DEBUG_LOG(format, ...)
1866  /*
1867  #define VMA_DEBUG_LOG(format, ...) do { \
1868  printf(format, __VA_ARGS__); \
1869  printf("\n"); \
1870  } while(false)
1871  */
1872 #endif
1873 
1874 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
1875 #if VMA_STATS_STRING_ENABLED
1876  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
1877  {
1878  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
1879  }
1880  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
1881  {
1882  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
1883  }
1884  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
1885  {
1886  snprintf(outStr, strLen, "%p", ptr);
1887  }
1888 #endif
1889 
1890 #ifndef VMA_MUTEX
1891  class VmaMutex
1892  {
1893  public:
1894  VmaMutex() { }
1895  ~VmaMutex() { }
1896  void Lock() { m_Mutex.lock(); }
1897  void Unlock() { m_Mutex.unlock(); }
1898  private:
1899  std::mutex m_Mutex;
1900  };
1901  #define VMA_MUTEX VmaMutex
1902 #endif
1903 
1904 /*
1905 If providing your own implementation, you need to implement a subset of std::atomic:
1906 
1907 - Constructor(uint32_t desired)
1908 - uint32_t load() const
1909 - void store(uint32_t desired)
1910 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
1911 */
1912 #ifndef VMA_ATOMIC_UINT32
1913  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
1914 #endif
1915 
1916 #ifndef VMA_BEST_FIT
1917 
1929  #define VMA_BEST_FIT (1)
1930 #endif
1931 
1932 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
1933 
1937  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
1938 #endif
1939 
1940 #ifndef VMA_DEBUG_ALIGNMENT
1941 
1945  #define VMA_DEBUG_ALIGNMENT (1)
1946 #endif
1947 
1948 #ifndef VMA_DEBUG_MARGIN
1949 
1953  #define VMA_DEBUG_MARGIN (0)
1954 #endif
1955 
1956 #ifndef VMA_DEBUG_GLOBAL_MUTEX
1957 
1961  #define VMA_DEBUG_GLOBAL_MUTEX (0)
1962 #endif
1963 
1964 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
1965 
1969  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
1970 #endif
1971 
1972 #ifndef VMA_SMALL_HEAP_MAX_SIZE
1973  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
1975 #endif
1976 
1977 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
1978  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
1980 #endif
1981 
1982 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
1983 
1984 /*******************************************************************************
1985 END OF CONFIGURATION
1986 */
1987 
1988 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
1989  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
1990 
1991 // Returns number of bits set to 1 in (v).
1992 static inline uint32_t VmaCountBitsSet(uint32_t v)
1993 {
1994  uint32_t c = v - ((v >> 1) & 0x55555555);
1995  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
1996  c = ((c >> 4) + c) & 0x0F0F0F0F;
1997  c = ((c >> 8) + c) & 0x00FF00FF;
1998  c = ((c >> 16) + c) & 0x0000FFFF;
1999  return c;
2000 }
2001 
2002 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2003 // Use types like uint32_t, uint64_t as T.
2004 template <typename T>
2005 static inline T VmaAlignUp(T val, T align)
2006 {
2007  return (val + align - 1) / align * align;
2008 }
2009 
2010 // Division with mathematical rounding to nearest number.
2011 template <typename T>
2012 inline T VmaRoundDiv(T x, T y)
2013 {
2014  return (x + (y / (T)2)) / y;
2015 }
2016 
2017 #ifndef VMA_SORT
2018 
2019 template<typename Iterator, typename Compare>
2020 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2021 {
2022  Iterator centerValue = end; --centerValue;
2023  Iterator insertIndex = beg;
2024  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2025  {
2026  if(cmp(*memTypeIndex, *centerValue))
2027  {
2028  if(insertIndex != memTypeIndex)
2029  {
2030  VMA_SWAP(*memTypeIndex, *insertIndex);
2031  }
2032  ++insertIndex;
2033  }
2034  }
2035  if(insertIndex != centerValue)
2036  {
2037  VMA_SWAP(*insertIndex, *centerValue);
2038  }
2039  return insertIndex;
2040 }
2041 
2042 template<typename Iterator, typename Compare>
2043 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2044 {
2045  if(beg < end)
2046  {
2047  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2048  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2049  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2050  }
2051 }
2052 
2053 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2054 
2055 #endif // #ifndef VMA_SORT
2056 
2057 /*
2058 Returns true if two memory blocks occupy overlapping pages.
2059 ResourceA must be in less memory offset than ResourceB.
2060 
2061 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2062 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2063 */
2064 static inline bool VmaBlocksOnSamePage(
2065  VkDeviceSize resourceAOffset,
2066  VkDeviceSize resourceASize,
2067  VkDeviceSize resourceBOffset,
2068  VkDeviceSize pageSize)
2069 {
2070  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2071  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2072  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2073  VkDeviceSize resourceBStart = resourceBOffset;
2074  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2075  return resourceAEndPage == resourceBStartPage;
2076 }
2077 
2078 enum VmaSuballocationType
2079 {
2080  VMA_SUBALLOCATION_TYPE_FREE = 0,
2081  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2082  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2083  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2084  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2085  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2086  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2087 };
2088 
2089 /*
2090 Returns true if given suballocation types could conflict and must respect
2091 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2092 or linear image and another one is optimal image. If type is unknown, behave
2093 conservatively.
2094 */
2095 static inline bool VmaIsBufferImageGranularityConflict(
2096  VmaSuballocationType suballocType1,
2097  VmaSuballocationType suballocType2)
2098 {
2099  if(suballocType1 > suballocType2)
2100  {
2101  VMA_SWAP(suballocType1, suballocType2);
2102  }
2103 
2104  switch(suballocType1)
2105  {
2106  case VMA_SUBALLOCATION_TYPE_FREE:
2107  return false;
2108  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
2109  return true;
2110  case VMA_SUBALLOCATION_TYPE_BUFFER:
2111  return
2112  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2113  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2114  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
2115  return
2116  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2117  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
2118  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2119  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
2120  return
2121  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2122  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
2123  return false;
2124  default:
2125  VMA_ASSERT(0);
2126  return true;
2127  }
2128 }
2129 
2130 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
2131 struct VmaMutexLock
2132 {
2133 public:
2134  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
2135  m_pMutex(useMutex ? &mutex : VMA_NULL)
2136  {
2137  if(m_pMutex)
2138  {
2139  m_pMutex->Lock();
2140  }
2141  }
2142 
2143  ~VmaMutexLock()
2144  {
2145  if(m_pMutex)
2146  {
2147  m_pMutex->Unlock();
2148  }
2149  }
2150 
2151 private:
2152  VMA_MUTEX* m_pMutex;
2153 };
2154 
2155 #if VMA_DEBUG_GLOBAL_MUTEX
2156  static VMA_MUTEX gDebugGlobalMutex;
2157  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
2158 #else
2159  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
2160 #endif
2161 
2162 // Minimum size of a free suballocation to register it in the free suballocation collection.
2163 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
2164 
2165 /*
2166 Performs binary search and returns iterator to first element that is greater or
2167 equal to (key), according to comparison (cmp).
2168 
2169 Cmp should return true if first argument is less than second argument.
2170 
2171 Returned value is the found element, if present in the collection or place where
2172 new element with value (key) should be inserted.
2173 */
2174 template <typename IterT, typename KeyT, typename CmpT>
2175 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
2176 {
2177  size_t down = 0, up = (end - beg);
2178  while(down < up)
2179  {
2180  const size_t mid = (down + up) / 2;
2181  if(cmp(*(beg+mid), key))
2182  {
2183  down = mid + 1;
2184  }
2185  else
2186  {
2187  up = mid;
2188  }
2189  }
2190  return beg + down;
2191 }
2192 
2194 // Memory allocation
2195 
2196 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
2197 {
2198  if((pAllocationCallbacks != VMA_NULL) &&
2199  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
2200  {
2201  return (*pAllocationCallbacks->pfnAllocation)(
2202  pAllocationCallbacks->pUserData,
2203  size,
2204  alignment,
2205  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2206  }
2207  else
2208  {
2209  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
2210  }
2211 }
2212 
2213 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
2214 {
2215  if((pAllocationCallbacks != VMA_NULL) &&
2216  (pAllocationCallbacks->pfnFree != VMA_NULL))
2217  {
2218  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
2219  }
2220  else
2221  {
2222  VMA_SYSTEM_FREE(ptr);
2223  }
2224 }
2225 
2226 template<typename T>
2227 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
2228 {
2229  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
2230 }
2231 
2232 template<typename T>
2233 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
2234 {
2235  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
2236 }
2237 
2238 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
2239 
2240 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
2241 
2242 template<typename T>
2243 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
2244 {
2245  ptr->~T();
2246  VmaFree(pAllocationCallbacks, ptr);
2247 }
2248 
2249 template<typename T>
2250 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
2251 {
2252  if(ptr != VMA_NULL)
2253  {
2254  for(size_t i = count; i--; )
2255  {
2256  ptr[i].~T();
2257  }
2258  VmaFree(pAllocationCallbacks, ptr);
2259  }
2260 }
2261 
2262 // STL-compatible allocator.
2263 template<typename T>
2264 class VmaStlAllocator
2265 {
2266 public:
2267  const VkAllocationCallbacks* const m_pCallbacks;
2268  typedef T value_type;
2269 
2270  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
2271  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
2272 
2273  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
2274  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
2275 
2276  template<typename U>
2277  bool operator==(const VmaStlAllocator<U>& rhs) const
2278  {
2279  return m_pCallbacks == rhs.m_pCallbacks;
2280  }
2281  template<typename U>
2282  bool operator!=(const VmaStlAllocator<U>& rhs) const
2283  {
2284  return m_pCallbacks != rhs.m_pCallbacks;
2285  }
2286 
2287  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
2288 };
2289 
2290 #if VMA_USE_STL_VECTOR
2291 
2292 #define VmaVector std::vector
2293 
2294 template<typename T, typename allocatorT>
2295 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
2296 {
2297  vec.insert(vec.begin() + index, item);
2298 }
2299 
2300 template<typename T, typename allocatorT>
2301 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
2302 {
2303  vec.erase(vec.begin() + index);
2304 }
2305 
2306 #else // #if VMA_USE_STL_VECTOR
2307 
2308 /* Class with interface compatible with subset of std::vector.
2309 T must be POD because constructors and destructors are not called and memcpy is
2310 used for these objects. */
2311 template<typename T, typename AllocatorT>
2312 class VmaVector
2313 {
2314 public:
2315  typedef T value_type;
2316 
2317  VmaVector(const AllocatorT& allocator) :
2318  m_Allocator(allocator),
2319  m_pArray(VMA_NULL),
2320  m_Count(0),
2321  m_Capacity(0)
2322  {
2323  }
2324 
2325  VmaVector(size_t count, const AllocatorT& allocator) :
2326  m_Allocator(allocator),
2327  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
2328  m_Count(count),
2329  m_Capacity(count)
2330  {
2331  }
2332 
2333  VmaVector(const VmaVector<T, AllocatorT>& src) :
2334  m_Allocator(src.m_Allocator),
2335  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
2336  m_Count(src.m_Count),
2337  m_Capacity(src.m_Count)
2338  {
2339  if(m_Count != 0)
2340  {
2341  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
2342  }
2343  }
2344 
2345  ~VmaVector()
2346  {
2347  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2348  }
2349 
2350  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
2351  {
2352  if(&rhs != this)
2353  {
2354  resize(rhs.m_Count);
2355  if(m_Count != 0)
2356  {
2357  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
2358  }
2359  }
2360  return *this;
2361  }
2362 
2363  bool empty() const { return m_Count == 0; }
2364  size_t size() const { return m_Count; }
2365  T* data() { return m_pArray; }
2366  const T* data() const { return m_pArray; }
2367 
2368  T& operator[](size_t index)
2369  {
2370  VMA_HEAVY_ASSERT(index < m_Count);
2371  return m_pArray[index];
2372  }
2373  const T& operator[](size_t index) const
2374  {
2375  VMA_HEAVY_ASSERT(index < m_Count);
2376  return m_pArray[index];
2377  }
2378 
2379  T& front()
2380  {
2381  VMA_HEAVY_ASSERT(m_Count > 0);
2382  return m_pArray[0];
2383  }
2384  const T& front() const
2385  {
2386  VMA_HEAVY_ASSERT(m_Count > 0);
2387  return m_pArray[0];
2388  }
2389  T& back()
2390  {
2391  VMA_HEAVY_ASSERT(m_Count > 0);
2392  return m_pArray[m_Count - 1];
2393  }
2394  const T& back() const
2395  {
2396  VMA_HEAVY_ASSERT(m_Count > 0);
2397  return m_pArray[m_Count - 1];
2398  }
2399 
2400  void reserve(size_t newCapacity, bool freeMemory = false)
2401  {
2402  newCapacity = VMA_MAX(newCapacity, m_Count);
2403 
2404  if((newCapacity < m_Capacity) && !freeMemory)
2405  {
2406  newCapacity = m_Capacity;
2407  }
2408 
2409  if(newCapacity != m_Capacity)
2410  {
2411  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
2412  if(m_Count != 0)
2413  {
2414  memcpy(newArray, m_pArray, m_Count * sizeof(T));
2415  }
2416  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2417  m_Capacity = newCapacity;
2418  m_pArray = newArray;
2419  }
2420  }
2421 
2422  void resize(size_t newCount, bool freeMemory = false)
2423  {
2424  size_t newCapacity = m_Capacity;
2425  if(newCount > m_Capacity)
2426  {
2427  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
2428  }
2429  else if(freeMemory)
2430  {
2431  newCapacity = newCount;
2432  }
2433 
2434  if(newCapacity != m_Capacity)
2435  {
2436  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
2437  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
2438  if(elementsToCopy != 0)
2439  {
2440  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
2441  }
2442  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2443  m_Capacity = newCapacity;
2444  m_pArray = newArray;
2445  }
2446 
2447  m_Count = newCount;
2448  }
2449 
2450  void clear(bool freeMemory = false)
2451  {
2452  resize(0, freeMemory);
2453  }
2454 
2455  void insert(size_t index, const T& src)
2456  {
2457  VMA_HEAVY_ASSERT(index <= m_Count);
2458  const size_t oldCount = size();
2459  resize(oldCount + 1);
2460  if(index < oldCount)
2461  {
2462  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
2463  }
2464  m_pArray[index] = src;
2465  }
2466 
2467  void remove(size_t index)
2468  {
2469  VMA_HEAVY_ASSERT(index < m_Count);
2470  const size_t oldCount = size();
2471  if(index < oldCount - 1)
2472  {
2473  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
2474  }
2475  resize(oldCount - 1);
2476  }
2477 
2478  void push_back(const T& src)
2479  {
2480  const size_t newIndex = size();
2481  resize(newIndex + 1);
2482  m_pArray[newIndex] = src;
2483  }
2484 
2485  void pop_back()
2486  {
2487  VMA_HEAVY_ASSERT(m_Count > 0);
2488  resize(size() - 1);
2489  }
2490 
2491  void push_front(const T& src)
2492  {
2493  insert(0, src);
2494  }
2495 
2496  void pop_front()
2497  {
2498  VMA_HEAVY_ASSERT(m_Count > 0);
2499  remove(0);
2500  }
2501 
2502  typedef T* iterator;
2503 
2504  iterator begin() { return m_pArray; }
2505  iterator end() { return m_pArray + m_Count; }
2506 
2507 private:
2508  AllocatorT m_Allocator;
2509  T* m_pArray;
2510  size_t m_Count;
2511  size_t m_Capacity;
2512 };
2513 
2514 template<typename T, typename allocatorT>
2515 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2516 {
2517  vec.insert(index, item);
2518 }
2519 
2520 template<typename T, typename allocatorT>
2521 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2522 {
2523  vec.remove(index);
2524 }
2525 
2526 #endif // #if VMA_USE_STL_VECTOR
2527 
2528 template<typename CmpLess, typename VectorT>
2529 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2530 {
2531  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2532  vector.data(),
2533  vector.data() + vector.size(),
2534  value,
2535  CmpLess()) - vector.data();
2536  VmaVectorInsert(vector, indexToInsert, value);
2537  return indexToInsert;
2538 }
2539 
2540 template<typename CmpLess, typename VectorT>
2541 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2542 {
2543  CmpLess comparator;
2544  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2545  vector.begin(),
2546  vector.end(),
2547  value,
2548  comparator);
2549  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
2550  {
2551  size_t indexToRemove = it - vector.begin();
2552  VmaVectorRemove(vector, indexToRemove);
2553  return true;
2554  }
2555  return false;
2556 }
2557 
2558 template<typename CmpLess, typename VectorT>
2559 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
2560 {
2561  CmpLess comparator;
2562  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2563  vector.data(),
2564  vector.data() + vector.size(),
2565  value,
2566  comparator);
2567  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
2568  {
2569  return it - vector.begin();
2570  }
2571  else
2572  {
2573  return vector.size();
2574  }
2575 }
2576 
2578 // class VmaPoolAllocator
2579 
2580 /*
2581 Allocator for objects of type T using a list of arrays (pools) to speed up
2582 allocation. Number of elements that can be allocated is not bounded because
2583 allocator can create multiple blocks.
2584 */
2585 template<typename T>
2586 class VmaPoolAllocator
2587 {
2588 public:
2589  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
2590  ~VmaPoolAllocator();
2591  void Clear();
2592  T* Alloc();
2593  void Free(T* ptr);
2594 
2595 private:
2596  union Item
2597  {
2598  uint32_t NextFreeIndex;
2599  T Value;
2600  };
2601 
2602  struct ItemBlock
2603  {
2604  Item* pItems;
2605  uint32_t FirstFreeIndex;
2606  };
2607 
2608  const VkAllocationCallbacks* m_pAllocationCallbacks;
2609  size_t m_ItemsPerBlock;
2610  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
2611 
2612  ItemBlock& CreateNewBlock();
2613 };
2614 
2615 template<typename T>
2616 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
2617  m_pAllocationCallbacks(pAllocationCallbacks),
2618  m_ItemsPerBlock(itemsPerBlock),
2619  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
2620 {
2621  VMA_ASSERT(itemsPerBlock > 0);
2622 }
2623 
2624 template<typename T>
2625 VmaPoolAllocator<T>::~VmaPoolAllocator()
2626 {
2627  Clear();
2628 }
2629 
2630 template<typename T>
2631 void VmaPoolAllocator<T>::Clear()
2632 {
2633  for(size_t i = m_ItemBlocks.size(); i--; )
2634  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
2635  m_ItemBlocks.clear();
2636 }
2637 
2638 template<typename T>
2639 T* VmaPoolAllocator<T>::Alloc()
2640 {
2641  for(size_t i = m_ItemBlocks.size(); i--; )
2642  {
2643  ItemBlock& block = m_ItemBlocks[i];
2644  // This block has some free items: Use first one.
2645  if(block.FirstFreeIndex != UINT32_MAX)
2646  {
2647  Item* const pItem = &block.pItems[block.FirstFreeIndex];
2648  block.FirstFreeIndex = pItem->NextFreeIndex;
2649  return &pItem->Value;
2650  }
2651  }
2652 
2653  // No block has free item: Create new one and use it.
2654  ItemBlock& newBlock = CreateNewBlock();
2655  Item* const pItem = &newBlock.pItems[0];
2656  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
2657  return &pItem->Value;
2658 }
2659 
2660 template<typename T>
2661 void VmaPoolAllocator<T>::Free(T* ptr)
2662 {
2663  // Search all memory blocks to find ptr.
2664  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
2665  {
2666  ItemBlock& block = m_ItemBlocks[i];
2667 
2668  // Casting to union.
2669  Item* pItemPtr;
2670  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
2671 
2672  // Check if pItemPtr is in address range of this block.
2673  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
2674  {
2675  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
2676  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
2677  block.FirstFreeIndex = index;
2678  return;
2679  }
2680  }
2681  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
2682 }
2683 
2684 template<typename T>
2685 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
2686 {
2687  ItemBlock newBlock = {
2688  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
2689 
2690  m_ItemBlocks.push_back(newBlock);
2691 
2692  // Setup singly-linked list of all free items in this block.
2693  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
2694  newBlock.pItems[i].NextFreeIndex = i + 1;
2695  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
2696  return m_ItemBlocks.back();
2697 }
2698 
2700 // class VmaRawList, VmaList
2701 
2702 #if VMA_USE_STL_LIST
2703 
2704 #define VmaList std::list
2705 
2706 #else // #if VMA_USE_STL_LIST
2707 
2708 template<typename T>
2709 struct VmaListItem
2710 {
2711  VmaListItem* pPrev;
2712  VmaListItem* pNext;
2713  T Value;
2714 };
2715 
2716 // Doubly linked list.
2717 template<typename T>
2718 class VmaRawList
2719 {
2720 public:
2721  typedef VmaListItem<T> ItemType;
2722 
2723  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
2724  ~VmaRawList();
2725  void Clear();
2726 
2727  size_t GetCount() const { return m_Count; }
2728  bool IsEmpty() const { return m_Count == 0; }
2729 
2730  ItemType* Front() { return m_pFront; }
2731  const ItemType* Front() const { return m_pFront; }
2732  ItemType* Back() { return m_pBack; }
2733  const ItemType* Back() const { return m_pBack; }
2734 
2735  ItemType* PushBack();
2736  ItemType* PushFront();
2737  ItemType* PushBack(const T& value);
2738  ItemType* PushFront(const T& value);
2739  void PopBack();
2740  void PopFront();
2741 
2742  // Item can be null - it means PushBack.
2743  ItemType* InsertBefore(ItemType* pItem);
2744  // Item can be null - it means PushFront.
2745  ItemType* InsertAfter(ItemType* pItem);
2746 
2747  ItemType* InsertBefore(ItemType* pItem, const T& value);
2748  ItemType* InsertAfter(ItemType* pItem, const T& value);
2749 
2750  void Remove(ItemType* pItem);
2751 
2752 private:
2753  const VkAllocationCallbacks* const m_pAllocationCallbacks;
2754  VmaPoolAllocator<ItemType> m_ItemAllocator;
2755  ItemType* m_pFront;
2756  ItemType* m_pBack;
2757  size_t m_Count;
2758 
2759  // Declared not defined, to block copy constructor and assignment operator.
2760  VmaRawList(const VmaRawList<T>& src);
2761  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
2762 };
2763 
2764 template<typename T>
2765 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
2766  m_pAllocationCallbacks(pAllocationCallbacks),
2767  m_ItemAllocator(pAllocationCallbacks, 128),
2768  m_pFront(VMA_NULL),
2769  m_pBack(VMA_NULL),
2770  m_Count(0)
2771 {
2772 }
2773 
2774 template<typename T>
2775 VmaRawList<T>::~VmaRawList()
2776 {
2777  // Intentionally not calling Clear, because that would be unnecessary
2778  // computations to return all items to m_ItemAllocator as free.
2779 }
2780 
2781 template<typename T>
2782 void VmaRawList<T>::Clear()
2783 {
2784  if(IsEmpty() == false)
2785  {
2786  ItemType* pItem = m_pBack;
2787  while(pItem != VMA_NULL)
2788  {
2789  ItemType* const pPrevItem = pItem->pPrev;
2790  m_ItemAllocator.Free(pItem);
2791  pItem = pPrevItem;
2792  }
2793  m_pFront = VMA_NULL;
2794  m_pBack = VMA_NULL;
2795  m_Count = 0;
2796  }
2797 }
2798 
2799 template<typename T>
2800 VmaListItem<T>* VmaRawList<T>::PushBack()
2801 {
2802  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2803  pNewItem->pNext = VMA_NULL;
2804  if(IsEmpty())
2805  {
2806  pNewItem->pPrev = VMA_NULL;
2807  m_pFront = pNewItem;
2808  m_pBack = pNewItem;
2809  m_Count = 1;
2810  }
2811  else
2812  {
2813  pNewItem->pPrev = m_pBack;
2814  m_pBack->pNext = pNewItem;
2815  m_pBack = pNewItem;
2816  ++m_Count;
2817  }
2818  return pNewItem;
2819 }
2820 
2821 template<typename T>
2822 VmaListItem<T>* VmaRawList<T>::PushFront()
2823 {
2824  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2825  pNewItem->pPrev = VMA_NULL;
2826  if(IsEmpty())
2827  {
2828  pNewItem->pNext = VMA_NULL;
2829  m_pFront = pNewItem;
2830  m_pBack = pNewItem;
2831  m_Count = 1;
2832  }
2833  else
2834  {
2835  pNewItem->pNext = m_pFront;
2836  m_pFront->pPrev = pNewItem;
2837  m_pFront = pNewItem;
2838  ++m_Count;
2839  }
2840  return pNewItem;
2841 }
2842 
2843 template<typename T>
2844 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
2845 {
2846  ItemType* const pNewItem = PushBack();
2847  pNewItem->Value = value;
2848  return pNewItem;
2849 }
2850 
2851 template<typename T>
2852 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
2853 {
2854  ItemType* const pNewItem = PushFront();
2855  pNewItem->Value = value;
2856  return pNewItem;
2857 }
2858 
2859 template<typename T>
2860 void VmaRawList<T>::PopBack()
2861 {
2862  VMA_HEAVY_ASSERT(m_Count > 0);
2863  ItemType* const pBackItem = m_pBack;
2864  ItemType* const pPrevItem = pBackItem->pPrev;
2865  if(pPrevItem != VMA_NULL)
2866  {
2867  pPrevItem->pNext = VMA_NULL;
2868  }
2869  m_pBack = pPrevItem;
2870  m_ItemAllocator.Free(pBackItem);
2871  --m_Count;
2872 }
2873 
2874 template<typename T>
2875 void VmaRawList<T>::PopFront()
2876 {
2877  VMA_HEAVY_ASSERT(m_Count > 0);
2878  ItemType* const pFrontItem = m_pFront;
2879  ItemType* const pNextItem = pFrontItem->pNext;
2880  if(pNextItem != VMA_NULL)
2881  {
2882  pNextItem->pPrev = VMA_NULL;
2883  }
2884  m_pFront = pNextItem;
2885  m_ItemAllocator.Free(pFrontItem);
2886  --m_Count;
2887 }
2888 
2889 template<typename T>
2890 void VmaRawList<T>::Remove(ItemType* pItem)
2891 {
2892  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
2893  VMA_HEAVY_ASSERT(m_Count > 0);
2894 
2895  if(pItem->pPrev != VMA_NULL)
2896  {
2897  pItem->pPrev->pNext = pItem->pNext;
2898  }
2899  else
2900  {
2901  VMA_HEAVY_ASSERT(m_pFront == pItem);
2902  m_pFront = pItem->pNext;
2903  }
2904 
2905  if(pItem->pNext != VMA_NULL)
2906  {
2907  pItem->pNext->pPrev = pItem->pPrev;
2908  }
2909  else
2910  {
2911  VMA_HEAVY_ASSERT(m_pBack == pItem);
2912  m_pBack = pItem->pPrev;
2913  }
2914 
2915  m_ItemAllocator.Free(pItem);
2916  --m_Count;
2917 }
2918 
2919 template<typename T>
2920 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
2921 {
2922  if(pItem != VMA_NULL)
2923  {
2924  ItemType* const prevItem = pItem->pPrev;
2925  ItemType* const newItem = m_ItemAllocator.Alloc();
2926  newItem->pPrev = prevItem;
2927  newItem->pNext = pItem;
2928  pItem->pPrev = newItem;
2929  if(prevItem != VMA_NULL)
2930  {
2931  prevItem->pNext = newItem;
2932  }
2933  else
2934  {
2935  VMA_HEAVY_ASSERT(m_pFront == pItem);
2936  m_pFront = newItem;
2937  }
2938  ++m_Count;
2939  return newItem;
2940  }
2941  else
2942  return PushBack();
2943 }
2944 
2945 template<typename T>
2946 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
2947 {
2948  if(pItem != VMA_NULL)
2949  {
2950  ItemType* const nextItem = pItem->pNext;
2951  ItemType* const newItem = m_ItemAllocator.Alloc();
2952  newItem->pNext = nextItem;
2953  newItem->pPrev = pItem;
2954  pItem->pNext = newItem;
2955  if(nextItem != VMA_NULL)
2956  {
2957  nextItem->pPrev = newItem;
2958  }
2959  else
2960  {
2961  VMA_HEAVY_ASSERT(m_pBack == pItem);
2962  m_pBack = newItem;
2963  }
2964  ++m_Count;
2965  return newItem;
2966  }
2967  else
2968  return PushFront();
2969 }
2970 
2971 template<typename T>
2972 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
2973 {
2974  ItemType* const newItem = InsertBefore(pItem);
2975  newItem->Value = value;
2976  return newItem;
2977 }
2978 
2979 template<typename T>
2980 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
2981 {
2982  ItemType* const newItem = InsertAfter(pItem);
2983  newItem->Value = value;
2984  return newItem;
2985 }
2986 
2987 template<typename T, typename AllocatorT>
2988 class VmaList
2989 {
2990 public:
2991  class iterator
2992  {
2993  public:
2994  iterator() :
2995  m_pList(VMA_NULL),
2996  m_pItem(VMA_NULL)
2997  {
2998  }
2999 
3000  T& operator*() const
3001  {
3002  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3003  return m_pItem->Value;
3004  }
3005  T* operator->() const
3006  {
3007  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3008  return &m_pItem->Value;
3009  }
3010 
3011  iterator& operator++()
3012  {
3013  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3014  m_pItem = m_pItem->pNext;
3015  return *this;
3016  }
3017  iterator& operator--()
3018  {
3019  if(m_pItem != VMA_NULL)
3020  {
3021  m_pItem = m_pItem->pPrev;
3022  }
3023  else
3024  {
3025  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3026  m_pItem = m_pList->Back();
3027  }
3028  return *this;
3029  }
3030 
3031  iterator operator++(int)
3032  {
3033  iterator result = *this;
3034  ++*this;
3035  return result;
3036  }
3037  iterator operator--(int)
3038  {
3039  iterator result = *this;
3040  --*this;
3041  return result;
3042  }
3043 
3044  bool operator==(const iterator& rhs) const
3045  {
3046  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3047  return m_pItem == rhs.m_pItem;
3048  }
3049  bool operator!=(const iterator& rhs) const
3050  {
3051  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3052  return m_pItem != rhs.m_pItem;
3053  }
3054 
3055  private:
3056  VmaRawList<T>* m_pList;
3057  VmaListItem<T>* m_pItem;
3058 
3059  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3060  m_pList(pList),
3061  m_pItem(pItem)
3062  {
3063  }
3064 
3065  friend class VmaList<T, AllocatorT>;
3066  };
3067 
3068  class const_iterator
3069  {
3070  public:
3071  const_iterator() :
3072  m_pList(VMA_NULL),
3073  m_pItem(VMA_NULL)
3074  {
3075  }
3076 
3077  const_iterator(const iterator& src) :
3078  m_pList(src.m_pList),
3079  m_pItem(src.m_pItem)
3080  {
3081  }
3082 
3083  const T& operator*() const
3084  {
3085  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3086  return m_pItem->Value;
3087  }
3088  const T* operator->() const
3089  {
3090  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3091  return &m_pItem->Value;
3092  }
3093 
3094  const_iterator& operator++()
3095  {
3096  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3097  m_pItem = m_pItem->pNext;
3098  return *this;
3099  }
3100  const_iterator& operator--()
3101  {
3102  if(m_pItem != VMA_NULL)
3103  {
3104  m_pItem = m_pItem->pPrev;
3105  }
3106  else
3107  {
3108  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3109  m_pItem = m_pList->Back();
3110  }
3111  return *this;
3112  }
3113 
3114  const_iterator operator++(int)
3115  {
3116  const_iterator result = *this;
3117  ++*this;
3118  return result;
3119  }
3120  const_iterator operator--(int)
3121  {
3122  const_iterator result = *this;
3123  --*this;
3124  return result;
3125  }
3126 
3127  bool operator==(const const_iterator& rhs) const
3128  {
3129  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3130  return m_pItem == rhs.m_pItem;
3131  }
3132  bool operator!=(const const_iterator& rhs) const
3133  {
3134  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3135  return m_pItem != rhs.m_pItem;
3136  }
3137 
3138  private:
3139  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
3140  m_pList(pList),
3141  m_pItem(pItem)
3142  {
3143  }
3144 
3145  const VmaRawList<T>* m_pList;
3146  const VmaListItem<T>* m_pItem;
3147 
3148  friend class VmaList<T, AllocatorT>;
3149  };
3150 
3151  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
3152 
3153  bool empty() const { return m_RawList.IsEmpty(); }
3154  size_t size() const { return m_RawList.GetCount(); }
3155 
3156  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
3157  iterator end() { return iterator(&m_RawList, VMA_NULL); }
3158 
3159  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
3160  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
3161 
3162  void clear() { m_RawList.Clear(); }
3163  void push_back(const T& value) { m_RawList.PushBack(value); }
3164  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
3165  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
3166 
3167 private:
3168  VmaRawList<T> m_RawList;
3169 };
3170 
3171 #endif // #if VMA_USE_STL_LIST
3172 
3174 // class VmaMap
3175 
3176 // Unused in this version.
3177 #if 0
3178 
3179 #if VMA_USE_STL_UNORDERED_MAP
3180 
3181 #define VmaPair std::pair
3182 
3183 #define VMA_MAP_TYPE(KeyT, ValueT) \
3184  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
3185 
3186 #else // #if VMA_USE_STL_UNORDERED_MAP
3187 
3188 template<typename T1, typename T2>
3189 struct VmaPair
3190 {
3191  T1 first;
3192  T2 second;
3193 
3194  VmaPair() : first(), second() { }
3195  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
3196 };
3197 
3198 /* Class compatible with subset of interface of std::unordered_map.
3199 KeyT, ValueT must be POD because they will be stored in VmaVector.
3200 */
3201 template<typename KeyT, typename ValueT>
3202 class VmaMap
3203 {
3204 public:
3205  typedef VmaPair<KeyT, ValueT> PairType;
3206  typedef PairType* iterator;
3207 
3208  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
3209 
3210  iterator begin() { return m_Vector.begin(); }
3211  iterator end() { return m_Vector.end(); }
3212 
3213  void insert(const PairType& pair);
3214  iterator find(const KeyT& key);
3215  void erase(iterator it);
3216 
3217 private:
3218  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
3219 };
3220 
3221 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
3222 
3223 template<typename FirstT, typename SecondT>
3224 struct VmaPairFirstLess
3225 {
3226  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
3227  {
3228  return lhs.first < rhs.first;
3229  }
3230  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
3231  {
3232  return lhs.first < rhsFirst;
3233  }
3234 };
3235 
3236 template<typename KeyT, typename ValueT>
3237 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
3238 {
3239  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3240  m_Vector.data(),
3241  m_Vector.data() + m_Vector.size(),
3242  pair,
3243  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
3244  VmaVectorInsert(m_Vector, indexToInsert, pair);
3245 }
3246 
3247 template<typename KeyT, typename ValueT>
3248 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
3249 {
3250  PairType* it = VmaBinaryFindFirstNotLess(
3251  m_Vector.data(),
3252  m_Vector.data() + m_Vector.size(),
3253  key,
3254  VmaPairFirstLess<KeyT, ValueT>());
3255  if((it != m_Vector.end()) && (it->first == key))
3256  {
3257  return it;
3258  }
3259  else
3260  {
3261  return m_Vector.end();
3262  }
3263 }
3264 
3265 template<typename KeyT, typename ValueT>
3266 void VmaMap<KeyT, ValueT>::erase(iterator it)
3267 {
3268  VmaVectorRemove(m_Vector, it - m_Vector.begin());
3269 }
3270 
3271 #endif // #if VMA_USE_STL_UNORDERED_MAP
3272 
3273 #endif // #if 0
3274 
3276 
3277 class VmaDeviceMemoryBlock;
3278 
3279 struct VmaAllocation_T
3280 {
3281 private:
3282  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
3283 
3284  enum FLAGS
3285  {
3286  FLAG_USER_DATA_STRING = 0x01,
3287  };
3288 
3289 public:
3290  enum ALLOCATION_TYPE
3291  {
3292  ALLOCATION_TYPE_NONE,
3293  ALLOCATION_TYPE_BLOCK,
3294  ALLOCATION_TYPE_DEDICATED,
3295  };
3296 
3297  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
3298  m_Alignment(1),
3299  m_Size(0),
3300  m_pUserData(VMA_NULL),
3301  m_LastUseFrameIndex(currentFrameIndex),
3302  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
3303  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
3304  m_MapCount(0),
3305  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
3306  {
3307  }
3308 
3309  ~VmaAllocation_T()
3310  {
3311  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
3312 
3313  // Check if owned string was freed.
3314  VMA_ASSERT(m_pUserData == VMA_NULL);
3315  }
3316 
3317  void InitBlockAllocation(
3318  VmaPool hPool,
3319  VmaDeviceMemoryBlock* block,
3320  VkDeviceSize offset,
3321  VkDeviceSize alignment,
3322  VkDeviceSize size,
3323  VmaSuballocationType suballocationType,
3324  bool mapped,
3325  bool canBecomeLost)
3326  {
3327  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3328  VMA_ASSERT(block != VMA_NULL);
3329  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3330  m_Alignment = alignment;
3331  m_Size = size;
3332  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3333  m_SuballocationType = (uint8_t)suballocationType;
3334  m_BlockAllocation.m_hPool = hPool;
3335  m_BlockAllocation.m_Block = block;
3336  m_BlockAllocation.m_Offset = offset;
3337  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
3338  }
3339 
3340  void InitLost()
3341  {
3342  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3343  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
3344  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3345  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
3346  m_BlockAllocation.m_Block = VMA_NULL;
3347  m_BlockAllocation.m_Offset = 0;
3348  m_BlockAllocation.m_CanBecomeLost = true;
3349  }
3350 
3351  void ChangeBlockAllocation(
3352  VmaAllocator hAllocator,
3353  VmaDeviceMemoryBlock* block,
3354  VkDeviceSize offset);
3355 
3356  // pMappedData not null means allocation is created with MAPPED flag.
3357  void InitDedicatedAllocation(
3358  uint32_t memoryTypeIndex,
3359  VkDeviceMemory hMemory,
3360  VmaSuballocationType suballocationType,
3361  void* pMappedData,
3362  VkDeviceSize size)
3363  {
3364  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3365  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
3366  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
3367  m_Alignment = 0;
3368  m_Size = size;
3369  m_SuballocationType = (uint8_t)suballocationType;
3370  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3371  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
3372  m_DedicatedAllocation.m_hMemory = hMemory;
3373  m_DedicatedAllocation.m_pMappedData = pMappedData;
3374  }
3375 
3376  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
3377  VkDeviceSize GetAlignment() const { return m_Alignment; }
3378  VkDeviceSize GetSize() const { return m_Size; }
3379  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
3380  void* GetUserData() const { return m_pUserData; }
3381  void SetUserData(VmaAllocator hAllocator, void* pUserData);
3382  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
3383 
3384  VmaDeviceMemoryBlock* GetBlock() const
3385  {
3386  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
3387  return m_BlockAllocation.m_Block;
3388  }
3389  VkDeviceSize GetOffset() const;
3390  VkDeviceMemory GetMemory() const;
3391  uint32_t GetMemoryTypeIndex() const;
3392  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
3393  void* GetMappedData() const;
3394  bool CanBecomeLost() const;
3395  VmaPool GetPool() const;
3396 
3397  uint32_t GetLastUseFrameIndex() const
3398  {
3399  return m_LastUseFrameIndex.load();
3400  }
3401  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
3402  {
3403  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
3404  }
3405  /*
3406  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
3407  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
3408  - Else, returns false.
3409 
3410  If hAllocation is already lost, assert - you should not call it then.
3411  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
3412  */
3413  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3414 
3415  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
3416  {
3417  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
3418  outInfo.blockCount = 1;
3419  outInfo.allocationCount = 1;
3420  outInfo.unusedRangeCount = 0;
3421  outInfo.usedBytes = m_Size;
3422  outInfo.unusedBytes = 0;
3423  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
3424  outInfo.unusedRangeSizeMin = UINT64_MAX;
3425  outInfo.unusedRangeSizeMax = 0;
3426  }
3427 
3428  void BlockAllocMap();
3429  void BlockAllocUnmap();
3430  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
3431  void DedicatedAllocUnmap(VmaAllocator hAllocator);
3432 
3433 private:
3434  VkDeviceSize m_Alignment;
3435  VkDeviceSize m_Size;
3436  void* m_pUserData;
3437  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
3438  uint8_t m_Type; // ALLOCATION_TYPE
3439  uint8_t m_SuballocationType; // VmaSuballocationType
3440  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
3441  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
3442  uint8_t m_MapCount;
3443  uint8_t m_Flags; // enum FLAGS
3444 
3445  // Allocation out of VmaDeviceMemoryBlock.
3446  struct BlockAllocation
3447  {
3448  VmaPool m_hPool; // Null if belongs to general memory.
3449  VmaDeviceMemoryBlock* m_Block;
3450  VkDeviceSize m_Offset;
3451  bool m_CanBecomeLost;
3452  };
3453 
3454  // Allocation for an object that has its own private VkDeviceMemory.
3455  struct DedicatedAllocation
3456  {
3457  uint32_t m_MemoryTypeIndex;
3458  VkDeviceMemory m_hMemory;
3459  void* m_pMappedData; // Not null means memory is mapped.
3460  };
3461 
3462  union
3463  {
3464  // Allocation out of VmaDeviceMemoryBlock.
3465  BlockAllocation m_BlockAllocation;
3466  // Allocation for an object that has its own private VkDeviceMemory.
3467  DedicatedAllocation m_DedicatedAllocation;
3468  };
3469 
3470  void FreeUserDataString(VmaAllocator hAllocator);
3471 };
3472 
3473 /*
3474 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
3475 allocated memory block or free.
3476 */
3477 struct VmaSuballocation
3478 {
3479  VkDeviceSize offset;
3480  VkDeviceSize size;
3481  VmaAllocation hAllocation;
3482  VmaSuballocationType type;
3483 };
3484 
3485 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
3486 
3487 // Cost of one additional allocation lost, as equivalent in bytes.
3488 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
3489 
3490 /*
3491 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3492 
3493 If canMakeOtherLost was false:
3494 - item points to a FREE suballocation.
3495 - itemsToMakeLostCount is 0.
3496 
3497 If canMakeOtherLost was true:
3498 - item points to first of sequence of suballocations, which are either FREE,
3499  or point to VmaAllocations that can become lost.
3500 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3501  the requested allocation to succeed.
3502 */
3503 struct VmaAllocationRequest
3504 {
3505  VkDeviceSize offset;
3506  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3507  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3508  VmaSuballocationList::iterator item;
3509  size_t itemsToMakeLostCount;
3510 
3511  VkDeviceSize CalcCost() const
3512  {
3513  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3514  }
3515 };
3516 
3517 /*
3518 Data structure used for bookkeeping of allocations and unused ranges of memory
3519 in a single VkDeviceMemory block.
3520 */
3521 class VmaBlockMetadata
3522 {
3523 public:
3524  VmaBlockMetadata(VmaAllocator hAllocator);
3525  ~VmaBlockMetadata();
3526  void Init(VkDeviceSize size);
3527 
3528  // Validates all data structures inside this object. If not valid, returns false.
3529  bool Validate() const;
3530  VkDeviceSize GetSize() const { return m_Size; }
3531  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
3532  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3533  VkDeviceSize GetUnusedRangeSizeMax() const;
3534  // Returns true if this block is empty - contains only single free suballocation.
3535  bool IsEmpty() const;
3536 
3537  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3538  void AddPoolStats(VmaPoolStats& inoutStats) const;
3539 
3540 #if VMA_STATS_STRING_ENABLED
3541  void PrintDetailedMap(class VmaJsonWriter& json) const;
3542 #endif
3543 
3544  // Creates trivial request for case when block is empty.
3545  void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3546 
3547  // Tries to find a place for suballocation with given parameters inside this block.
3548  // If succeeded, fills pAllocationRequest and returns true.
3549  // If failed, returns false.
3550  bool CreateAllocationRequest(
3551  uint32_t currentFrameIndex,
3552  uint32_t frameInUseCount,
3553  VkDeviceSize bufferImageGranularity,
3554  VkDeviceSize allocSize,
3555  VkDeviceSize allocAlignment,
3556  VmaSuballocationType allocType,
3557  bool canMakeOtherLost,
3558  VmaAllocationRequest* pAllocationRequest);
3559 
3560  bool MakeRequestedAllocationsLost(
3561  uint32_t currentFrameIndex,
3562  uint32_t frameInUseCount,
3563  VmaAllocationRequest* pAllocationRequest);
3564 
3565  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3566 
3567  // Makes actual allocation based on request. Request must already be checked and valid.
3568  void Alloc(
3569  const VmaAllocationRequest& request,
3570  VmaSuballocationType type,
3571  VkDeviceSize allocSize,
3572  VmaAllocation hAllocation);
3573 
3574  // Frees suballocation assigned to given memory region.
3575  void Free(const VmaAllocation allocation);
3576  void FreeAtOffset(VkDeviceSize offset);
3577 
3578 private:
3579  VkDeviceSize m_Size;
3580  uint32_t m_FreeCount;
3581  VkDeviceSize m_SumFreeSize;
3582  VmaSuballocationList m_Suballocations;
3583  // Suballocations that are free and have size greater than certain threshold.
3584  // Sorted by size, ascending.
3585  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
3586 
3587  bool ValidateFreeSuballocationList() const;
3588 
3589  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
3590  // If yes, fills pOffset and returns true. If no, returns false.
3591  bool CheckAllocation(
3592  uint32_t currentFrameIndex,
3593  uint32_t frameInUseCount,
3594  VkDeviceSize bufferImageGranularity,
3595  VkDeviceSize allocSize,
3596  VkDeviceSize allocAlignment,
3597  VmaSuballocationType allocType,
3598  VmaSuballocationList::const_iterator suballocItem,
3599  bool canMakeOtherLost,
3600  VkDeviceSize* pOffset,
3601  size_t* itemsToMakeLostCount,
3602  VkDeviceSize* pSumFreeSize,
3603  VkDeviceSize* pSumItemSize) const;
3604  // Given free suballocation, it merges it with following one, which must also be free.
3605  void MergeFreeWithNext(VmaSuballocationList::iterator item);
3606  // Releases given suballocation, making it free.
3607  // Merges it with adjacent free suballocations if applicable.
3608  // Returns iterator to new free suballocation at this place.
3609  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
3610  // Given free suballocation, it inserts it into sorted list of
3611  // m_FreeSuballocationsBySize if it's suitable.
3612  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
3613  // Given free suballocation, it removes it from sorted list of
3614  // m_FreeSuballocationsBySize if it's suitable.
3615  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
3616 };
3617 
3618 // Helper class that represents mapped memory. Synchronized internally.
3619 class VmaDeviceMemoryMapping
3620 {
3621 public:
3622  VmaDeviceMemoryMapping();
3623  ~VmaDeviceMemoryMapping();
3624 
3625  void* GetMappedData() const { return m_pMappedData; }
3626 
3627  // ppData can be null.
3628  VkResult Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData);
3629  void Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count);
3630 
3631 private:
3632  VMA_MUTEX m_Mutex;
3633  uint32_t m_MapCount;
3634  void* m_pMappedData;
3635 };
3636 
3637 /*
3638 Represents a single block of device memory (`VkDeviceMemory`) with all the
3639 data about its regions (aka suballocations, `VmaAllocation`), assigned and free.
3640 
3641 Thread-safety: This class must be externally synchronized.
3642 */
3643 class VmaDeviceMemoryBlock
3644 {
3645 public:
3646  uint32_t m_MemoryTypeIndex;
3647  VkDeviceMemory m_hMemory;
3648  VmaDeviceMemoryMapping m_Mapping;
3649  VmaBlockMetadata m_Metadata;
3650 
3651  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
3652 
3653  ~VmaDeviceMemoryBlock()
3654  {
3655  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
3656  }
3657 
3658  // Always call after construction.
3659  void Init(
3660  uint32_t newMemoryTypeIndex,
3661  VkDeviceMemory newMemory,
3662  VkDeviceSize newSize);
3663  // Always call before destruction.
3664  void Destroy(VmaAllocator allocator);
3665 
3666  // Validates all data structures inside this object. If not valid, returns false.
3667  bool Validate() const;
3668 
3669  // ppData can be null.
3670  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
3671  void Unmap(VmaAllocator hAllocator, uint32_t count);
3672 };
3673 
3674 struct VmaPointerLess
3675 {
3676  bool operator()(const void* lhs, const void* rhs) const
3677  {
3678  return lhs < rhs;
3679  }
3680 };
3681 
3682 class VmaDefragmentator;
3683 
3684 /*
3685 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
3686 Vulkan memory type.
3687 
3688 Synchronized internally with a mutex.
3689 */
3690 struct VmaBlockVector
3691 {
3692  VmaBlockVector(
3693  VmaAllocator hAllocator,
3694  uint32_t memoryTypeIndex,
3695  VkDeviceSize preferredBlockSize,
3696  size_t minBlockCount,
3697  size_t maxBlockCount,
3698  VkDeviceSize bufferImageGranularity,
3699  uint32_t frameInUseCount,
3700  bool isCustomPool);
3701  ~VmaBlockVector();
3702 
3703  VkResult CreateMinBlocks();
3704 
3705  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
3706  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
3707  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
3708  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
3709 
3710  void GetPoolStats(VmaPoolStats* pStats);
3711 
3712  bool IsEmpty() const { return m_Blocks.empty(); }
3713 
3714  VkResult Allocate(
3715  VmaPool hCurrentPool,
3716  uint32_t currentFrameIndex,
3717  const VkMemoryRequirements& vkMemReq,
3718  const VmaAllocationCreateInfo& createInfo,
3719  VmaSuballocationType suballocType,
3720  VmaAllocation* pAllocation);
3721 
3722  void Free(
3723  VmaAllocation hAllocation);
3724 
3725  // Adds statistics of this BlockVector to pStats.
3726  void AddStats(VmaStats* pStats);
3727 
3728 #if VMA_STATS_STRING_ENABLED
3729  void PrintDetailedMap(class VmaJsonWriter& json);
3730 #endif
3731 
3732  void MakePoolAllocationsLost(
3733  uint32_t currentFrameIndex,
3734  size_t* pLostAllocationCount);
3735 
3736  VmaDefragmentator* EnsureDefragmentator(
3737  VmaAllocator hAllocator,
3738  uint32_t currentFrameIndex);
3739 
3740  VkResult Defragment(
3741  VmaDefragmentationStats* pDefragmentationStats,
3742  VkDeviceSize& maxBytesToMove,
3743  uint32_t& maxAllocationsToMove);
3744 
3745  void DestroyDefragmentator();
3746 
3747 private:
3748  friend class VmaDefragmentator;
3749 
3750  const VmaAllocator m_hAllocator;
3751  const uint32_t m_MemoryTypeIndex;
3752  const VkDeviceSize m_PreferredBlockSize;
3753  const size_t m_MinBlockCount;
3754  const size_t m_MaxBlockCount;
3755  const VkDeviceSize m_BufferImageGranularity;
3756  const uint32_t m_FrameInUseCount;
3757  const bool m_IsCustomPool;
3758  VMA_MUTEX m_Mutex;
3759  // Incrementally sorted by sumFreeSize, ascending.
3760  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
3761  /* There can be at most one allocation that is completely empty - a
3762  hysteresis to avoid pessimistic case of alternating creation and destruction
3763  of a VkDeviceMemory. */
3764  bool m_HasEmptyBlock;
3765  VmaDefragmentator* m_pDefragmentator;
3766 
3767  size_t CalcMaxBlockSize() const;
3768 
3769  // Finds and removes given block from vector.
3770  void Remove(VmaDeviceMemoryBlock* pBlock);
3771 
3772  // Performs single step in sorting m_Blocks. They may not be fully sorted
3773  // after this call.
3774  void IncrementallySortBlocks();
3775 
3776  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
3777 };
3778 
3779 struct VmaPool_T
3780 {
3781 public:
3782  VmaBlockVector m_BlockVector;
3783 
3784  // Takes ownership.
3785  VmaPool_T(
3786  VmaAllocator hAllocator,
3787  const VmaPoolCreateInfo& createInfo);
3788  ~VmaPool_T();
3789 
3790  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
3791 
3792 #if VMA_STATS_STRING_ENABLED
3793  //void PrintDetailedMap(class VmaStringBuilder& sb);
3794 #endif
3795 };
3796 
3797 class VmaDefragmentator
3798 {
3799  const VmaAllocator m_hAllocator;
3800  VmaBlockVector* const m_pBlockVector;
3801  uint32_t m_CurrentFrameIndex;
3802  VkDeviceSize m_BytesMoved;
3803  uint32_t m_AllocationsMoved;
3804 
3805  struct AllocationInfo
3806  {
3807  VmaAllocation m_hAllocation;
3808  VkBool32* m_pChanged;
3809 
3810  AllocationInfo() :
3811  m_hAllocation(VK_NULL_HANDLE),
3812  m_pChanged(VMA_NULL)
3813  {
3814  }
3815  };
3816 
3817  struct AllocationInfoSizeGreater
3818  {
3819  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
3820  {
3821  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
3822  }
3823  };
3824 
3825  // Used between AddAllocation and Defragment.
3826  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3827 
3828  struct BlockInfo
3829  {
3830  VmaDeviceMemoryBlock* m_pBlock;
3831  bool m_HasNonMovableAllocations;
3832  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3833 
3834  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
3835  m_pBlock(VMA_NULL),
3836  m_HasNonMovableAllocations(true),
3837  m_Allocations(pAllocationCallbacks),
3838  m_pMappedDataForDefragmentation(VMA_NULL)
3839  {
3840  }
3841 
3842  void CalcHasNonMovableAllocations()
3843  {
3844  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
3845  const size_t defragmentAllocCount = m_Allocations.size();
3846  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
3847  }
3848 
3849  void SortAllocationsBySizeDescecnding()
3850  {
3851  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
3852  }
3853 
3854  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
3855  void Unmap(VmaAllocator hAllocator);
3856 
3857  private:
3858  // Not null if mapped for defragmentation only, not originally mapped.
3859  void* m_pMappedDataForDefragmentation;
3860  };
3861 
3862  struct BlockPointerLess
3863  {
3864  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
3865  {
3866  return pLhsBlockInfo->m_pBlock < pRhsBlock;
3867  }
3868  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3869  {
3870  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
3871  }
3872  };
3873 
3874  // 1. Blocks with some non-movable allocations go first.
3875  // 2. Blocks with smaller sumFreeSize go first.
3876  struct BlockInfoCompareMoveDestination
3877  {
3878  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3879  {
3880  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
3881  {
3882  return true;
3883  }
3884  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
3885  {
3886  return false;
3887  }
3888  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
3889  {
3890  return true;
3891  }
3892  return false;
3893  }
3894  };
3895 
3896  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
3897  BlockInfoVector m_Blocks;
3898 
3899  VkResult DefragmentRound(
3900  VkDeviceSize maxBytesToMove,
3901  uint32_t maxAllocationsToMove);
3902 
3903  static bool MoveMakesSense(
3904  size_t dstBlockIndex, VkDeviceSize dstOffset,
3905  size_t srcBlockIndex, VkDeviceSize srcOffset);
3906 
3907 public:
3908  VmaDefragmentator(
3909  VmaAllocator hAllocator,
3910  VmaBlockVector* pBlockVector,
3911  uint32_t currentFrameIndex);
3912 
3913  ~VmaDefragmentator();
3914 
3915  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
3916  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
3917 
3918  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
3919 
3920  VkResult Defragment(
3921  VkDeviceSize maxBytesToMove,
3922  uint32_t maxAllocationsToMove);
3923 };
3924 
3925 // Main allocator object.
3926 struct VmaAllocator_T
3927 {
3928  bool m_UseMutex;
3929  bool m_UseKhrDedicatedAllocation;
3930  VkDevice m_hDevice;
3931  bool m_AllocationCallbacksSpecified;
3932  VkAllocationCallbacks m_AllocationCallbacks;
3933  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
3934 
3935  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
3936  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
3937  VMA_MUTEX m_HeapSizeLimitMutex;
3938 
3939  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
3940  VkPhysicalDeviceMemoryProperties m_MemProps;
3941 
3942  // Default pools.
3943  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
3944 
3945  // Each vector is sorted by memory (handle value).
3946  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
3947  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
3948  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
3949 
3950  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
3951  ~VmaAllocator_T();
3952 
3953  const VkAllocationCallbacks* GetAllocationCallbacks() const
3954  {
3955  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
3956  }
3957  const VmaVulkanFunctions& GetVulkanFunctions() const
3958  {
3959  return m_VulkanFunctions;
3960  }
3961 
3962  VkDeviceSize GetBufferImageGranularity() const
3963  {
3964  return VMA_MAX(
3965  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
3966  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
3967  }
3968 
3969  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
3970  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
3971 
3972  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
3973  {
3974  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
3975  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
3976  }
3977 
3978  void GetBufferMemoryRequirements(
3979  VkBuffer hBuffer,
3980  VkMemoryRequirements& memReq,
3981  bool& requiresDedicatedAllocation,
3982  bool& prefersDedicatedAllocation) const;
3983  void GetImageMemoryRequirements(
3984  VkImage hImage,
3985  VkMemoryRequirements& memReq,
3986  bool& requiresDedicatedAllocation,
3987  bool& prefersDedicatedAllocation) const;
3988 
3989  // Main allocation function.
3990  VkResult AllocateMemory(
3991  const VkMemoryRequirements& vkMemReq,
3992  bool requiresDedicatedAllocation,
3993  bool prefersDedicatedAllocation,
3994  VkBuffer dedicatedBuffer,
3995  VkImage dedicatedImage,
3996  const VmaAllocationCreateInfo& createInfo,
3997  VmaSuballocationType suballocType,
3998  VmaAllocation* pAllocation);
3999 
4000  // Main deallocation function.
4001  void FreeMemory(const VmaAllocation allocation);
4002 
4003  void CalculateStats(VmaStats* pStats);
4004 
4005 #if VMA_STATS_STRING_ENABLED
4006  void PrintDetailedMap(class VmaJsonWriter& json);
4007 #endif
4008 
4009  VkResult Defragment(
4010  VmaAllocation* pAllocations,
4011  size_t allocationCount,
4012  VkBool32* pAllocationsChanged,
4013  const VmaDefragmentationInfo* pDefragmentationInfo,
4014  VmaDefragmentationStats* pDefragmentationStats);
4015 
4016  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
4017 
4018  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
4019  void DestroyPool(VmaPool pool);
4020  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
4021 
4022  void SetCurrentFrameIndex(uint32_t frameIndex);
4023 
4024  void MakePoolAllocationsLost(
4025  VmaPool hPool,
4026  size_t* pLostAllocationCount);
4027 
4028  void CreateLostAllocation(VmaAllocation* pAllocation);
4029 
4030  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
4031  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
4032 
4033  VkResult Map(VmaAllocation hAllocation, void** ppData);
4034  void Unmap(VmaAllocation hAllocation);
4035 
4036 private:
4037  VkDeviceSize m_PreferredLargeHeapBlockSize;
4038 
4039  VkPhysicalDevice m_PhysicalDevice;
4040  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
4041 
4042  VMA_MUTEX m_PoolsMutex;
4043  // Protected by m_PoolsMutex. Sorted by pointer value.
4044  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
4045 
4046  VmaVulkanFunctions m_VulkanFunctions;
4047 
4048  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
4049 
4050  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
4051 
4052  VkResult AllocateMemoryOfType(
4053  const VkMemoryRequirements& vkMemReq,
4054  bool dedicatedAllocation,
4055  VkBuffer dedicatedBuffer,
4056  VkImage dedicatedImage,
4057  const VmaAllocationCreateInfo& createInfo,
4058  uint32_t memTypeIndex,
4059  VmaSuballocationType suballocType,
4060  VmaAllocation* pAllocation);
4061 
4062  // Allocates and registers new VkDeviceMemory specifically for single allocation.
4063  VkResult AllocateDedicatedMemory(
4064  VkDeviceSize size,
4065  VmaSuballocationType suballocType,
4066  uint32_t memTypeIndex,
4067  bool map,
4068  bool isUserDataString,
4069  void* pUserData,
4070  VkBuffer dedicatedBuffer,
4071  VkImage dedicatedImage,
4072  VmaAllocation* pAllocation);
4073 
4074  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
4075  void FreeDedicatedMemory(VmaAllocation allocation);
4076 };
4077 
4079 // Memory allocation #2 after VmaAllocator_T definition
4080 
4081 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
4082 {
4083  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
4084 }
4085 
4086 static void VmaFree(VmaAllocator hAllocator, void* ptr)
4087 {
4088  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
4089 }
4090 
4091 template<typename T>
4092 static T* VmaAllocate(VmaAllocator hAllocator)
4093 {
4094  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
4095 }
4096 
4097 template<typename T>
4098 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
4099 {
4100  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
4101 }
4102 
4103 template<typename T>
4104 static void vma_delete(VmaAllocator hAllocator, T* ptr)
4105 {
4106  if(ptr != VMA_NULL)
4107  {
4108  ptr->~T();
4109  VmaFree(hAllocator, ptr);
4110  }
4111 }
4112 
4113 template<typename T>
4114 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
4115 {
4116  if(ptr != VMA_NULL)
4117  {
4118  for(size_t i = count; i--; )
4119  ptr[i].~T();
4120  VmaFree(hAllocator, ptr);
4121  }
4122 }
4123 
4125 // VmaStringBuilder
4126 
4127 #if VMA_STATS_STRING_ENABLED
4128 
4129 class VmaStringBuilder
4130 {
4131 public:
4132  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
4133  size_t GetLength() const { return m_Data.size(); }
4134  const char* GetData() const { return m_Data.data(); }
4135 
4136  void Add(char ch) { m_Data.push_back(ch); }
4137  void Add(const char* pStr);
4138  void AddNewLine() { Add('\n'); }
4139  void AddNumber(uint32_t num);
4140  void AddNumber(uint64_t num);
4141  void AddPointer(const void* ptr);
4142 
4143 private:
4144  VmaVector< char, VmaStlAllocator<char> > m_Data;
4145 };
4146 
4147 void VmaStringBuilder::Add(const char* pStr)
4148 {
4149  const size_t strLen = strlen(pStr);
4150  if(strLen > 0)
4151  {
4152  const size_t oldCount = m_Data.size();
4153  m_Data.resize(oldCount + strLen);
4154  memcpy(m_Data.data() + oldCount, pStr, strLen);
4155  }
4156 }
4157 
4158 void VmaStringBuilder::AddNumber(uint32_t num)
4159 {
4160  char buf[11];
4161  VmaUint32ToStr(buf, sizeof(buf), num);
4162  Add(buf);
4163 }
4164 
4165 void VmaStringBuilder::AddNumber(uint64_t num)
4166 {
4167  char buf[21];
4168  VmaUint64ToStr(buf, sizeof(buf), num);
4169  Add(buf);
4170 }
4171 
4172 void VmaStringBuilder::AddPointer(const void* ptr)
4173 {
4174  char buf[21];
4175  VmaPtrToStr(buf, sizeof(buf), ptr);
4176  Add(buf);
4177 }
4178 
4179 #endif // #if VMA_STATS_STRING_ENABLED
4180 
4182 // VmaJsonWriter
4183 
4184 #if VMA_STATS_STRING_ENABLED
4185 
4186 class VmaJsonWriter
4187 {
4188 public:
4189  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
4190  ~VmaJsonWriter();
4191 
4192  void BeginObject(bool singleLine = false);
4193  void EndObject();
4194 
4195  void BeginArray(bool singleLine = false);
4196  void EndArray();
4197 
4198  void WriteString(const char* pStr);
4199  void BeginString(const char* pStr = VMA_NULL);
4200  void ContinueString(const char* pStr);
4201  void ContinueString(uint32_t n);
4202  void ContinueString(uint64_t n);
4203  void ContinueString_Pointer(const void* ptr);
4204  void EndString(const char* pStr = VMA_NULL);
4205 
4206  void WriteNumber(uint32_t n);
4207  void WriteNumber(uint64_t n);
4208  void WriteBool(bool b);
4209  void WriteNull();
4210 
4211 private:
4212  static const char* const INDENT;
4213 
4214  enum COLLECTION_TYPE
4215  {
4216  COLLECTION_TYPE_OBJECT,
4217  COLLECTION_TYPE_ARRAY,
4218  };
4219  struct StackItem
4220  {
4221  COLLECTION_TYPE type;
4222  uint32_t valueCount;
4223  bool singleLineMode;
4224  };
4225 
4226  VmaStringBuilder& m_SB;
4227  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
4228  bool m_InsideString;
4229 
4230  void BeginValue(bool isString);
4231  void WriteIndent(bool oneLess = false);
4232 };
4233 
4234 const char* const VmaJsonWriter::INDENT = " ";
4235 
4236 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
4237  m_SB(sb),
4238  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
4239  m_InsideString(false)
4240 {
4241 }
4242 
4243 VmaJsonWriter::~VmaJsonWriter()
4244 {
4245  VMA_ASSERT(!m_InsideString);
4246  VMA_ASSERT(m_Stack.empty());
4247 }
4248 
4249 void VmaJsonWriter::BeginObject(bool singleLine)
4250 {
4251  VMA_ASSERT(!m_InsideString);
4252 
4253  BeginValue(false);
4254  m_SB.Add('{');
4255 
4256  StackItem item;
4257  item.type = COLLECTION_TYPE_OBJECT;
4258  item.valueCount = 0;
4259  item.singleLineMode = singleLine;
4260  m_Stack.push_back(item);
4261 }
4262 
4263 void VmaJsonWriter::EndObject()
4264 {
4265  VMA_ASSERT(!m_InsideString);
4266 
4267  WriteIndent(true);
4268  m_SB.Add('}');
4269 
4270  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
4271  m_Stack.pop_back();
4272 }
4273 
4274 void VmaJsonWriter::BeginArray(bool singleLine)
4275 {
4276  VMA_ASSERT(!m_InsideString);
4277 
4278  BeginValue(false);
4279  m_SB.Add('[');
4280 
4281  StackItem item;
4282  item.type = COLLECTION_TYPE_ARRAY;
4283  item.valueCount = 0;
4284  item.singleLineMode = singleLine;
4285  m_Stack.push_back(item);
4286 }
4287 
4288 void VmaJsonWriter::EndArray()
4289 {
4290  VMA_ASSERT(!m_InsideString);
4291 
4292  WriteIndent(true);
4293  m_SB.Add(']');
4294 
4295  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
4296  m_Stack.pop_back();
4297 }
4298 
4299 void VmaJsonWriter::WriteString(const char* pStr)
4300 {
4301  BeginString(pStr);
4302  EndString();
4303 }
4304 
4305 void VmaJsonWriter::BeginString(const char* pStr)
4306 {
4307  VMA_ASSERT(!m_InsideString);
4308 
4309  BeginValue(true);
4310  m_SB.Add('"');
4311  m_InsideString = true;
4312  if(pStr != VMA_NULL && pStr[0] != '\0')
4313  {
4314  ContinueString(pStr);
4315  }
4316 }
4317 
4318 void VmaJsonWriter::ContinueString(const char* pStr)
4319 {
4320  VMA_ASSERT(m_InsideString);
4321 
4322  const size_t strLen = strlen(pStr);
4323  for(size_t i = 0; i < strLen; ++i)
4324  {
4325  char ch = pStr[i];
4326  if(ch == '\'')
4327  {
4328  m_SB.Add("\\\\");
4329  }
4330  else if(ch == '"')
4331  {
4332  m_SB.Add("\\\"");
4333  }
4334  else if(ch >= 32)
4335  {
4336  m_SB.Add(ch);
4337  }
4338  else switch(ch)
4339  {
4340  case '\b':
4341  m_SB.Add("\\b");
4342  break;
4343  case '\f':
4344  m_SB.Add("\\f");
4345  break;
4346  case '\n':
4347  m_SB.Add("\\n");
4348  break;
4349  case '\r':
4350  m_SB.Add("\\r");
4351  break;
4352  case '\t':
4353  m_SB.Add("\\t");
4354  break;
4355  default:
4356  VMA_ASSERT(0 && "Character not currently supported.");
4357  break;
4358  }
4359  }
4360 }
4361 
4362 void VmaJsonWriter::ContinueString(uint32_t n)
4363 {
4364  VMA_ASSERT(m_InsideString);
4365  m_SB.AddNumber(n);
4366 }
4367 
4368 void VmaJsonWriter::ContinueString(uint64_t n)
4369 {
4370  VMA_ASSERT(m_InsideString);
4371  m_SB.AddNumber(n);
4372 }
4373 
4374 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
4375 {
4376  VMA_ASSERT(m_InsideString);
4377  m_SB.AddPointer(ptr);
4378 }
4379 
4380 void VmaJsonWriter::EndString(const char* pStr)
4381 {
4382  VMA_ASSERT(m_InsideString);
4383  if(pStr != VMA_NULL && pStr[0] != '\0')
4384  {
4385  ContinueString(pStr);
4386  }
4387  m_SB.Add('"');
4388  m_InsideString = false;
4389 }
4390 
4391 void VmaJsonWriter::WriteNumber(uint32_t n)
4392 {
4393  VMA_ASSERT(!m_InsideString);
4394  BeginValue(false);
4395  m_SB.AddNumber(n);
4396 }
4397 
4398 void VmaJsonWriter::WriteNumber(uint64_t n)
4399 {
4400  VMA_ASSERT(!m_InsideString);
4401  BeginValue(false);
4402  m_SB.AddNumber(n);
4403 }
4404 
4405 void VmaJsonWriter::WriteBool(bool b)
4406 {
4407  VMA_ASSERT(!m_InsideString);
4408  BeginValue(false);
4409  m_SB.Add(b ? "true" : "false");
4410 }
4411 
4412 void VmaJsonWriter::WriteNull()
4413 {
4414  VMA_ASSERT(!m_InsideString);
4415  BeginValue(false);
4416  m_SB.Add("null");
4417 }
4418 
4419 void VmaJsonWriter::BeginValue(bool isString)
4420 {
4421  if(!m_Stack.empty())
4422  {
4423  StackItem& currItem = m_Stack.back();
4424  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4425  currItem.valueCount % 2 == 0)
4426  {
4427  VMA_ASSERT(isString);
4428  }
4429 
4430  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4431  currItem.valueCount % 2 != 0)
4432  {
4433  m_SB.Add(": ");
4434  }
4435  else if(currItem.valueCount > 0)
4436  {
4437  m_SB.Add(", ");
4438  WriteIndent();
4439  }
4440  else
4441  {
4442  WriteIndent();
4443  }
4444  ++currItem.valueCount;
4445  }
4446 }
4447 
4448 void VmaJsonWriter::WriteIndent(bool oneLess)
4449 {
4450  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
4451  {
4452  m_SB.AddNewLine();
4453 
4454  size_t count = m_Stack.size();
4455  if(count > 0 && oneLess)
4456  {
4457  --count;
4458  }
4459  for(size_t i = 0; i < count; ++i)
4460  {
4461  m_SB.Add(INDENT);
4462  }
4463  }
4464 }
4465 
4466 #endif // #if VMA_STATS_STRING_ENABLED
4467 
4469 
4470 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
4471 {
4472  if(IsUserDataString())
4473  {
4474  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
4475 
4476  FreeUserDataString(hAllocator);
4477 
4478  if(pUserData != VMA_NULL)
4479  {
4480  const char* const newStrSrc = (char*)pUserData;
4481  const size_t newStrLen = strlen(newStrSrc);
4482  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
4483  memcpy(newStrDst, newStrSrc, newStrLen + 1);
4484  m_pUserData = newStrDst;
4485  }
4486  }
4487  else
4488  {
4489  m_pUserData = pUserData;
4490  }
4491 }
4492 
4493 void VmaAllocation_T::ChangeBlockAllocation(
4494  VmaAllocator hAllocator,
4495  VmaDeviceMemoryBlock* block,
4496  VkDeviceSize offset)
4497 {
4498  VMA_ASSERT(block != VMA_NULL);
4499  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4500 
4501  // Move mapping reference counter from old block to new block.
4502  if(block != m_BlockAllocation.m_Block)
4503  {
4504  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
4505  if(IsPersistentMap())
4506  ++mapRefCount;
4507  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
4508  block->Map(hAllocator, mapRefCount, VMA_NULL);
4509  }
4510 
4511  m_BlockAllocation.m_Block = block;
4512  m_BlockAllocation.m_Offset = offset;
4513 }
4514 
4515 VkDeviceSize VmaAllocation_T::GetOffset() const
4516 {
4517  switch(m_Type)
4518  {
4519  case ALLOCATION_TYPE_BLOCK:
4520  return m_BlockAllocation.m_Offset;
4521  case ALLOCATION_TYPE_DEDICATED:
4522  return 0;
4523  default:
4524  VMA_ASSERT(0);
4525  return 0;
4526  }
4527 }
4528 
4529 VkDeviceMemory VmaAllocation_T::GetMemory() const
4530 {
4531  switch(m_Type)
4532  {
4533  case ALLOCATION_TYPE_BLOCK:
4534  return m_BlockAllocation.m_Block->m_hMemory;
4535  case ALLOCATION_TYPE_DEDICATED:
4536  return m_DedicatedAllocation.m_hMemory;
4537  default:
4538  VMA_ASSERT(0);
4539  return VK_NULL_HANDLE;
4540  }
4541 }
4542 
4543 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
4544 {
4545  switch(m_Type)
4546  {
4547  case ALLOCATION_TYPE_BLOCK:
4548  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
4549  case ALLOCATION_TYPE_DEDICATED:
4550  return m_DedicatedAllocation.m_MemoryTypeIndex;
4551  default:
4552  VMA_ASSERT(0);
4553  return UINT32_MAX;
4554  }
4555 }
4556 
4557 void* VmaAllocation_T::GetMappedData() const
4558 {
4559  switch(m_Type)
4560  {
4561  case ALLOCATION_TYPE_BLOCK:
4562  if(m_MapCount != 0)
4563  {
4564  void* pBlockData = m_BlockAllocation.m_Block->m_Mapping.GetMappedData();
4565  VMA_ASSERT(pBlockData != VMA_NULL);
4566  return (char*)pBlockData + m_BlockAllocation.m_Offset;
4567  }
4568  else
4569  {
4570  return VMA_NULL;
4571  }
4572  break;
4573  case ALLOCATION_TYPE_DEDICATED:
4574  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
4575  return m_DedicatedAllocation.m_pMappedData;
4576  default:
4577  VMA_ASSERT(0);
4578  return VMA_NULL;
4579  }
4580 }
4581 
4582 bool VmaAllocation_T::CanBecomeLost() const
4583 {
4584  switch(m_Type)
4585  {
4586  case ALLOCATION_TYPE_BLOCK:
4587  return m_BlockAllocation.m_CanBecomeLost;
4588  case ALLOCATION_TYPE_DEDICATED:
4589  return false;
4590  default:
4591  VMA_ASSERT(0);
4592  return false;
4593  }
4594 }
4595 
4596 VmaPool VmaAllocation_T::GetPool() const
4597 {
4598  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4599  return m_BlockAllocation.m_hPool;
4600 }
4601 
4602 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4603 {
4604  VMA_ASSERT(CanBecomeLost());
4605 
4606  /*
4607  Warning: This is a carefully designed algorithm.
4608  Do not modify unless you really know what you're doing :)
4609  */
4610  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
4611  for(;;)
4612  {
4613  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
4614  {
4615  VMA_ASSERT(0);
4616  return false;
4617  }
4618  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
4619  {
4620  return false;
4621  }
4622  else // Last use time earlier than current time.
4623  {
4624  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
4625  {
4626  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
4627  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
4628  return true;
4629  }
4630  }
4631  }
4632 }
4633 
4634 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
4635 {
4636  VMA_ASSERT(IsUserDataString());
4637  if(m_pUserData != VMA_NULL)
4638  {
4639  char* const oldStr = (char*)m_pUserData;
4640  const size_t oldStrLen = strlen(oldStr);
4641  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
4642  m_pUserData = VMA_NULL;
4643  }
4644 }
4645 
4646 void VmaAllocation_T::BlockAllocMap()
4647 {
4648  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4649 
4650  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4651  {
4652  ++m_MapCount;
4653  }
4654  else
4655  {
4656  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
4657  }
4658 }
4659 
4660 void VmaAllocation_T::BlockAllocUnmap()
4661 {
4662  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4663 
4664  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4665  {
4666  --m_MapCount;
4667  }
4668  else
4669  {
4670  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
4671  }
4672 }
4673 
4674 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
4675 {
4676  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4677 
4678  if(m_MapCount != 0)
4679  {
4680  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4681  {
4682  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
4683  *ppData = m_DedicatedAllocation.m_pMappedData;
4684  ++m_MapCount;
4685  return VK_SUCCESS;
4686  }
4687  else
4688  {
4689  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
4690  return VK_ERROR_MEMORY_MAP_FAILED;
4691  }
4692  }
4693  else
4694  {
4695  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
4696  hAllocator->m_hDevice,
4697  m_DedicatedAllocation.m_hMemory,
4698  0, // offset
4699  VK_WHOLE_SIZE,
4700  0, // flags
4701  ppData);
4702  if(result == VK_SUCCESS)
4703  {
4704  m_DedicatedAllocation.m_pMappedData = *ppData;
4705  m_MapCount = 1;
4706  }
4707  return result;
4708  }
4709 }
4710 
4711 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
4712 {
4713  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4714 
4715  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4716  {
4717  --m_MapCount;
4718  if(m_MapCount == 0)
4719  {
4720  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
4721  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
4722  hAllocator->m_hDevice,
4723  m_DedicatedAllocation.m_hMemory);
4724  }
4725  }
4726  else
4727  {
4728  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
4729  }
4730 }
4731 
4732 #if VMA_STATS_STRING_ENABLED
4733 
4734 // Correspond to values of enum VmaSuballocationType.
4735 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
4736  "FREE",
4737  "UNKNOWN",
4738  "BUFFER",
4739  "IMAGE_UNKNOWN",
4740  "IMAGE_LINEAR",
4741  "IMAGE_OPTIMAL",
4742 };
4743 
4744 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
4745 {
4746  json.BeginObject();
4747 
4748  json.WriteString("Blocks");
4749  json.WriteNumber(stat.blockCount);
4750 
4751  json.WriteString("Allocations");
4752  json.WriteNumber(stat.allocationCount);
4753 
4754  json.WriteString("UnusedRanges");
4755  json.WriteNumber(stat.unusedRangeCount);
4756 
4757  json.WriteString("UsedBytes");
4758  json.WriteNumber(stat.usedBytes);
4759 
4760  json.WriteString("UnusedBytes");
4761  json.WriteNumber(stat.unusedBytes);
4762 
4763  if(stat.allocationCount > 1)
4764  {
4765  json.WriteString("AllocationSize");
4766  json.BeginObject(true);
4767  json.WriteString("Min");
4768  json.WriteNumber(stat.allocationSizeMin);
4769  json.WriteString("Avg");
4770  json.WriteNumber(stat.allocationSizeAvg);
4771  json.WriteString("Max");
4772  json.WriteNumber(stat.allocationSizeMax);
4773  json.EndObject();
4774  }
4775 
4776  if(stat.unusedRangeCount > 1)
4777  {
4778  json.WriteString("UnusedRangeSize");
4779  json.BeginObject(true);
4780  json.WriteString("Min");
4781  json.WriteNumber(stat.unusedRangeSizeMin);
4782  json.WriteString("Avg");
4783  json.WriteNumber(stat.unusedRangeSizeAvg);
4784  json.WriteString("Max");
4785  json.WriteNumber(stat.unusedRangeSizeMax);
4786  json.EndObject();
4787  }
4788 
4789  json.EndObject();
4790 }
4791 
4792 #endif // #if VMA_STATS_STRING_ENABLED
4793 
4794 struct VmaSuballocationItemSizeLess
4795 {
4796  bool operator()(
4797  const VmaSuballocationList::iterator lhs,
4798  const VmaSuballocationList::iterator rhs) const
4799  {
4800  return lhs->size < rhs->size;
4801  }
4802  bool operator()(
4803  const VmaSuballocationList::iterator lhs,
4804  VkDeviceSize rhsSize) const
4805  {
4806  return lhs->size < rhsSize;
4807  }
4808 };
4809 
4811 // class VmaBlockMetadata
4812 
4813 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
4814  m_Size(0),
4815  m_FreeCount(0),
4816  m_SumFreeSize(0),
4817  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
4818  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
4819 {
4820 }
4821 
4822 VmaBlockMetadata::~VmaBlockMetadata()
4823 {
4824 }
4825 
4826 void VmaBlockMetadata::Init(VkDeviceSize size)
4827 {
4828  m_Size = size;
4829  m_FreeCount = 1;
4830  m_SumFreeSize = size;
4831 
4832  VmaSuballocation suballoc = {};
4833  suballoc.offset = 0;
4834  suballoc.size = size;
4835  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4836  suballoc.hAllocation = VK_NULL_HANDLE;
4837 
4838  m_Suballocations.push_back(suballoc);
4839  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
4840  --suballocItem;
4841  m_FreeSuballocationsBySize.push_back(suballocItem);
4842 }
4843 
4844 bool VmaBlockMetadata::Validate() const
4845 {
4846  if(m_Suballocations.empty())
4847  {
4848  return false;
4849  }
4850 
4851  // Expected offset of new suballocation as calculates from previous ones.
4852  VkDeviceSize calculatedOffset = 0;
4853  // Expected number of free suballocations as calculated from traversing their list.
4854  uint32_t calculatedFreeCount = 0;
4855  // Expected sum size of free suballocations as calculated from traversing their list.
4856  VkDeviceSize calculatedSumFreeSize = 0;
4857  // Expected number of free suballocations that should be registered in
4858  // m_FreeSuballocationsBySize calculated from traversing their list.
4859  size_t freeSuballocationsToRegister = 0;
4860  // True if previous visisted suballocation was free.
4861  bool prevFree = false;
4862 
4863  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4864  suballocItem != m_Suballocations.cend();
4865  ++suballocItem)
4866  {
4867  const VmaSuballocation& subAlloc = *suballocItem;
4868 
4869  // Actual offset of this suballocation doesn't match expected one.
4870  if(subAlloc.offset != calculatedOffset)
4871  {
4872  return false;
4873  }
4874 
4875  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
4876  // Two adjacent free suballocations are invalid. They should be merged.
4877  if(prevFree && currFree)
4878  {
4879  return false;
4880  }
4881 
4882  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
4883  {
4884  return false;
4885  }
4886 
4887  if(currFree)
4888  {
4889  calculatedSumFreeSize += subAlloc.size;
4890  ++calculatedFreeCount;
4891  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4892  {
4893  ++freeSuballocationsToRegister;
4894  }
4895  }
4896  else
4897  {
4898  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
4899  {
4900  return false;
4901  }
4902  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
4903  {
4904  return false;
4905  }
4906  }
4907 
4908  calculatedOffset += subAlloc.size;
4909  prevFree = currFree;
4910  }
4911 
4912  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
4913  // match expected one.
4914  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
4915  {
4916  return false;
4917  }
4918 
4919  VkDeviceSize lastSize = 0;
4920  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
4921  {
4922  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
4923 
4924  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
4925  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
4926  {
4927  return false;
4928  }
4929  // They must be sorted by size ascending.
4930  if(suballocItem->size < lastSize)
4931  {
4932  return false;
4933  }
4934 
4935  lastSize = suballocItem->size;
4936  }
4937 
4938  // Check if totals match calculacted values.
4939  if(!ValidateFreeSuballocationList() ||
4940  (calculatedOffset != m_Size) ||
4941  (calculatedSumFreeSize != m_SumFreeSize) ||
4942  (calculatedFreeCount != m_FreeCount))
4943  {
4944  return false;
4945  }
4946 
4947  return true;
4948 }
4949 
4950 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
4951 {
4952  if(!m_FreeSuballocationsBySize.empty())
4953  {
4954  return m_FreeSuballocationsBySize.back()->size;
4955  }
4956  else
4957  {
4958  return 0;
4959  }
4960 }
4961 
4962 bool VmaBlockMetadata::IsEmpty() const
4963 {
4964  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
4965 }
4966 
4967 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
4968 {
4969  outInfo.blockCount = 1;
4970 
4971  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
4972  outInfo.allocationCount = rangeCount - m_FreeCount;
4973  outInfo.unusedRangeCount = m_FreeCount;
4974 
4975  outInfo.unusedBytes = m_SumFreeSize;
4976  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
4977 
4978  outInfo.allocationSizeMin = UINT64_MAX;
4979  outInfo.allocationSizeMax = 0;
4980  outInfo.unusedRangeSizeMin = UINT64_MAX;
4981  outInfo.unusedRangeSizeMax = 0;
4982 
4983  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4984  suballocItem != m_Suballocations.cend();
4985  ++suballocItem)
4986  {
4987  const VmaSuballocation& suballoc = *suballocItem;
4988  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
4989  {
4990  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
4991  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
4992  }
4993  else
4994  {
4995  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
4996  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
4997  }
4998  }
4999 }
5000 
5001 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
5002 {
5003  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5004 
5005  inoutStats.size += m_Size;
5006  inoutStats.unusedSize += m_SumFreeSize;
5007  inoutStats.allocationCount += rangeCount - m_FreeCount;
5008  inoutStats.unusedRangeCount += m_FreeCount;
5009  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
5010 }
5011 
5012 #if VMA_STATS_STRING_ENABLED
5013 
5014 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
5015 {
5016  json.BeginObject();
5017 
5018  json.WriteString("TotalBytes");
5019  json.WriteNumber(m_Size);
5020 
5021  json.WriteString("UnusedBytes");
5022  json.WriteNumber(m_SumFreeSize);
5023 
5024  json.WriteString("Allocations");
5025  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
5026 
5027  json.WriteString("UnusedRanges");
5028  json.WriteNumber(m_FreeCount);
5029 
5030  json.WriteString("Suballocations");
5031  json.BeginArray();
5032  size_t i = 0;
5033  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5034  suballocItem != m_Suballocations.cend();
5035  ++suballocItem, ++i)
5036  {
5037  json.BeginObject(true);
5038 
5039  json.WriteString("Type");
5040  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
5041 
5042  json.WriteString("Size");
5043  json.WriteNumber(suballocItem->size);
5044 
5045  json.WriteString("Offset");
5046  json.WriteNumber(suballocItem->offset);
5047 
5048  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5049  {
5050  const void* pUserData = suballocItem->hAllocation->GetUserData();
5051  if(pUserData != VMA_NULL)
5052  {
5053  json.WriteString("UserData");
5054  if(suballocItem->hAllocation->IsUserDataString())
5055  {
5056  json.WriteString((const char*)pUserData);
5057  }
5058  else
5059  {
5060  json.BeginString();
5061  json.ContinueString_Pointer(pUserData);
5062  json.EndString();
5063  }
5064  }
5065  }
5066 
5067  json.EndObject();
5068  }
5069  json.EndArray();
5070 
5071  json.EndObject();
5072 }
5073 
5074 #endif // #if VMA_STATS_STRING_ENABLED
5075 
5076 /*
5077 How many suitable free suballocations to analyze before choosing best one.
5078 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
5079  be chosen.
5080 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
5081  suballocations will be analized and best one will be chosen.
5082 - Any other value is also acceptable.
5083 */
5084 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
5085 
5086 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
5087 {
5088  VMA_ASSERT(IsEmpty());
5089  pAllocationRequest->offset = 0;
5090  pAllocationRequest->sumFreeSize = m_SumFreeSize;
5091  pAllocationRequest->sumItemSize = 0;
5092  pAllocationRequest->item = m_Suballocations.begin();
5093  pAllocationRequest->itemsToMakeLostCount = 0;
5094 }
5095 
5096 bool VmaBlockMetadata::CreateAllocationRequest(
5097  uint32_t currentFrameIndex,
5098  uint32_t frameInUseCount,
5099  VkDeviceSize bufferImageGranularity,
5100  VkDeviceSize allocSize,
5101  VkDeviceSize allocAlignment,
5102  VmaSuballocationType allocType,
5103  bool canMakeOtherLost,
5104  VmaAllocationRequest* pAllocationRequest)
5105 {
5106  VMA_ASSERT(allocSize > 0);
5107  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5108  VMA_ASSERT(pAllocationRequest != VMA_NULL);
5109  VMA_HEAVY_ASSERT(Validate());
5110 
5111  // There is not enough total free space in this block to fullfill the request: Early return.
5112  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
5113  {
5114  return false;
5115  }
5116 
5117  // New algorithm, efficiently searching freeSuballocationsBySize.
5118  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
5119  if(freeSuballocCount > 0)
5120  {
5121  if(VMA_BEST_FIT)
5122  {
5123  // Find first free suballocation with size not less than allocSize.
5124  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5125  m_FreeSuballocationsBySize.data(),
5126  m_FreeSuballocationsBySize.data() + freeSuballocCount,
5127  allocSize,
5128  VmaSuballocationItemSizeLess());
5129  size_t index = it - m_FreeSuballocationsBySize.data();
5130  for(; index < freeSuballocCount; ++index)
5131  {
5132  if(CheckAllocation(
5133  currentFrameIndex,
5134  frameInUseCount,
5135  bufferImageGranularity,
5136  allocSize,
5137  allocAlignment,
5138  allocType,
5139  m_FreeSuballocationsBySize[index],
5140  false, // canMakeOtherLost
5141  &pAllocationRequest->offset,
5142  &pAllocationRequest->itemsToMakeLostCount,
5143  &pAllocationRequest->sumFreeSize,
5144  &pAllocationRequest->sumItemSize))
5145  {
5146  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5147  return true;
5148  }
5149  }
5150  }
5151  else
5152  {
5153  // Search staring from biggest suballocations.
5154  for(size_t index = freeSuballocCount; index--; )
5155  {
5156  if(CheckAllocation(
5157  currentFrameIndex,
5158  frameInUseCount,
5159  bufferImageGranularity,
5160  allocSize,
5161  allocAlignment,
5162  allocType,
5163  m_FreeSuballocationsBySize[index],
5164  false, // canMakeOtherLost
5165  &pAllocationRequest->offset,
5166  &pAllocationRequest->itemsToMakeLostCount,
5167  &pAllocationRequest->sumFreeSize,
5168  &pAllocationRequest->sumItemSize))
5169  {
5170  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5171  return true;
5172  }
5173  }
5174  }
5175  }
5176 
5177  if(canMakeOtherLost)
5178  {
5179  // Brute-force algorithm. TODO: Come up with something better.
5180 
5181  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
5182  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
5183 
5184  VmaAllocationRequest tmpAllocRequest = {};
5185  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
5186  suballocIt != m_Suballocations.end();
5187  ++suballocIt)
5188  {
5189  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
5190  suballocIt->hAllocation->CanBecomeLost())
5191  {
5192  if(CheckAllocation(
5193  currentFrameIndex,
5194  frameInUseCount,
5195  bufferImageGranularity,
5196  allocSize,
5197  allocAlignment,
5198  allocType,
5199  suballocIt,
5200  canMakeOtherLost,
5201  &tmpAllocRequest.offset,
5202  &tmpAllocRequest.itemsToMakeLostCount,
5203  &tmpAllocRequest.sumFreeSize,
5204  &tmpAllocRequest.sumItemSize))
5205  {
5206  tmpAllocRequest.item = suballocIt;
5207 
5208  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
5209  {
5210  *pAllocationRequest = tmpAllocRequest;
5211  }
5212  }
5213  }
5214  }
5215 
5216  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
5217  {
5218  return true;
5219  }
5220  }
5221 
5222  return false;
5223 }
5224 
5225 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
5226  uint32_t currentFrameIndex,
5227  uint32_t frameInUseCount,
5228  VmaAllocationRequest* pAllocationRequest)
5229 {
5230  while(pAllocationRequest->itemsToMakeLostCount > 0)
5231  {
5232  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
5233  {
5234  ++pAllocationRequest->item;
5235  }
5236  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5237  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
5238  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
5239  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5240  {
5241  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
5242  --pAllocationRequest->itemsToMakeLostCount;
5243  }
5244  else
5245  {
5246  return false;
5247  }
5248  }
5249 
5250  VMA_HEAVY_ASSERT(Validate());
5251  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5252  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
5253 
5254  return true;
5255 }
5256 
5257 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5258 {
5259  uint32_t lostAllocationCount = 0;
5260  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
5261  it != m_Suballocations.end();
5262  ++it)
5263  {
5264  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
5265  it->hAllocation->CanBecomeLost() &&
5266  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5267  {
5268  it = FreeSuballocation(it);
5269  ++lostAllocationCount;
5270  }
5271  }
5272  return lostAllocationCount;
5273 }
5274 
5275 void VmaBlockMetadata::Alloc(
5276  const VmaAllocationRequest& request,
5277  VmaSuballocationType type,
5278  VkDeviceSize allocSize,
5279  VmaAllocation hAllocation)
5280 {
5281  VMA_ASSERT(request.item != m_Suballocations.end());
5282  VmaSuballocation& suballoc = *request.item;
5283  // Given suballocation is a free block.
5284  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5285  // Given offset is inside this suballocation.
5286  VMA_ASSERT(request.offset >= suballoc.offset);
5287  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
5288  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
5289  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
5290 
5291  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
5292  // it to become used.
5293  UnregisterFreeSuballocation(request.item);
5294 
5295  suballoc.offset = request.offset;
5296  suballoc.size = allocSize;
5297  suballoc.type = type;
5298  suballoc.hAllocation = hAllocation;
5299 
5300  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
5301  if(paddingEnd)
5302  {
5303  VmaSuballocation paddingSuballoc = {};
5304  paddingSuballoc.offset = request.offset + allocSize;
5305  paddingSuballoc.size = paddingEnd;
5306  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5307  VmaSuballocationList::iterator next = request.item;
5308  ++next;
5309  const VmaSuballocationList::iterator paddingEndItem =
5310  m_Suballocations.insert(next, paddingSuballoc);
5311  RegisterFreeSuballocation(paddingEndItem);
5312  }
5313 
5314  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
5315  if(paddingBegin)
5316  {
5317  VmaSuballocation paddingSuballoc = {};
5318  paddingSuballoc.offset = request.offset - paddingBegin;
5319  paddingSuballoc.size = paddingBegin;
5320  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5321  const VmaSuballocationList::iterator paddingBeginItem =
5322  m_Suballocations.insert(request.item, paddingSuballoc);
5323  RegisterFreeSuballocation(paddingBeginItem);
5324  }
5325 
5326  // Update totals.
5327  m_FreeCount = m_FreeCount - 1;
5328  if(paddingBegin > 0)
5329  {
5330  ++m_FreeCount;
5331  }
5332  if(paddingEnd > 0)
5333  {
5334  ++m_FreeCount;
5335  }
5336  m_SumFreeSize -= allocSize;
5337 }
5338 
5339 void VmaBlockMetadata::Free(const VmaAllocation allocation)
5340 {
5341  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5342  suballocItem != m_Suballocations.end();
5343  ++suballocItem)
5344  {
5345  VmaSuballocation& suballoc = *suballocItem;
5346  if(suballoc.hAllocation == allocation)
5347  {
5348  FreeSuballocation(suballocItem);
5349  VMA_HEAVY_ASSERT(Validate());
5350  return;
5351  }
5352  }
5353  VMA_ASSERT(0 && "Not found!");
5354 }
5355 
5356 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
5357 {
5358  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5359  suballocItem != m_Suballocations.end();
5360  ++suballocItem)
5361  {
5362  VmaSuballocation& suballoc = *suballocItem;
5363  if(suballoc.offset == offset)
5364  {
5365  FreeSuballocation(suballocItem);
5366  return;
5367  }
5368  }
5369  VMA_ASSERT(0 && "Not found!");
5370 }
5371 
5372 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
5373 {
5374  VkDeviceSize lastSize = 0;
5375  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
5376  {
5377  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
5378 
5379  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
5380  {
5381  VMA_ASSERT(0);
5382  return false;
5383  }
5384  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5385  {
5386  VMA_ASSERT(0);
5387  return false;
5388  }
5389  if(it->size < lastSize)
5390  {
5391  VMA_ASSERT(0);
5392  return false;
5393  }
5394 
5395  lastSize = it->size;
5396  }
5397  return true;
5398 }
5399 
5400 bool VmaBlockMetadata::CheckAllocation(
5401  uint32_t currentFrameIndex,
5402  uint32_t frameInUseCount,
5403  VkDeviceSize bufferImageGranularity,
5404  VkDeviceSize allocSize,
5405  VkDeviceSize allocAlignment,
5406  VmaSuballocationType allocType,
5407  VmaSuballocationList::const_iterator suballocItem,
5408  bool canMakeOtherLost,
5409  VkDeviceSize* pOffset,
5410  size_t* itemsToMakeLostCount,
5411  VkDeviceSize* pSumFreeSize,
5412  VkDeviceSize* pSumItemSize) const
5413 {
5414  VMA_ASSERT(allocSize > 0);
5415  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5416  VMA_ASSERT(suballocItem != m_Suballocations.cend());
5417  VMA_ASSERT(pOffset != VMA_NULL);
5418 
5419  *itemsToMakeLostCount = 0;
5420  *pSumFreeSize = 0;
5421  *pSumItemSize = 0;
5422 
5423  if(canMakeOtherLost)
5424  {
5425  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5426  {
5427  *pSumFreeSize = suballocItem->size;
5428  }
5429  else
5430  {
5431  if(suballocItem->hAllocation->CanBecomeLost() &&
5432  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5433  {
5434  ++*itemsToMakeLostCount;
5435  *pSumItemSize = suballocItem->size;
5436  }
5437  else
5438  {
5439  return false;
5440  }
5441  }
5442 
5443  // Remaining size is too small for this request: Early return.
5444  if(m_Size - suballocItem->offset < allocSize)
5445  {
5446  return false;
5447  }
5448 
5449  // Start from offset equal to beginning of this suballocation.
5450  *pOffset = suballocItem->offset;
5451 
5452  // Apply VMA_DEBUG_MARGIN at the beginning.
5453  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5454  {
5455  *pOffset += VMA_DEBUG_MARGIN;
5456  }
5457 
5458  // Apply alignment.
5459  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5460  *pOffset = VmaAlignUp(*pOffset, alignment);
5461 
5462  // Check previous suballocations for BufferImageGranularity conflicts.
5463  // Make bigger alignment if necessary.
5464  if(bufferImageGranularity > 1)
5465  {
5466  bool bufferImageGranularityConflict = false;
5467  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5468  while(prevSuballocItem != m_Suballocations.cbegin())
5469  {
5470  --prevSuballocItem;
5471  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5472  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5473  {
5474  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5475  {
5476  bufferImageGranularityConflict = true;
5477  break;
5478  }
5479  }
5480  else
5481  // Already on previous page.
5482  break;
5483  }
5484  if(bufferImageGranularityConflict)
5485  {
5486  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5487  }
5488  }
5489 
5490  // Now that we have final *pOffset, check if we are past suballocItem.
5491  // If yes, return false - this function should be called for another suballocItem as starting point.
5492  if(*pOffset >= suballocItem->offset + suballocItem->size)
5493  {
5494  return false;
5495  }
5496 
5497  // Calculate padding at the beginning based on current offset.
5498  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
5499 
5500  // Calculate required margin at the end if this is not last suballocation.
5501  VmaSuballocationList::const_iterator next = suballocItem;
5502  ++next;
5503  const VkDeviceSize requiredEndMargin =
5504  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5505 
5506  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
5507  // Another early return check.
5508  if(suballocItem->offset + totalSize > m_Size)
5509  {
5510  return false;
5511  }
5512 
5513  // Advance lastSuballocItem until desired size is reached.
5514  // Update itemsToMakeLostCount.
5515  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
5516  if(totalSize > suballocItem->size)
5517  {
5518  VkDeviceSize remainingSize = totalSize - suballocItem->size;
5519  while(remainingSize > 0)
5520  {
5521  ++lastSuballocItem;
5522  if(lastSuballocItem == m_Suballocations.cend())
5523  {
5524  return false;
5525  }
5526  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5527  {
5528  *pSumFreeSize += lastSuballocItem->size;
5529  }
5530  else
5531  {
5532  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
5533  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
5534  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5535  {
5536  ++*itemsToMakeLostCount;
5537  *pSumItemSize += lastSuballocItem->size;
5538  }
5539  else
5540  {
5541  return false;
5542  }
5543  }
5544  remainingSize = (lastSuballocItem->size < remainingSize) ?
5545  remainingSize - lastSuballocItem->size : 0;
5546  }
5547  }
5548 
5549  // Check next suballocations for BufferImageGranularity conflicts.
5550  // If conflict exists, we must mark more allocations lost or fail.
5551  if(bufferImageGranularity > 1)
5552  {
5553  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
5554  ++nextSuballocItem;
5555  while(nextSuballocItem != m_Suballocations.cend())
5556  {
5557  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5558  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5559  {
5560  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5561  {
5562  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
5563  if(nextSuballoc.hAllocation->CanBecomeLost() &&
5564  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5565  {
5566  ++*itemsToMakeLostCount;
5567  }
5568  else
5569  {
5570  return false;
5571  }
5572  }
5573  }
5574  else
5575  {
5576  // Already on next page.
5577  break;
5578  }
5579  ++nextSuballocItem;
5580  }
5581  }
5582  }
5583  else
5584  {
5585  const VmaSuballocation& suballoc = *suballocItem;
5586  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5587 
5588  *pSumFreeSize = suballoc.size;
5589 
5590  // Size of this suballocation is too small for this request: Early return.
5591  if(suballoc.size < allocSize)
5592  {
5593  return false;
5594  }
5595 
5596  // Start from offset equal to beginning of this suballocation.
5597  *pOffset = suballoc.offset;
5598 
5599  // Apply VMA_DEBUG_MARGIN at the beginning.
5600  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5601  {
5602  *pOffset += VMA_DEBUG_MARGIN;
5603  }
5604 
5605  // Apply alignment.
5606  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5607  *pOffset = VmaAlignUp(*pOffset, alignment);
5608 
5609  // Check previous suballocations for BufferImageGranularity conflicts.
5610  // Make bigger alignment if necessary.
5611  if(bufferImageGranularity > 1)
5612  {
5613  bool bufferImageGranularityConflict = false;
5614  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5615  while(prevSuballocItem != m_Suballocations.cbegin())
5616  {
5617  --prevSuballocItem;
5618  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5619  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5620  {
5621  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5622  {
5623  bufferImageGranularityConflict = true;
5624  break;
5625  }
5626  }
5627  else
5628  // Already on previous page.
5629  break;
5630  }
5631  if(bufferImageGranularityConflict)
5632  {
5633  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5634  }
5635  }
5636 
5637  // Calculate padding at the beginning based on current offset.
5638  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
5639 
5640  // Calculate required margin at the end if this is not last suballocation.
5641  VmaSuballocationList::const_iterator next = suballocItem;
5642  ++next;
5643  const VkDeviceSize requiredEndMargin =
5644  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5645 
5646  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
5647  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
5648  {
5649  return false;
5650  }
5651 
5652  // Check next suballocations for BufferImageGranularity conflicts.
5653  // If conflict exists, allocation cannot be made here.
5654  if(bufferImageGranularity > 1)
5655  {
5656  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
5657  ++nextSuballocItem;
5658  while(nextSuballocItem != m_Suballocations.cend())
5659  {
5660  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5661  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5662  {
5663  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5664  {
5665  return false;
5666  }
5667  }
5668  else
5669  {
5670  // Already on next page.
5671  break;
5672  }
5673  ++nextSuballocItem;
5674  }
5675  }
5676  }
5677 
5678  // All tests passed: Success. pOffset is already filled.
5679  return true;
5680 }
5681 
5682 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
5683 {
5684  VMA_ASSERT(item != m_Suballocations.end());
5685  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5686 
5687  VmaSuballocationList::iterator nextItem = item;
5688  ++nextItem;
5689  VMA_ASSERT(nextItem != m_Suballocations.end());
5690  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
5691 
5692  item->size += nextItem->size;
5693  --m_FreeCount;
5694  m_Suballocations.erase(nextItem);
5695 }
5696 
5697 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
5698 {
5699  // Change this suballocation to be marked as free.
5700  VmaSuballocation& suballoc = *suballocItem;
5701  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5702  suballoc.hAllocation = VK_NULL_HANDLE;
5703 
5704  // Update totals.
5705  ++m_FreeCount;
5706  m_SumFreeSize += suballoc.size;
5707 
5708  // Merge with previous and/or next suballocation if it's also free.
5709  bool mergeWithNext = false;
5710  bool mergeWithPrev = false;
5711 
5712  VmaSuballocationList::iterator nextItem = suballocItem;
5713  ++nextItem;
5714  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
5715  {
5716  mergeWithNext = true;
5717  }
5718 
5719  VmaSuballocationList::iterator prevItem = suballocItem;
5720  if(suballocItem != m_Suballocations.begin())
5721  {
5722  --prevItem;
5723  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5724  {
5725  mergeWithPrev = true;
5726  }
5727  }
5728 
5729  if(mergeWithNext)
5730  {
5731  UnregisterFreeSuballocation(nextItem);
5732  MergeFreeWithNext(suballocItem);
5733  }
5734 
5735  if(mergeWithPrev)
5736  {
5737  UnregisterFreeSuballocation(prevItem);
5738  MergeFreeWithNext(prevItem);
5739  RegisterFreeSuballocation(prevItem);
5740  return prevItem;
5741  }
5742  else
5743  {
5744  RegisterFreeSuballocation(suballocItem);
5745  return suballocItem;
5746  }
5747 }
5748 
5749 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
5750 {
5751  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5752  VMA_ASSERT(item->size > 0);
5753 
5754  // You may want to enable this validation at the beginning or at the end of
5755  // this function, depending on what do you want to check.
5756  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5757 
5758  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5759  {
5760  if(m_FreeSuballocationsBySize.empty())
5761  {
5762  m_FreeSuballocationsBySize.push_back(item);
5763  }
5764  else
5765  {
5766  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
5767  }
5768  }
5769 
5770  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5771 }
5772 
5773 
5774 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
5775 {
5776  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5777  VMA_ASSERT(item->size > 0);
5778 
5779  // You may want to enable this validation at the beginning or at the end of
5780  // this function, depending on what do you want to check.
5781  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5782 
5783  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5784  {
5785  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5786  m_FreeSuballocationsBySize.data(),
5787  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
5788  item,
5789  VmaSuballocationItemSizeLess());
5790  for(size_t index = it - m_FreeSuballocationsBySize.data();
5791  index < m_FreeSuballocationsBySize.size();
5792  ++index)
5793  {
5794  if(m_FreeSuballocationsBySize[index] == item)
5795  {
5796  VmaVectorRemove(m_FreeSuballocationsBySize, index);
5797  return;
5798  }
5799  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
5800  }
5801  VMA_ASSERT(0 && "Not found.");
5802  }
5803 
5804  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5805 }
5806 
5808 // class VmaDeviceMemoryMapping
5809 
5810 VmaDeviceMemoryMapping::VmaDeviceMemoryMapping() :
5811  m_MapCount(0),
5812  m_pMappedData(VMA_NULL)
5813 {
5814 }
5815 
5816 VmaDeviceMemoryMapping::~VmaDeviceMemoryMapping()
5817 {
5818  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
5819 }
5820 
5821 VkResult VmaDeviceMemoryMapping::Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData)
5822 {
5823  if(count == 0)
5824  {
5825  return VK_SUCCESS;
5826  }
5827 
5828  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
5829  if(m_MapCount != 0)
5830  {
5831  m_MapCount += count;
5832  VMA_ASSERT(m_pMappedData != VMA_NULL);
5833  if(ppData != VMA_NULL)
5834  {
5835  *ppData = m_pMappedData;
5836  }
5837  return VK_SUCCESS;
5838  }
5839  else
5840  {
5841  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5842  hAllocator->m_hDevice,
5843  hMemory,
5844  0, // offset
5845  VK_WHOLE_SIZE,
5846  0, // flags
5847  &m_pMappedData);
5848  if(result == VK_SUCCESS)
5849  {
5850  if(ppData != VMA_NULL)
5851  {
5852  *ppData = m_pMappedData;
5853  }
5854  m_MapCount = count;
5855  }
5856  return result;
5857  }
5858 }
5859 
5860 void VmaDeviceMemoryMapping::Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count)
5861 {
5862  if(count == 0)
5863  {
5864  return;
5865  }
5866 
5867  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
5868  if(m_MapCount >= count)
5869  {
5870  m_MapCount -= count;
5871  if(m_MapCount == 0)
5872  {
5873  m_pMappedData = VMA_NULL;
5874  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, hMemory);
5875  }
5876  }
5877  else
5878  {
5879  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
5880  }
5881 }
5882 
5884 // class VmaDeviceMemoryBlock
5885 
5886 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
5887  m_MemoryTypeIndex(UINT32_MAX),
5888  m_hMemory(VK_NULL_HANDLE),
5889  m_Metadata(hAllocator)
5890 {
5891 }
5892 
5893 void VmaDeviceMemoryBlock::Init(
5894  uint32_t newMemoryTypeIndex,
5895  VkDeviceMemory newMemory,
5896  VkDeviceSize newSize)
5897 {
5898  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
5899 
5900  m_MemoryTypeIndex = newMemoryTypeIndex;
5901  m_hMemory = newMemory;
5902 
5903  m_Metadata.Init(newSize);
5904 }
5905 
5906 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
5907 {
5908  // This is the most important assert in the entire library.
5909  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
5910  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
5911 
5912  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
5913  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
5914  m_hMemory = VK_NULL_HANDLE;
5915 }
5916 
5917 bool VmaDeviceMemoryBlock::Validate() const
5918 {
5919  if((m_hMemory == VK_NULL_HANDLE) ||
5920  (m_Metadata.GetSize() == 0))
5921  {
5922  return false;
5923  }
5924 
5925  return m_Metadata.Validate();
5926 }
5927 
5928 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
5929 {
5930  return m_Mapping.Map(hAllocator, m_hMemory, count, ppData);
5931 }
5932 
5933 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
5934 {
5935  m_Mapping.Unmap(hAllocator, m_hMemory, count);
5936 }
5937 
5938 static void InitStatInfo(VmaStatInfo& outInfo)
5939 {
5940  memset(&outInfo, 0, sizeof(outInfo));
5941  outInfo.allocationSizeMin = UINT64_MAX;
5942  outInfo.unusedRangeSizeMin = UINT64_MAX;
5943 }
5944 
5945 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
5946 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
5947 {
5948  inoutInfo.blockCount += srcInfo.blockCount;
5949  inoutInfo.allocationCount += srcInfo.allocationCount;
5950  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
5951  inoutInfo.usedBytes += srcInfo.usedBytes;
5952  inoutInfo.unusedBytes += srcInfo.unusedBytes;
5953  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
5954  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
5955  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
5956  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
5957 }
5958 
5959 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
5960 {
5961  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
5962  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
5963  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
5964  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
5965 }
5966 
5967 VmaPool_T::VmaPool_T(
5968  VmaAllocator hAllocator,
5969  const VmaPoolCreateInfo& createInfo) :
5970  m_BlockVector(
5971  hAllocator,
5972  createInfo.memoryTypeIndex,
5973  createInfo.blockSize,
5974  createInfo.minBlockCount,
5975  createInfo.maxBlockCount,
5976  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
5977  createInfo.frameInUseCount,
5978  true) // isCustomPool
5979 {
5980 }
5981 
5982 VmaPool_T::~VmaPool_T()
5983 {
5984 }
5985 
5986 #if VMA_STATS_STRING_ENABLED
5987 
5988 #endif // #if VMA_STATS_STRING_ENABLED
5989 
5990 VmaBlockVector::VmaBlockVector(
5991  VmaAllocator hAllocator,
5992  uint32_t memoryTypeIndex,
5993  VkDeviceSize preferredBlockSize,
5994  size_t minBlockCount,
5995  size_t maxBlockCount,
5996  VkDeviceSize bufferImageGranularity,
5997  uint32_t frameInUseCount,
5998  bool isCustomPool) :
5999  m_hAllocator(hAllocator),
6000  m_MemoryTypeIndex(memoryTypeIndex),
6001  m_PreferredBlockSize(preferredBlockSize),
6002  m_MinBlockCount(minBlockCount),
6003  m_MaxBlockCount(maxBlockCount),
6004  m_BufferImageGranularity(bufferImageGranularity),
6005  m_FrameInUseCount(frameInUseCount),
6006  m_IsCustomPool(isCustomPool),
6007  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
6008  m_HasEmptyBlock(false),
6009  m_pDefragmentator(VMA_NULL)
6010 {
6011 }
6012 
6013 VmaBlockVector::~VmaBlockVector()
6014 {
6015  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
6016 
6017  for(size_t i = m_Blocks.size(); i--; )
6018  {
6019  m_Blocks[i]->Destroy(m_hAllocator);
6020  vma_delete(m_hAllocator, m_Blocks[i]);
6021  }
6022 }
6023 
6024 VkResult VmaBlockVector::CreateMinBlocks()
6025 {
6026  for(size_t i = 0; i < m_MinBlockCount; ++i)
6027  {
6028  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
6029  if(res != VK_SUCCESS)
6030  {
6031  return res;
6032  }
6033  }
6034  return VK_SUCCESS;
6035 }
6036 
6037 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
6038 {
6039  pStats->size = 0;
6040  pStats->unusedSize = 0;
6041  pStats->allocationCount = 0;
6042  pStats->unusedRangeCount = 0;
6043  pStats->unusedRangeSizeMax = 0;
6044 
6045  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6046 
6047  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6048  {
6049  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6050  VMA_ASSERT(pBlock);
6051  VMA_HEAVY_ASSERT(pBlock->Validate());
6052  pBlock->m_Metadata.AddPoolStats(*pStats);
6053  }
6054 }
6055 
6056 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
6057 
6058 VkResult VmaBlockVector::Allocate(
6059  VmaPool hCurrentPool,
6060  uint32_t currentFrameIndex,
6061  const VkMemoryRequirements& vkMemReq,
6062  const VmaAllocationCreateInfo& createInfo,
6063  VmaSuballocationType suballocType,
6064  VmaAllocation* pAllocation)
6065 {
6066  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
6067  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
6068 
6069  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6070 
6071  // 1. Search existing allocations. Try to allocate without making other allocations lost.
6072  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6073  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6074  {
6075  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6076  VMA_ASSERT(pCurrBlock);
6077  VmaAllocationRequest currRequest = {};
6078  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6079  currentFrameIndex,
6080  m_FrameInUseCount,
6081  m_BufferImageGranularity,
6082  vkMemReq.size,
6083  vkMemReq.alignment,
6084  suballocType,
6085  false, // canMakeOtherLost
6086  &currRequest))
6087  {
6088  // Allocate from pCurrBlock.
6089  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
6090 
6091  if(mapped)
6092  {
6093  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
6094  if(res != VK_SUCCESS)
6095  {
6096  return res;
6097  }
6098  }
6099 
6100  // We no longer have an empty Allocation.
6101  if(pCurrBlock->m_Metadata.IsEmpty())
6102  {
6103  m_HasEmptyBlock = false;
6104  }
6105 
6106  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6107  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
6108  (*pAllocation)->InitBlockAllocation(
6109  hCurrentPool,
6110  pCurrBlock,
6111  currRequest.offset,
6112  vkMemReq.alignment,
6113  vkMemReq.size,
6114  suballocType,
6115  mapped,
6116  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6117  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
6118  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6119  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6120  return VK_SUCCESS;
6121  }
6122  }
6123 
6124  const bool canCreateNewBlock =
6125  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
6126  (m_Blocks.size() < m_MaxBlockCount);
6127 
6128  // 2. Try to create new block.
6129  if(canCreateNewBlock)
6130  {
6131  // Calculate optimal size for new block.
6132  VkDeviceSize newBlockSize = m_PreferredBlockSize;
6133  uint32_t newBlockSizeShift = 0;
6134  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
6135 
6136  // Allocating blocks of other sizes is allowed only in default pools.
6137  // In custom pools block size is fixed.
6138  if(m_IsCustomPool == false)
6139  {
6140  // Allocate 1/8, 1/4, 1/2 as first blocks.
6141  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
6142  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
6143  {
6144  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6145  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= vkMemReq.size * 2)
6146  {
6147  newBlockSize = smallerNewBlockSize;
6148  ++newBlockSizeShift;
6149  }
6150  else
6151  {
6152  break;
6153  }
6154  }
6155  }
6156 
6157  size_t newBlockIndex = 0;
6158  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
6159  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
6160  if(m_IsCustomPool == false)
6161  {
6162  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
6163  {
6164  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6165  if(smallerNewBlockSize >= vkMemReq.size)
6166  {
6167  newBlockSize = smallerNewBlockSize;
6168  ++newBlockSizeShift;
6169  res = CreateBlock(newBlockSize, &newBlockIndex);
6170  }
6171  else
6172  {
6173  break;
6174  }
6175  }
6176  }
6177 
6178  if(res == VK_SUCCESS)
6179  {
6180  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
6181  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= vkMemReq.size);
6182 
6183  if(mapped)
6184  {
6185  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
6186  if(res != VK_SUCCESS)
6187  {
6188  return res;
6189  }
6190  }
6191 
6192  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
6193  VmaAllocationRequest allocRequest;
6194  pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
6195  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6196  pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
6197  (*pAllocation)->InitBlockAllocation(
6198  hCurrentPool,
6199  pBlock,
6200  allocRequest.offset,
6201  vkMemReq.alignment,
6202  vkMemReq.size,
6203  suballocType,
6204  mapped,
6205  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6206  VMA_HEAVY_ASSERT(pBlock->Validate());
6207  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
6208  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6209  return VK_SUCCESS;
6210  }
6211  }
6212 
6213  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
6214 
6215  // 3. Try to allocate from existing blocks with making other allocations lost.
6216  if(canMakeOtherLost)
6217  {
6218  uint32_t tryIndex = 0;
6219  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
6220  {
6221  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
6222  VmaAllocationRequest bestRequest = {};
6223  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
6224 
6225  // 1. Search existing allocations.
6226  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6227  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6228  {
6229  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6230  VMA_ASSERT(pCurrBlock);
6231  VmaAllocationRequest currRequest = {};
6232  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6233  currentFrameIndex,
6234  m_FrameInUseCount,
6235  m_BufferImageGranularity,
6236  vkMemReq.size,
6237  vkMemReq.alignment,
6238  suballocType,
6239  canMakeOtherLost,
6240  &currRequest))
6241  {
6242  const VkDeviceSize currRequestCost = currRequest.CalcCost();
6243  if(pBestRequestBlock == VMA_NULL ||
6244  currRequestCost < bestRequestCost)
6245  {
6246  pBestRequestBlock = pCurrBlock;
6247  bestRequest = currRequest;
6248  bestRequestCost = currRequestCost;
6249 
6250  if(bestRequestCost == 0)
6251  {
6252  break;
6253  }
6254  }
6255  }
6256  }
6257 
6258  if(pBestRequestBlock != VMA_NULL)
6259  {
6260  if(mapped)
6261  {
6262  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
6263  if(res != VK_SUCCESS)
6264  {
6265  return res;
6266  }
6267  }
6268 
6269  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
6270  currentFrameIndex,
6271  m_FrameInUseCount,
6272  &bestRequest))
6273  {
6274  // We no longer have an empty Allocation.
6275  if(pBestRequestBlock->m_Metadata.IsEmpty())
6276  {
6277  m_HasEmptyBlock = false;
6278  }
6279  // Allocate from this pBlock.
6280  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6281  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
6282  (*pAllocation)->InitBlockAllocation(
6283  hCurrentPool,
6284  pBestRequestBlock,
6285  bestRequest.offset,
6286  vkMemReq.alignment,
6287  vkMemReq.size,
6288  suballocType,
6289  mapped,
6290  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6291  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
6292  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6293  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6294  return VK_SUCCESS;
6295  }
6296  // else: Some allocations must have been touched while we are here. Next try.
6297  }
6298  else
6299  {
6300  // Could not find place in any of the blocks - break outer loop.
6301  break;
6302  }
6303  }
6304  /* Maximum number of tries exceeded - a very unlike event when many other
6305  threads are simultaneously touching allocations making it impossible to make
6306  lost at the same time as we try to allocate. */
6307  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
6308  {
6309  return VK_ERROR_TOO_MANY_OBJECTS;
6310  }
6311  }
6312 
6313  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6314 }
6315 
6316 void VmaBlockVector::Free(
6317  VmaAllocation hAllocation)
6318 {
6319  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
6320 
6321  // Scope for lock.
6322  {
6323  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6324 
6325  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
6326 
6327  if(hAllocation->IsPersistentMap())
6328  {
6329  pBlock->m_Mapping.Unmap(m_hAllocator, pBlock->m_hMemory, 1);
6330  }
6331 
6332  pBlock->m_Metadata.Free(hAllocation);
6333  VMA_HEAVY_ASSERT(pBlock->Validate());
6334 
6335  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
6336 
6337  // pBlock became empty after this deallocation.
6338  if(pBlock->m_Metadata.IsEmpty())
6339  {
6340  // Already has empty Allocation. We don't want to have two, so delete this one.
6341  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
6342  {
6343  pBlockToDelete = pBlock;
6344  Remove(pBlock);
6345  }
6346  // We now have first empty Allocation.
6347  else
6348  {
6349  m_HasEmptyBlock = true;
6350  }
6351  }
6352  // pBlock didn't become empty, but we have another empty block - find and free that one.
6353  // (This is optional, heuristics.)
6354  else if(m_HasEmptyBlock)
6355  {
6356  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
6357  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
6358  {
6359  pBlockToDelete = pLastBlock;
6360  m_Blocks.pop_back();
6361  m_HasEmptyBlock = false;
6362  }
6363  }
6364 
6365  IncrementallySortBlocks();
6366  }
6367 
6368  // Destruction of a free Allocation. Deferred until this point, outside of mutex
6369  // lock, for performance reason.
6370  if(pBlockToDelete != VMA_NULL)
6371  {
6372  VMA_DEBUG_LOG(" Deleted empty allocation");
6373  pBlockToDelete->Destroy(m_hAllocator);
6374  vma_delete(m_hAllocator, pBlockToDelete);
6375  }
6376 }
6377 
6378 size_t VmaBlockVector::CalcMaxBlockSize() const
6379 {
6380  size_t result = 0;
6381  for(size_t i = m_Blocks.size(); i--; )
6382  {
6383  result = VMA_MAX((uint64_t)result, (uint64_t)m_Blocks[i]->m_Metadata.GetSize());
6384  if(result >= m_PreferredBlockSize)
6385  {
6386  break;
6387  }
6388  }
6389  return result;
6390 }
6391 
6392 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
6393 {
6394  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6395  {
6396  if(m_Blocks[blockIndex] == pBlock)
6397  {
6398  VmaVectorRemove(m_Blocks, blockIndex);
6399  return;
6400  }
6401  }
6402  VMA_ASSERT(0);
6403 }
6404 
6405 void VmaBlockVector::IncrementallySortBlocks()
6406 {
6407  // Bubble sort only until first swap.
6408  for(size_t i = 1; i < m_Blocks.size(); ++i)
6409  {
6410  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
6411  {
6412  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
6413  return;
6414  }
6415  }
6416 }
6417 
6418 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
6419 {
6420  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6421  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
6422  allocInfo.allocationSize = blockSize;
6423  VkDeviceMemory mem = VK_NULL_HANDLE;
6424  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
6425  if(res < 0)
6426  {
6427  return res;
6428  }
6429 
6430  // New VkDeviceMemory successfully created.
6431 
6432  // Create new Allocation for it.
6433  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
6434  pBlock->Init(
6435  m_MemoryTypeIndex,
6436  mem,
6437  allocInfo.allocationSize);
6438 
6439  m_Blocks.push_back(pBlock);
6440  if(pNewBlockIndex != VMA_NULL)
6441  {
6442  *pNewBlockIndex = m_Blocks.size() - 1;
6443  }
6444 
6445  return VK_SUCCESS;
6446 }
6447 
6448 #if VMA_STATS_STRING_ENABLED
6449 
6450 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
6451 {
6452  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6453 
6454  json.BeginObject();
6455 
6456  if(m_IsCustomPool)
6457  {
6458  json.WriteString("MemoryTypeIndex");
6459  json.WriteNumber(m_MemoryTypeIndex);
6460 
6461  json.WriteString("BlockSize");
6462  json.WriteNumber(m_PreferredBlockSize);
6463 
6464  json.WriteString("BlockCount");
6465  json.BeginObject(true);
6466  if(m_MinBlockCount > 0)
6467  {
6468  json.WriteString("Min");
6469  json.WriteNumber((uint64_t)m_MinBlockCount);
6470  }
6471  if(m_MaxBlockCount < SIZE_MAX)
6472  {
6473  json.WriteString("Max");
6474  json.WriteNumber((uint64_t)m_MaxBlockCount);
6475  }
6476  json.WriteString("Cur");
6477  json.WriteNumber((uint64_t)m_Blocks.size());
6478  json.EndObject();
6479 
6480  if(m_FrameInUseCount > 0)
6481  {
6482  json.WriteString("FrameInUseCount");
6483  json.WriteNumber(m_FrameInUseCount);
6484  }
6485  }
6486  else
6487  {
6488  json.WriteString("PreferredBlockSize");
6489  json.WriteNumber(m_PreferredBlockSize);
6490  }
6491 
6492  json.WriteString("Blocks");
6493  json.BeginArray();
6494  for(size_t i = 0; i < m_Blocks.size(); ++i)
6495  {
6496  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
6497  }
6498  json.EndArray();
6499 
6500  json.EndObject();
6501 }
6502 
6503 #endif // #if VMA_STATS_STRING_ENABLED
6504 
6505 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
6506  VmaAllocator hAllocator,
6507  uint32_t currentFrameIndex)
6508 {
6509  if(m_pDefragmentator == VMA_NULL)
6510  {
6511  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
6512  hAllocator,
6513  this,
6514  currentFrameIndex);
6515  }
6516 
6517  return m_pDefragmentator;
6518 }
6519 
6520 VkResult VmaBlockVector::Defragment(
6521  VmaDefragmentationStats* pDefragmentationStats,
6522  VkDeviceSize& maxBytesToMove,
6523  uint32_t& maxAllocationsToMove)
6524 {
6525  if(m_pDefragmentator == VMA_NULL)
6526  {
6527  return VK_SUCCESS;
6528  }
6529 
6530  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6531 
6532  // Defragment.
6533  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
6534 
6535  // Accumulate statistics.
6536  if(pDefragmentationStats != VMA_NULL)
6537  {
6538  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
6539  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
6540  pDefragmentationStats->bytesMoved += bytesMoved;
6541  pDefragmentationStats->allocationsMoved += allocationsMoved;
6542  VMA_ASSERT(bytesMoved <= maxBytesToMove);
6543  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
6544  maxBytesToMove -= bytesMoved;
6545  maxAllocationsToMove -= allocationsMoved;
6546  }
6547 
6548  // Free empty blocks.
6549  m_HasEmptyBlock = false;
6550  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
6551  {
6552  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
6553  if(pBlock->m_Metadata.IsEmpty())
6554  {
6555  if(m_Blocks.size() > m_MinBlockCount)
6556  {
6557  if(pDefragmentationStats != VMA_NULL)
6558  {
6559  ++pDefragmentationStats->deviceMemoryBlocksFreed;
6560  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
6561  }
6562 
6563  VmaVectorRemove(m_Blocks, blockIndex);
6564  pBlock->Destroy(m_hAllocator);
6565  vma_delete(m_hAllocator, pBlock);
6566  }
6567  else
6568  {
6569  m_HasEmptyBlock = true;
6570  }
6571  }
6572  }
6573 
6574  return result;
6575 }
6576 
6577 void VmaBlockVector::DestroyDefragmentator()
6578 {
6579  if(m_pDefragmentator != VMA_NULL)
6580  {
6581  vma_delete(m_hAllocator, m_pDefragmentator);
6582  m_pDefragmentator = VMA_NULL;
6583  }
6584 }
6585 
6586 void VmaBlockVector::MakePoolAllocationsLost(
6587  uint32_t currentFrameIndex,
6588  size_t* pLostAllocationCount)
6589 {
6590  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6591  size_t lostAllocationCount = 0;
6592  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6593  {
6594  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6595  VMA_ASSERT(pBlock);
6596  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
6597  }
6598  if(pLostAllocationCount != VMA_NULL)
6599  {
6600  *pLostAllocationCount = lostAllocationCount;
6601  }
6602 }
6603 
6604 void VmaBlockVector::AddStats(VmaStats* pStats)
6605 {
6606  const uint32_t memTypeIndex = m_MemoryTypeIndex;
6607  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
6608 
6609  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6610 
6611  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6612  {
6613  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6614  VMA_ASSERT(pBlock);
6615  VMA_HEAVY_ASSERT(pBlock->Validate());
6616  VmaStatInfo allocationStatInfo;
6617  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
6618  VmaAddStatInfo(pStats->total, allocationStatInfo);
6619  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
6620  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
6621  }
6622 }
6623 
6625 // VmaDefragmentator members definition
6626 
6627 VmaDefragmentator::VmaDefragmentator(
6628  VmaAllocator hAllocator,
6629  VmaBlockVector* pBlockVector,
6630  uint32_t currentFrameIndex) :
6631  m_hAllocator(hAllocator),
6632  m_pBlockVector(pBlockVector),
6633  m_CurrentFrameIndex(currentFrameIndex),
6634  m_BytesMoved(0),
6635  m_AllocationsMoved(0),
6636  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
6637  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
6638 {
6639 }
6640 
6641 VmaDefragmentator::~VmaDefragmentator()
6642 {
6643  for(size_t i = m_Blocks.size(); i--; )
6644  {
6645  vma_delete(m_hAllocator, m_Blocks[i]);
6646  }
6647 }
6648 
6649 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
6650 {
6651  AllocationInfo allocInfo;
6652  allocInfo.m_hAllocation = hAlloc;
6653  allocInfo.m_pChanged = pChanged;
6654  m_Allocations.push_back(allocInfo);
6655 }
6656 
6657 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
6658 {
6659  // It has already been mapped for defragmentation.
6660  if(m_pMappedDataForDefragmentation)
6661  {
6662  *ppMappedData = m_pMappedDataForDefragmentation;
6663  return VK_SUCCESS;
6664  }
6665 
6666  // It is originally mapped.
6667  if(m_pBlock->m_Mapping.GetMappedData())
6668  {
6669  *ppMappedData = m_pBlock->m_Mapping.GetMappedData();
6670  return VK_SUCCESS;
6671  }
6672 
6673  // Map on first usage.
6674  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
6675  *ppMappedData = m_pMappedDataForDefragmentation;
6676  return res;
6677 }
6678 
6679 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
6680 {
6681  if(m_pMappedDataForDefragmentation != VMA_NULL)
6682  {
6683  m_pBlock->Unmap(hAllocator, 1);
6684  }
6685 }
6686 
6687 VkResult VmaDefragmentator::DefragmentRound(
6688  VkDeviceSize maxBytesToMove,
6689  uint32_t maxAllocationsToMove)
6690 {
6691  if(m_Blocks.empty())
6692  {
6693  return VK_SUCCESS;
6694  }
6695 
6696  size_t srcBlockIndex = m_Blocks.size() - 1;
6697  size_t srcAllocIndex = SIZE_MAX;
6698  for(;;)
6699  {
6700  // 1. Find next allocation to move.
6701  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
6702  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
6703  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
6704  {
6705  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
6706  {
6707  // Finished: no more allocations to process.
6708  if(srcBlockIndex == 0)
6709  {
6710  return VK_SUCCESS;
6711  }
6712  else
6713  {
6714  --srcBlockIndex;
6715  srcAllocIndex = SIZE_MAX;
6716  }
6717  }
6718  else
6719  {
6720  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
6721  }
6722  }
6723 
6724  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
6725  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
6726 
6727  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
6728  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
6729  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
6730  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
6731 
6732  // 2. Try to find new place for this allocation in preceding or current block.
6733  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
6734  {
6735  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
6736  VmaAllocationRequest dstAllocRequest;
6737  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
6738  m_CurrentFrameIndex,
6739  m_pBlockVector->GetFrameInUseCount(),
6740  m_pBlockVector->GetBufferImageGranularity(),
6741  size,
6742  alignment,
6743  suballocType,
6744  false, // canMakeOtherLost
6745  &dstAllocRequest) &&
6746  MoveMakesSense(
6747  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
6748  {
6749  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
6750 
6751  // Reached limit on number of allocations or bytes to move.
6752  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
6753  (m_BytesMoved + size > maxBytesToMove))
6754  {
6755  return VK_INCOMPLETE;
6756  }
6757 
6758  void* pDstMappedData = VMA_NULL;
6759  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
6760  if(res != VK_SUCCESS)
6761  {
6762  return res;
6763  }
6764 
6765  void* pSrcMappedData = VMA_NULL;
6766  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
6767  if(res != VK_SUCCESS)
6768  {
6769  return res;
6770  }
6771 
6772  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
6773  memcpy(
6774  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
6775  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
6776  static_cast<size_t>(size));
6777 
6778  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
6779  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
6780 
6781  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
6782 
6783  if(allocInfo.m_pChanged != VMA_NULL)
6784  {
6785  *allocInfo.m_pChanged = VK_TRUE;
6786  }
6787 
6788  ++m_AllocationsMoved;
6789  m_BytesMoved += size;
6790 
6791  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
6792 
6793  break;
6794  }
6795  }
6796 
6797  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
6798 
6799  if(srcAllocIndex > 0)
6800  {
6801  --srcAllocIndex;
6802  }
6803  else
6804  {
6805  if(srcBlockIndex > 0)
6806  {
6807  --srcBlockIndex;
6808  srcAllocIndex = SIZE_MAX;
6809  }
6810  else
6811  {
6812  return VK_SUCCESS;
6813  }
6814  }
6815  }
6816 }
6817 
6818 VkResult VmaDefragmentator::Defragment(
6819  VkDeviceSize maxBytesToMove,
6820  uint32_t maxAllocationsToMove)
6821 {
6822  if(m_Allocations.empty())
6823  {
6824  return VK_SUCCESS;
6825  }
6826 
6827  // Create block info for each block.
6828  const size_t blockCount = m_pBlockVector->m_Blocks.size();
6829  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6830  {
6831  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
6832  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
6833  m_Blocks.push_back(pBlockInfo);
6834  }
6835 
6836  // Sort them by m_pBlock pointer value.
6837  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
6838 
6839  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
6840  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
6841  {
6842  AllocationInfo& allocInfo = m_Allocations[blockIndex];
6843  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
6844  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6845  {
6846  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
6847  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
6848  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
6849  {
6850  (*it)->m_Allocations.push_back(allocInfo);
6851  }
6852  else
6853  {
6854  VMA_ASSERT(0);
6855  }
6856  }
6857  }
6858  m_Allocations.clear();
6859 
6860  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6861  {
6862  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
6863  pBlockInfo->CalcHasNonMovableAllocations();
6864  pBlockInfo->SortAllocationsBySizeDescecnding();
6865  }
6866 
6867  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
6868  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
6869 
6870  // Execute defragmentation rounds (the main part).
6871  VkResult result = VK_SUCCESS;
6872  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
6873  {
6874  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
6875  }
6876 
6877  // Unmap blocks that were mapped for defragmentation.
6878  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6879  {
6880  m_Blocks[blockIndex]->Unmap(m_hAllocator);
6881  }
6882 
6883  return result;
6884 }
6885 
6886 bool VmaDefragmentator::MoveMakesSense(
6887  size_t dstBlockIndex, VkDeviceSize dstOffset,
6888  size_t srcBlockIndex, VkDeviceSize srcOffset)
6889 {
6890  if(dstBlockIndex < srcBlockIndex)
6891  {
6892  return true;
6893  }
6894  if(dstBlockIndex > srcBlockIndex)
6895  {
6896  return false;
6897  }
6898  if(dstOffset < srcOffset)
6899  {
6900  return true;
6901  }
6902  return false;
6903 }
6904 
6906 // VmaAllocator_T
6907 
6908 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
6909  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
6910  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
6911  m_hDevice(pCreateInfo->device),
6912  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
6913  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
6914  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
6915  m_PreferredLargeHeapBlockSize(0),
6916  m_PhysicalDevice(pCreateInfo->physicalDevice),
6917  m_CurrentFrameIndex(0),
6918  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
6919 {
6920  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
6921 
6922  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
6923  memset(&m_MemProps, 0, sizeof(m_MemProps));
6924  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
6925 
6926  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
6927  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
6928 
6929  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6930  {
6931  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
6932  }
6933 
6934  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
6935  {
6936  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
6937  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
6938  }
6939 
6940  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
6941 
6942  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
6943  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
6944 
6945  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
6946  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
6947 
6948  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
6949  {
6950  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
6951  {
6952  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
6953  if(limit != VK_WHOLE_SIZE)
6954  {
6955  m_HeapSizeLimit[heapIndex] = limit;
6956  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
6957  {
6958  m_MemProps.memoryHeaps[heapIndex].size = limit;
6959  }
6960  }
6961  }
6962  }
6963 
6964  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6965  {
6966  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
6967 
6968  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
6969  this,
6970  memTypeIndex,
6971  preferredBlockSize,
6972  0,
6973  SIZE_MAX,
6974  GetBufferImageGranularity(),
6975  pCreateInfo->frameInUseCount,
6976  false); // isCustomPool
6977  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
6978  // becase minBlockCount is 0.
6979  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
6980  }
6981 }
6982 
6983 VmaAllocator_T::~VmaAllocator_T()
6984 {
6985  VMA_ASSERT(m_Pools.empty());
6986 
6987  for(size_t i = GetMemoryTypeCount(); i--; )
6988  {
6989  vma_delete(this, m_pDedicatedAllocations[i]);
6990  vma_delete(this, m_pBlockVectors[i]);
6991  }
6992 }
6993 
6994 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
6995 {
6996 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
6997  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
6998  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
6999  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
7000  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
7001  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
7002  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
7003  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
7004  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
7005  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
7006  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
7007  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
7008  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
7009  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
7010  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
7011  if(m_UseKhrDedicatedAllocation)
7012  {
7013  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
7014  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
7015  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
7016  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
7017  }
7018 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7019 
7020 #define VMA_COPY_IF_NOT_NULL(funcName) \
7021  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
7022 
7023  if(pVulkanFunctions != VMA_NULL)
7024  {
7025  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
7026  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
7027  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
7028  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
7029  VMA_COPY_IF_NOT_NULL(vkMapMemory);
7030  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
7031  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
7032  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
7033  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
7034  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
7035  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
7036  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
7037  VMA_COPY_IF_NOT_NULL(vkCreateImage);
7038  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
7039  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
7040  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
7041  }
7042 
7043 #undef VMA_COPY_IF_NOT_NULL
7044 
7045  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
7046  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
7047  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
7048  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
7049  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
7050  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
7051  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
7052  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
7053  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
7054  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
7055  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
7056  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
7057  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
7058  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
7059  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
7060  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
7061  if(m_UseKhrDedicatedAllocation)
7062  {
7063  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
7064  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
7065  }
7066 }
7067 
7068 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
7069 {
7070  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7071  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
7072  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
7073  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
7074 }
7075 
7076 VkResult VmaAllocator_T::AllocateMemoryOfType(
7077  const VkMemoryRequirements& vkMemReq,
7078  bool dedicatedAllocation,
7079  VkBuffer dedicatedBuffer,
7080  VkImage dedicatedImage,
7081  const VmaAllocationCreateInfo& createInfo,
7082  uint32_t memTypeIndex,
7083  VmaSuballocationType suballocType,
7084  VmaAllocation* pAllocation)
7085 {
7086  VMA_ASSERT(pAllocation != VMA_NULL);
7087  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
7088 
7089  VmaAllocationCreateInfo finalCreateInfo = createInfo;
7090 
7091  // If memory type is not HOST_VISIBLE, disable MAPPED.
7092  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7093  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
7094  {
7095  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
7096  }
7097 
7098  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
7099  VMA_ASSERT(blockVector);
7100 
7101  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
7102  bool preferDedicatedMemory =
7103  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
7104  dedicatedAllocation ||
7105  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
7106  vkMemReq.size > preferredBlockSize / 2;
7107 
7108  if(preferDedicatedMemory &&
7109  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
7110  finalCreateInfo.pool == VK_NULL_HANDLE)
7111  {
7112  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
7113  }
7114 
7115  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
7116  {
7117  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7118  {
7119  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7120  }
7121  else
7122  {
7123  return AllocateDedicatedMemory(
7124  vkMemReq.size,
7125  suballocType,
7126  memTypeIndex,
7127  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7128  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7129  finalCreateInfo.pUserData,
7130  dedicatedBuffer,
7131  dedicatedImage,
7132  pAllocation);
7133  }
7134  }
7135  else
7136  {
7137  VkResult res = blockVector->Allocate(
7138  VK_NULL_HANDLE, // hCurrentPool
7139  m_CurrentFrameIndex.load(),
7140  vkMemReq,
7141  finalCreateInfo,
7142  suballocType,
7143  pAllocation);
7144  if(res == VK_SUCCESS)
7145  {
7146  return res;
7147  }
7148 
7149  // 5. Try dedicated memory.
7150  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7151  {
7152  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7153  }
7154  else
7155  {
7156  res = AllocateDedicatedMemory(
7157  vkMemReq.size,
7158  suballocType,
7159  memTypeIndex,
7160  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7161  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7162  finalCreateInfo.pUserData,
7163  dedicatedBuffer,
7164  dedicatedImage,
7165  pAllocation);
7166  if(res == VK_SUCCESS)
7167  {
7168  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
7169  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
7170  return VK_SUCCESS;
7171  }
7172  else
7173  {
7174  // Everything failed: Return error code.
7175  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7176  return res;
7177  }
7178  }
7179  }
7180 }
7181 
7182 VkResult VmaAllocator_T::AllocateDedicatedMemory(
7183  VkDeviceSize size,
7184  VmaSuballocationType suballocType,
7185  uint32_t memTypeIndex,
7186  bool map,
7187  bool isUserDataString,
7188  void* pUserData,
7189  VkBuffer dedicatedBuffer,
7190  VkImage dedicatedImage,
7191  VmaAllocation* pAllocation)
7192 {
7193  VMA_ASSERT(pAllocation);
7194 
7195  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
7196  allocInfo.memoryTypeIndex = memTypeIndex;
7197  allocInfo.allocationSize = size;
7198 
7199  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
7200  if(m_UseKhrDedicatedAllocation)
7201  {
7202  if(dedicatedBuffer != VK_NULL_HANDLE)
7203  {
7204  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
7205  dedicatedAllocInfo.buffer = dedicatedBuffer;
7206  allocInfo.pNext = &dedicatedAllocInfo;
7207  }
7208  else if(dedicatedImage != VK_NULL_HANDLE)
7209  {
7210  dedicatedAllocInfo.image = dedicatedImage;
7211  allocInfo.pNext = &dedicatedAllocInfo;
7212  }
7213  }
7214 
7215  // Allocate VkDeviceMemory.
7216  VkDeviceMemory hMemory = VK_NULL_HANDLE;
7217  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
7218  if(res < 0)
7219  {
7220  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7221  return res;
7222  }
7223 
7224  void* pMappedData = VMA_NULL;
7225  if(map)
7226  {
7227  res = (*m_VulkanFunctions.vkMapMemory)(
7228  m_hDevice,
7229  hMemory,
7230  0,
7231  VK_WHOLE_SIZE,
7232  0,
7233  &pMappedData);
7234  if(res < 0)
7235  {
7236  VMA_DEBUG_LOG(" vkMapMemory FAILED");
7237  FreeVulkanMemory(memTypeIndex, size, hMemory);
7238  return res;
7239  }
7240  }
7241 
7242  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
7243  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
7244  (*pAllocation)->SetUserData(this, pUserData);
7245 
7246  // Register it in m_pDedicatedAllocations.
7247  {
7248  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7249  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7250  VMA_ASSERT(pDedicatedAllocations);
7251  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
7252  }
7253 
7254  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
7255 
7256  return VK_SUCCESS;
7257 }
7258 
7259 void VmaAllocator_T::GetBufferMemoryRequirements(
7260  VkBuffer hBuffer,
7261  VkMemoryRequirements& memReq,
7262  bool& requiresDedicatedAllocation,
7263  bool& prefersDedicatedAllocation) const
7264 {
7265  if(m_UseKhrDedicatedAllocation)
7266  {
7267  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
7268  memReqInfo.buffer = hBuffer;
7269 
7270  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7271 
7272  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7273  memReq2.pNext = &memDedicatedReq;
7274 
7275  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7276 
7277  memReq = memReq2.memoryRequirements;
7278  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7279  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7280  }
7281  else
7282  {
7283  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
7284  requiresDedicatedAllocation = false;
7285  prefersDedicatedAllocation = false;
7286  }
7287 }
7288 
7289 void VmaAllocator_T::GetImageMemoryRequirements(
7290  VkImage hImage,
7291  VkMemoryRequirements& memReq,
7292  bool& requiresDedicatedAllocation,
7293  bool& prefersDedicatedAllocation) const
7294 {
7295  if(m_UseKhrDedicatedAllocation)
7296  {
7297  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
7298  memReqInfo.image = hImage;
7299 
7300  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7301 
7302  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7303  memReq2.pNext = &memDedicatedReq;
7304 
7305  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7306 
7307  memReq = memReq2.memoryRequirements;
7308  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7309  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7310  }
7311  else
7312  {
7313  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
7314  requiresDedicatedAllocation = false;
7315  prefersDedicatedAllocation = false;
7316  }
7317 }
7318 
7319 VkResult VmaAllocator_T::AllocateMemory(
7320  const VkMemoryRequirements& vkMemReq,
7321  bool requiresDedicatedAllocation,
7322  bool prefersDedicatedAllocation,
7323  VkBuffer dedicatedBuffer,
7324  VkImage dedicatedImage,
7325  const VmaAllocationCreateInfo& createInfo,
7326  VmaSuballocationType suballocType,
7327  VmaAllocation* pAllocation)
7328 {
7329  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
7330  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7331  {
7332  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
7333  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7334  }
7335  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7336  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
7337  {
7338  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
7339  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7340  }
7341  if(requiresDedicatedAllocation)
7342  {
7343  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7344  {
7345  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
7346  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7347  }
7348  if(createInfo.pool != VK_NULL_HANDLE)
7349  {
7350  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
7351  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7352  }
7353  }
7354  if((createInfo.pool != VK_NULL_HANDLE) &&
7355  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
7356  {
7357  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
7358  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7359  }
7360 
7361  if(createInfo.pool != VK_NULL_HANDLE)
7362  {
7363  return createInfo.pool->m_BlockVector.Allocate(
7364  createInfo.pool,
7365  m_CurrentFrameIndex.load(),
7366  vkMemReq,
7367  createInfo,
7368  suballocType,
7369  pAllocation);
7370  }
7371  else
7372  {
7373  // Bit mask of memory Vulkan types acceptable for this allocation.
7374  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
7375  uint32_t memTypeIndex = UINT32_MAX;
7376  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7377  if(res == VK_SUCCESS)
7378  {
7379  res = AllocateMemoryOfType(
7380  vkMemReq,
7381  requiresDedicatedAllocation || prefersDedicatedAllocation,
7382  dedicatedBuffer,
7383  dedicatedImage,
7384  createInfo,
7385  memTypeIndex,
7386  suballocType,
7387  pAllocation);
7388  // Succeeded on first try.
7389  if(res == VK_SUCCESS)
7390  {
7391  return res;
7392  }
7393  // Allocation from this memory type failed. Try other compatible memory types.
7394  else
7395  {
7396  for(;;)
7397  {
7398  // Remove old memTypeIndex from list of possibilities.
7399  memoryTypeBits &= ~(1u << memTypeIndex);
7400  // Find alternative memTypeIndex.
7401  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7402  if(res == VK_SUCCESS)
7403  {
7404  res = AllocateMemoryOfType(
7405  vkMemReq,
7406  requiresDedicatedAllocation || prefersDedicatedAllocation,
7407  dedicatedBuffer,
7408  dedicatedImage,
7409  createInfo,
7410  memTypeIndex,
7411  suballocType,
7412  pAllocation);
7413  // Allocation from this alternative memory type succeeded.
7414  if(res == VK_SUCCESS)
7415  {
7416  return res;
7417  }
7418  // else: Allocation from this memory type failed. Try next one - next loop iteration.
7419  }
7420  // No other matching memory type index could be found.
7421  else
7422  {
7423  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
7424  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7425  }
7426  }
7427  }
7428  }
7429  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
7430  else
7431  return res;
7432  }
7433 }
7434 
7435 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
7436 {
7437  VMA_ASSERT(allocation);
7438 
7439  if(allocation->CanBecomeLost() == false ||
7440  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
7441  {
7442  switch(allocation->GetType())
7443  {
7444  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7445  {
7446  VmaBlockVector* pBlockVector = VMA_NULL;
7447  VmaPool hPool = allocation->GetPool();
7448  if(hPool != VK_NULL_HANDLE)
7449  {
7450  pBlockVector = &hPool->m_BlockVector;
7451  }
7452  else
7453  {
7454  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7455  pBlockVector = m_pBlockVectors[memTypeIndex];
7456  }
7457  pBlockVector->Free(allocation);
7458  }
7459  break;
7460  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7461  FreeDedicatedMemory(allocation);
7462  break;
7463  default:
7464  VMA_ASSERT(0);
7465  }
7466  }
7467 
7468  allocation->SetUserData(this, VMA_NULL);
7469  vma_delete(this, allocation);
7470 }
7471 
7472 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
7473 {
7474  // Initialize.
7475  InitStatInfo(pStats->total);
7476  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
7477  InitStatInfo(pStats->memoryType[i]);
7478  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
7479  InitStatInfo(pStats->memoryHeap[i]);
7480 
7481  // Process default pools.
7482  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7483  {
7484  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
7485  VMA_ASSERT(pBlockVector);
7486  pBlockVector->AddStats(pStats);
7487  }
7488 
7489  // Process custom pools.
7490  {
7491  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7492  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
7493  {
7494  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
7495  }
7496  }
7497 
7498  // Process dedicated allocations.
7499  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7500  {
7501  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7502  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7503  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7504  VMA_ASSERT(pDedicatedAllocVector);
7505  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
7506  {
7507  VmaStatInfo allocationStatInfo;
7508  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
7509  VmaAddStatInfo(pStats->total, allocationStatInfo);
7510  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
7511  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
7512  }
7513  }
7514 
7515  // Postprocess.
7516  VmaPostprocessCalcStatInfo(pStats->total);
7517  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
7518  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
7519  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
7520  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
7521 }
7522 
7523 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
7524 
7525 VkResult VmaAllocator_T::Defragment(
7526  VmaAllocation* pAllocations,
7527  size_t allocationCount,
7528  VkBool32* pAllocationsChanged,
7529  const VmaDefragmentationInfo* pDefragmentationInfo,
7530  VmaDefragmentationStats* pDefragmentationStats)
7531 {
7532  if(pAllocationsChanged != VMA_NULL)
7533  {
7534  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
7535  }
7536  if(pDefragmentationStats != VMA_NULL)
7537  {
7538  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
7539  }
7540 
7541  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
7542 
7543  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
7544 
7545  const size_t poolCount = m_Pools.size();
7546 
7547  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
7548  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
7549  {
7550  VmaAllocation hAlloc = pAllocations[allocIndex];
7551  VMA_ASSERT(hAlloc);
7552  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
7553  // DedicatedAlloc cannot be defragmented.
7554  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
7555  // Only HOST_VISIBLE memory types can be defragmented.
7556  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
7557  // Lost allocation cannot be defragmented.
7558  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
7559  {
7560  VmaBlockVector* pAllocBlockVector = VMA_NULL;
7561 
7562  const VmaPool hAllocPool = hAlloc->GetPool();
7563  // This allocation belongs to custom pool.
7564  if(hAllocPool != VK_NULL_HANDLE)
7565  {
7566  pAllocBlockVector = &hAllocPool->GetBlockVector();
7567  }
7568  // This allocation belongs to general pool.
7569  else
7570  {
7571  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
7572  }
7573 
7574  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
7575 
7576  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
7577  &pAllocationsChanged[allocIndex] : VMA_NULL;
7578  pDefragmentator->AddAllocation(hAlloc, pChanged);
7579  }
7580  }
7581 
7582  VkResult result = VK_SUCCESS;
7583 
7584  // ======== Main processing.
7585 
7586  VkDeviceSize maxBytesToMove = SIZE_MAX;
7587  uint32_t maxAllocationsToMove = UINT32_MAX;
7588  if(pDefragmentationInfo != VMA_NULL)
7589  {
7590  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
7591  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
7592  }
7593 
7594  // Process standard memory.
7595  for(uint32_t memTypeIndex = 0;
7596  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
7597  ++memTypeIndex)
7598  {
7599  // Only HOST_VISIBLE memory types can be defragmented.
7600  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7601  {
7602  result = m_pBlockVectors[memTypeIndex]->Defragment(
7603  pDefragmentationStats,
7604  maxBytesToMove,
7605  maxAllocationsToMove);
7606  }
7607  }
7608 
7609  // Process custom pools.
7610  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
7611  {
7612  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
7613  pDefragmentationStats,
7614  maxBytesToMove,
7615  maxAllocationsToMove);
7616  }
7617 
7618  // ======== Destroy defragmentators.
7619 
7620  // Process custom pools.
7621  for(size_t poolIndex = poolCount; poolIndex--; )
7622  {
7623  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
7624  }
7625 
7626  // Process standard memory.
7627  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
7628  {
7629  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7630  {
7631  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
7632  }
7633  }
7634 
7635  return result;
7636 }
7637 
7638 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
7639 {
7640  if(hAllocation->CanBecomeLost())
7641  {
7642  /*
7643  Warning: This is a carefully designed algorithm.
7644  Do not modify unless you really know what you're doing :)
7645  */
7646  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
7647  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
7648  for(;;)
7649  {
7650  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
7651  {
7652  pAllocationInfo->memoryType = UINT32_MAX;
7653  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
7654  pAllocationInfo->offset = 0;
7655  pAllocationInfo->size = hAllocation->GetSize();
7656  pAllocationInfo->pMappedData = VMA_NULL;
7657  pAllocationInfo->pUserData = hAllocation->GetUserData();
7658  return;
7659  }
7660  else if(localLastUseFrameIndex == localCurrFrameIndex)
7661  {
7662  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7663  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7664  pAllocationInfo->offset = hAllocation->GetOffset();
7665  pAllocationInfo->size = hAllocation->GetSize();
7666  pAllocationInfo->pMappedData = VMA_NULL;
7667  pAllocationInfo->pUserData = hAllocation->GetUserData();
7668  return;
7669  }
7670  else // Last use time earlier than current time.
7671  {
7672  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
7673  {
7674  localLastUseFrameIndex = localCurrFrameIndex;
7675  }
7676  }
7677  }
7678  }
7679  else
7680  {
7681  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7682  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7683  pAllocationInfo->offset = hAllocation->GetOffset();
7684  pAllocationInfo->size = hAllocation->GetSize();
7685  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
7686  pAllocationInfo->pUserData = hAllocation->GetUserData();
7687  }
7688 }
7689 
7690 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
7691 {
7692  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
7693 
7694  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
7695 
7696  if(newCreateInfo.maxBlockCount == 0)
7697  {
7698  newCreateInfo.maxBlockCount = SIZE_MAX;
7699  }
7700  if(newCreateInfo.blockSize == 0)
7701  {
7702  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
7703  }
7704 
7705  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
7706 
7707  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
7708  if(res != VK_SUCCESS)
7709  {
7710  vma_delete(this, *pPool);
7711  *pPool = VMA_NULL;
7712  return res;
7713  }
7714 
7715  // Add to m_Pools.
7716  {
7717  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7718  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
7719  }
7720 
7721  return VK_SUCCESS;
7722 }
7723 
7724 void VmaAllocator_T::DestroyPool(VmaPool pool)
7725 {
7726  // Remove from m_Pools.
7727  {
7728  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7729  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
7730  VMA_ASSERT(success && "Pool not found in Allocator.");
7731  }
7732 
7733  vma_delete(this, pool);
7734 }
7735 
7736 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
7737 {
7738  pool->m_BlockVector.GetPoolStats(pPoolStats);
7739 }
7740 
7741 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
7742 {
7743  m_CurrentFrameIndex.store(frameIndex);
7744 }
7745 
7746 void VmaAllocator_T::MakePoolAllocationsLost(
7747  VmaPool hPool,
7748  size_t* pLostAllocationCount)
7749 {
7750  hPool->m_BlockVector.MakePoolAllocationsLost(
7751  m_CurrentFrameIndex.load(),
7752  pLostAllocationCount);
7753 }
7754 
7755 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
7756 {
7757  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
7758  (*pAllocation)->InitLost();
7759 }
7760 
7761 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
7762 {
7763  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
7764 
7765  VkResult res;
7766  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7767  {
7768  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7769  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
7770  {
7771  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
7772  if(res == VK_SUCCESS)
7773  {
7774  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
7775  }
7776  }
7777  else
7778  {
7779  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
7780  }
7781  }
7782  else
7783  {
7784  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
7785  }
7786 
7787  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
7788  {
7789  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
7790  }
7791 
7792  return res;
7793 }
7794 
7795 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
7796 {
7797  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
7798  {
7799  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
7800  }
7801 
7802  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
7803 
7804  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
7805  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7806  {
7807  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7808  m_HeapSizeLimit[heapIndex] += size;
7809  }
7810 }
7811 
7812 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
7813 {
7814  if(hAllocation->CanBecomeLost())
7815  {
7816  return VK_ERROR_MEMORY_MAP_FAILED;
7817  }
7818 
7819  switch(hAllocation->GetType())
7820  {
7821  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7822  {
7823  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
7824  char *pBytes = VMA_NULL;
7825  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
7826  if(res == VK_SUCCESS)
7827  {
7828  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
7829  hAllocation->BlockAllocMap();
7830  }
7831  return res;
7832  }
7833  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7834  return hAllocation->DedicatedAllocMap(this, ppData);
7835  default:
7836  VMA_ASSERT(0);
7837  return VK_ERROR_MEMORY_MAP_FAILED;
7838  }
7839 }
7840 
7841 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
7842 {
7843  switch(hAllocation->GetType())
7844  {
7845  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7846  {
7847  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
7848  hAllocation->BlockAllocUnmap();
7849  pBlock->Unmap(this, 1);
7850  }
7851  break;
7852  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7853  hAllocation->DedicatedAllocUnmap(this);
7854  break;
7855  default:
7856  VMA_ASSERT(0);
7857  }
7858 }
7859 
7860 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
7861 {
7862  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
7863 
7864  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7865  {
7866  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7867  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7868  VMA_ASSERT(pDedicatedAllocations);
7869  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
7870  VMA_ASSERT(success);
7871  }
7872 
7873  VkDeviceMemory hMemory = allocation->GetMemory();
7874 
7875  if(allocation->GetMappedData() != VMA_NULL)
7876  {
7877  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
7878  }
7879 
7880  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
7881 
7882  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
7883 }
7884 
7885 #if VMA_STATS_STRING_ENABLED
7886 
7887 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
7888 {
7889  bool dedicatedAllocationsStarted = false;
7890  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7891  {
7892  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7893  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7894  VMA_ASSERT(pDedicatedAllocVector);
7895  if(pDedicatedAllocVector->empty() == false)
7896  {
7897  if(dedicatedAllocationsStarted == false)
7898  {
7899  dedicatedAllocationsStarted = true;
7900  json.WriteString("DedicatedAllocations");
7901  json.BeginObject();
7902  }
7903 
7904  json.BeginString("Type ");
7905  json.ContinueString(memTypeIndex);
7906  json.EndString();
7907 
7908  json.BeginArray();
7909 
7910  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
7911  {
7912  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
7913  json.BeginObject(true);
7914 
7915  json.WriteString("Type");
7916  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
7917 
7918  json.WriteString("Size");
7919  json.WriteNumber(hAlloc->GetSize());
7920 
7921  const void* pUserData = hAlloc->GetUserData();
7922  if(pUserData != VMA_NULL)
7923  {
7924  json.WriteString("UserData");
7925  if(hAlloc->IsUserDataString())
7926  {
7927  json.WriteString((const char*)pUserData);
7928  }
7929  else
7930  {
7931  json.BeginString();
7932  json.ContinueString_Pointer(pUserData);
7933  json.EndString();
7934  }
7935  }
7936 
7937  json.EndObject();
7938  }
7939 
7940  json.EndArray();
7941  }
7942  }
7943  if(dedicatedAllocationsStarted)
7944  {
7945  json.EndObject();
7946  }
7947 
7948  {
7949  bool allocationsStarted = false;
7950  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7951  {
7952  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
7953  {
7954  if(allocationsStarted == false)
7955  {
7956  allocationsStarted = true;
7957  json.WriteString("DefaultPools");
7958  json.BeginObject();
7959  }
7960 
7961  json.BeginString("Type ");
7962  json.ContinueString(memTypeIndex);
7963  json.EndString();
7964 
7965  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
7966  }
7967  }
7968  if(allocationsStarted)
7969  {
7970  json.EndObject();
7971  }
7972  }
7973 
7974  {
7975  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7976  const size_t poolCount = m_Pools.size();
7977  if(poolCount > 0)
7978  {
7979  json.WriteString("Pools");
7980  json.BeginArray();
7981  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
7982  {
7983  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
7984  }
7985  json.EndArray();
7986  }
7987  }
7988 }
7989 
7990 #endif // #if VMA_STATS_STRING_ENABLED
7991 
7992 static VkResult AllocateMemoryForImage(
7993  VmaAllocator allocator,
7994  VkImage image,
7995  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7996  VmaSuballocationType suballocType,
7997  VmaAllocation* pAllocation)
7998 {
7999  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
8000 
8001  VkMemoryRequirements vkMemReq = {};
8002  bool requiresDedicatedAllocation = false;
8003  bool prefersDedicatedAllocation = false;
8004  allocator->GetImageMemoryRequirements(image, vkMemReq,
8005  requiresDedicatedAllocation, prefersDedicatedAllocation);
8006 
8007  return allocator->AllocateMemory(
8008  vkMemReq,
8009  requiresDedicatedAllocation,
8010  prefersDedicatedAllocation,
8011  VK_NULL_HANDLE, // dedicatedBuffer
8012  image, // dedicatedImage
8013  *pAllocationCreateInfo,
8014  suballocType,
8015  pAllocation);
8016 }
8017 
8019 // Public interface
8020 
8021 VkResult vmaCreateAllocator(
8022  const VmaAllocatorCreateInfo* pCreateInfo,
8023  VmaAllocator* pAllocator)
8024 {
8025  VMA_ASSERT(pCreateInfo && pAllocator);
8026  VMA_DEBUG_LOG("vmaCreateAllocator");
8027  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
8028  return VK_SUCCESS;
8029 }
8030 
8031 void vmaDestroyAllocator(
8032  VmaAllocator allocator)
8033 {
8034  if(allocator != VK_NULL_HANDLE)
8035  {
8036  VMA_DEBUG_LOG("vmaDestroyAllocator");
8037  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
8038  vma_delete(&allocationCallbacks, allocator);
8039  }
8040 }
8041 
8042 void vmaGetPhysicalDeviceProperties(
8043  VmaAllocator allocator,
8044  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
8045 {
8046  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
8047  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
8048 }
8049 
8050 void vmaGetMemoryProperties(
8051  VmaAllocator allocator,
8052  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
8053 {
8054  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
8055  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
8056 }
8057 
8058 void vmaGetMemoryTypeProperties(
8059  VmaAllocator allocator,
8060  uint32_t memoryTypeIndex,
8061  VkMemoryPropertyFlags* pFlags)
8062 {
8063  VMA_ASSERT(allocator && pFlags);
8064  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
8065  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
8066 }
8067 
8068 void vmaSetCurrentFrameIndex(
8069  VmaAllocator allocator,
8070  uint32_t frameIndex)
8071 {
8072  VMA_ASSERT(allocator);
8073  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
8074 
8075  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8076 
8077  allocator->SetCurrentFrameIndex(frameIndex);
8078 }
8079 
8080 void vmaCalculateStats(
8081  VmaAllocator allocator,
8082  VmaStats* pStats)
8083 {
8084  VMA_ASSERT(allocator && pStats);
8085  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8086  allocator->CalculateStats(pStats);
8087 }
8088 
8089 #if VMA_STATS_STRING_ENABLED
8090 
8091 void vmaBuildStatsString(
8092  VmaAllocator allocator,
8093  char** ppStatsString,
8094  VkBool32 detailedMap)
8095 {
8096  VMA_ASSERT(allocator && ppStatsString);
8097  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8098 
8099  VmaStringBuilder sb(allocator);
8100  {
8101  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
8102  json.BeginObject();
8103 
8104  VmaStats stats;
8105  allocator->CalculateStats(&stats);
8106 
8107  json.WriteString("Total");
8108  VmaPrintStatInfo(json, stats.total);
8109 
8110  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
8111  {
8112  json.BeginString("Heap ");
8113  json.ContinueString(heapIndex);
8114  json.EndString();
8115  json.BeginObject();
8116 
8117  json.WriteString("Size");
8118  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
8119 
8120  json.WriteString("Flags");
8121  json.BeginArray(true);
8122  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
8123  {
8124  json.WriteString("DEVICE_LOCAL");
8125  }
8126  json.EndArray();
8127 
8128  if(stats.memoryHeap[heapIndex].blockCount > 0)
8129  {
8130  json.WriteString("Stats");
8131  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
8132  }
8133 
8134  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
8135  {
8136  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
8137  {
8138  json.BeginString("Type ");
8139  json.ContinueString(typeIndex);
8140  json.EndString();
8141 
8142  json.BeginObject();
8143 
8144  json.WriteString("Flags");
8145  json.BeginArray(true);
8146  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
8147  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
8148  {
8149  json.WriteString("DEVICE_LOCAL");
8150  }
8151  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
8152  {
8153  json.WriteString("HOST_VISIBLE");
8154  }
8155  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
8156  {
8157  json.WriteString("HOST_COHERENT");
8158  }
8159  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
8160  {
8161  json.WriteString("HOST_CACHED");
8162  }
8163  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
8164  {
8165  json.WriteString("LAZILY_ALLOCATED");
8166  }
8167  json.EndArray();
8168 
8169  if(stats.memoryType[typeIndex].blockCount > 0)
8170  {
8171  json.WriteString("Stats");
8172  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
8173  }
8174 
8175  json.EndObject();
8176  }
8177  }
8178 
8179  json.EndObject();
8180  }
8181  if(detailedMap == VK_TRUE)
8182  {
8183  allocator->PrintDetailedMap(json);
8184  }
8185 
8186  json.EndObject();
8187  }
8188 
8189  const size_t len = sb.GetLength();
8190  char* const pChars = vma_new_array(allocator, char, len + 1);
8191  if(len > 0)
8192  {
8193  memcpy(pChars, sb.GetData(), len);
8194  }
8195  pChars[len] = '\0';
8196  *ppStatsString = pChars;
8197 }
8198 
8199 void vmaFreeStatsString(
8200  VmaAllocator allocator,
8201  char* pStatsString)
8202 {
8203  if(pStatsString != VMA_NULL)
8204  {
8205  VMA_ASSERT(allocator);
8206  size_t len = strlen(pStatsString);
8207  vma_delete_array(allocator, pStatsString, len + 1);
8208  }
8209 }
8210 
8211 #endif // #if VMA_STATS_STRING_ENABLED
8212 
8213 /*
8214 This function is not protected by any mutex because it just reads immutable data.
8215 */
8216 VkResult vmaFindMemoryTypeIndex(
8217  VmaAllocator allocator,
8218  uint32_t memoryTypeBits,
8219  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8220  uint32_t* pMemoryTypeIndex)
8221 {
8222  VMA_ASSERT(allocator != VK_NULL_HANDLE);
8223  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8224  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8225 
8226  if(pAllocationCreateInfo->memoryTypeBits != 0)
8227  {
8228  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
8229  }
8230 
8231  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
8232  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
8233 
8234  // Convert usage to requiredFlags and preferredFlags.
8235  switch(pAllocationCreateInfo->usage)
8236  {
8237  case VMA_MEMORY_USAGE_UNKNOWN:
8238  break;
8239  case VMA_MEMORY_USAGE_GPU_ONLY:
8240  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8241  break;
8242  case VMA_MEMORY_USAGE_CPU_ONLY:
8243  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
8244  break;
8245  case VMA_MEMORY_USAGE_CPU_TO_GPU:
8246  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8247  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8248  break;
8249  case VMA_MEMORY_USAGE_GPU_TO_CPU:
8250  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8251  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
8252  break;
8253  default:
8254  break;
8255  }
8256 
8257  *pMemoryTypeIndex = UINT32_MAX;
8258  uint32_t minCost = UINT32_MAX;
8259  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
8260  memTypeIndex < allocator->GetMemoryTypeCount();
8261  ++memTypeIndex, memTypeBit <<= 1)
8262  {
8263  // This memory type is acceptable according to memoryTypeBits bitmask.
8264  if((memTypeBit & memoryTypeBits) != 0)
8265  {
8266  const VkMemoryPropertyFlags currFlags =
8267  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
8268  // This memory type contains requiredFlags.
8269  if((requiredFlags & ~currFlags) == 0)
8270  {
8271  // Calculate cost as number of bits from preferredFlags not present in this memory type.
8272  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
8273  // Remember memory type with lowest cost.
8274  if(currCost < minCost)
8275  {
8276  *pMemoryTypeIndex = memTypeIndex;
8277  if(currCost == 0)
8278  {
8279  return VK_SUCCESS;
8280  }
8281  minCost = currCost;
8282  }
8283  }
8284  }
8285  }
8286  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
8287 }
8288 
8289 VkResult vmaCreatePool(
8290  VmaAllocator allocator,
8291  const VmaPoolCreateInfo* pCreateInfo,
8292  VmaPool* pPool)
8293 {
8294  VMA_ASSERT(allocator && pCreateInfo && pPool);
8295 
8296  VMA_DEBUG_LOG("vmaCreatePool");
8297 
8298  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8299 
8300  return allocator->CreatePool(pCreateInfo, pPool);
8301 }
8302 
8303 void vmaDestroyPool(
8304  VmaAllocator allocator,
8305  VmaPool pool)
8306 {
8307  VMA_ASSERT(allocator);
8308 
8309  if(pool == VK_NULL_HANDLE)
8310  {
8311  return;
8312  }
8313 
8314  VMA_DEBUG_LOG("vmaDestroyPool");
8315 
8316  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8317 
8318  allocator->DestroyPool(pool);
8319 }
8320 
8321 void vmaGetPoolStats(
8322  VmaAllocator allocator,
8323  VmaPool pool,
8324  VmaPoolStats* pPoolStats)
8325 {
8326  VMA_ASSERT(allocator && pool && pPoolStats);
8327 
8328  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8329 
8330  allocator->GetPoolStats(pool, pPoolStats);
8331 }
8332 
8333 void vmaMakePoolAllocationsLost(
8334  VmaAllocator allocator,
8335  VmaPool pool,
8336  size_t* pLostAllocationCount)
8337 {
8338  VMA_ASSERT(allocator && pool);
8339 
8340  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8341 
8342  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
8343 }
8344 
8345 VkResult vmaAllocateMemory(
8346  VmaAllocator allocator,
8347  const VkMemoryRequirements* pVkMemoryRequirements,
8348  const VmaAllocationCreateInfo* pCreateInfo,
8349  VmaAllocation* pAllocation,
8350  VmaAllocationInfo* pAllocationInfo)
8351 {
8352  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
8353 
8354  VMA_DEBUG_LOG("vmaAllocateMemory");
8355 
8356  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8357 
8358  VkResult result = allocator->AllocateMemory(
8359  *pVkMemoryRequirements,
8360  false, // requiresDedicatedAllocation
8361  false, // prefersDedicatedAllocation
8362  VK_NULL_HANDLE, // dedicatedBuffer
8363  VK_NULL_HANDLE, // dedicatedImage
8364  *pCreateInfo,
8365  VMA_SUBALLOCATION_TYPE_UNKNOWN,
8366  pAllocation);
8367 
8368  if(pAllocationInfo && result == VK_SUCCESS)
8369  {
8370  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8371  }
8372 
8373  return result;
8374 }
8375 
8376 VkResult vmaAllocateMemoryForBuffer(
8377  VmaAllocator allocator,
8378  VkBuffer buffer,
8379  const VmaAllocationCreateInfo* pCreateInfo,
8380  VmaAllocation* pAllocation,
8381  VmaAllocationInfo* pAllocationInfo)
8382 {
8383  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8384 
8385  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
8386 
8387  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8388 
8389  VkMemoryRequirements vkMemReq = {};
8390  bool requiresDedicatedAllocation = false;
8391  bool prefersDedicatedAllocation = false;
8392  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
8393  requiresDedicatedAllocation,
8394  prefersDedicatedAllocation);
8395 
8396  VkResult result = allocator->AllocateMemory(
8397  vkMemReq,
8398  requiresDedicatedAllocation,
8399  prefersDedicatedAllocation,
8400  buffer, // dedicatedBuffer
8401  VK_NULL_HANDLE, // dedicatedImage
8402  *pCreateInfo,
8403  VMA_SUBALLOCATION_TYPE_BUFFER,
8404  pAllocation);
8405 
8406  if(pAllocationInfo && result == VK_SUCCESS)
8407  {
8408  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8409  }
8410 
8411  return result;
8412 }
8413 
8414 VkResult vmaAllocateMemoryForImage(
8415  VmaAllocator allocator,
8416  VkImage image,
8417  const VmaAllocationCreateInfo* pCreateInfo,
8418  VmaAllocation* pAllocation,
8419  VmaAllocationInfo* pAllocationInfo)
8420 {
8421  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8422 
8423  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
8424 
8425  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8426 
8427  VkResult result = AllocateMemoryForImage(
8428  allocator,
8429  image,
8430  pCreateInfo,
8431  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
8432  pAllocation);
8433 
8434  if(pAllocationInfo && result == VK_SUCCESS)
8435  {
8436  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8437  }
8438 
8439  return result;
8440 }
8441 
8442 void vmaFreeMemory(
8443  VmaAllocator allocator,
8444  VmaAllocation allocation)
8445 {
8446  VMA_ASSERT(allocator && allocation);
8447 
8448  VMA_DEBUG_LOG("vmaFreeMemory");
8449 
8450  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8451 
8452  allocator->FreeMemory(allocation);
8453 }
8454 
8455 void vmaGetAllocationInfo(
8456  VmaAllocator allocator,
8457  VmaAllocation allocation,
8458  VmaAllocationInfo* pAllocationInfo)
8459 {
8460  VMA_ASSERT(allocator && allocation && pAllocationInfo);
8461 
8462  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8463 
8464  allocator->GetAllocationInfo(allocation, pAllocationInfo);
8465 }
8466 
8467 void vmaSetAllocationUserData(
8468  VmaAllocator allocator,
8469  VmaAllocation allocation,
8470  void* pUserData)
8471 {
8472  VMA_ASSERT(allocator && allocation);
8473 
8474  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8475 
8476  allocation->SetUserData(allocator, pUserData);
8477 }
8478 
8479 void vmaCreateLostAllocation(
8480  VmaAllocator allocator,
8481  VmaAllocation* pAllocation)
8482 {
8483  VMA_ASSERT(allocator && pAllocation);
8484 
8485  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
8486 
8487  allocator->CreateLostAllocation(pAllocation);
8488 }
8489 
8490 VkResult vmaMapMemory(
8491  VmaAllocator allocator,
8492  VmaAllocation allocation,
8493  void** ppData)
8494 {
8495  VMA_ASSERT(allocator && allocation && ppData);
8496 
8497  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8498 
8499  return allocator->Map(allocation, ppData);
8500 }
8501 
8502 void vmaUnmapMemory(
8503  VmaAllocator allocator,
8504  VmaAllocation allocation)
8505 {
8506  VMA_ASSERT(allocator && allocation);
8507 
8508  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8509 
8510  allocator->Unmap(allocation);
8511 }
8512 
8513 VkResult vmaDefragment(
8514  VmaAllocator allocator,
8515  VmaAllocation* pAllocations,
8516  size_t allocationCount,
8517  VkBool32* pAllocationsChanged,
8518  const VmaDefragmentationInfo *pDefragmentationInfo,
8519  VmaDefragmentationStats* pDefragmentationStats)
8520 {
8521  VMA_ASSERT(allocator && pAllocations);
8522 
8523  VMA_DEBUG_LOG("vmaDefragment");
8524 
8525  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8526 
8527  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
8528 }
8529 
8530 VkResult vmaCreateBuffer(
8531  VmaAllocator allocator,
8532  const VkBufferCreateInfo* pBufferCreateInfo,
8533  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8534  VkBuffer* pBuffer,
8535  VmaAllocation* pAllocation,
8536  VmaAllocationInfo* pAllocationInfo)
8537 {
8538  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
8539 
8540  VMA_DEBUG_LOG("vmaCreateBuffer");
8541 
8542  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8543 
8544  *pBuffer = VK_NULL_HANDLE;
8545  *pAllocation = VK_NULL_HANDLE;
8546 
8547  // 1. Create VkBuffer.
8548  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
8549  allocator->m_hDevice,
8550  pBufferCreateInfo,
8551  allocator->GetAllocationCallbacks(),
8552  pBuffer);
8553  if(res >= 0)
8554  {
8555  // 2. vkGetBufferMemoryRequirements.
8556  VkMemoryRequirements vkMemReq = {};
8557  bool requiresDedicatedAllocation = false;
8558  bool prefersDedicatedAllocation = false;
8559  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
8560  requiresDedicatedAllocation, prefersDedicatedAllocation);
8561 
8562  // Make sure alignment requirements for specific buffer usages reported
8563  // in Physical Device Properties are included in alignment reported by memory requirements.
8564  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
8565  {
8566  VMA_ASSERT(vkMemReq.alignment %
8567  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
8568  }
8569  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
8570  {
8571  VMA_ASSERT(vkMemReq.alignment %
8572  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
8573  }
8574  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
8575  {
8576  VMA_ASSERT(vkMemReq.alignment %
8577  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
8578  }
8579 
8580  // 3. Allocate memory using allocator.
8581  res = allocator->AllocateMemory(
8582  vkMemReq,
8583  requiresDedicatedAllocation,
8584  prefersDedicatedAllocation,
8585  *pBuffer, // dedicatedBuffer
8586  VK_NULL_HANDLE, // dedicatedImage
8587  *pAllocationCreateInfo,
8588  VMA_SUBALLOCATION_TYPE_BUFFER,
8589  pAllocation);
8590  if(res >= 0)
8591  {
8592  // 3. Bind buffer with memory.
8593  res = (*allocator->GetVulkanFunctions().vkBindBufferMemory)(
8594  allocator->m_hDevice,
8595  *pBuffer,
8596  (*pAllocation)->GetMemory(),
8597  (*pAllocation)->GetOffset());
8598  if(res >= 0)
8599  {
8600  // All steps succeeded.
8601  if(pAllocationInfo != VMA_NULL)
8602  {
8603  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8604  }
8605  return VK_SUCCESS;
8606  }
8607  allocator->FreeMemory(*pAllocation);
8608  *pAllocation = VK_NULL_HANDLE;
8609  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8610  *pBuffer = VK_NULL_HANDLE;
8611  return res;
8612  }
8613  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8614  *pBuffer = VK_NULL_HANDLE;
8615  return res;
8616  }
8617  return res;
8618 }
8619 
8620 void vmaDestroyBuffer(
8621  VmaAllocator allocator,
8622  VkBuffer buffer,
8623  VmaAllocation allocation)
8624 {
8625  if(buffer != VK_NULL_HANDLE)
8626  {
8627  VMA_ASSERT(allocator);
8628 
8629  VMA_DEBUG_LOG("vmaDestroyBuffer");
8630 
8631  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8632 
8633  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
8634 
8635  allocator->FreeMemory(allocation);
8636  }
8637 }
8638 
8639 VkResult vmaCreateImage(
8640  VmaAllocator allocator,
8641  const VkImageCreateInfo* pImageCreateInfo,
8642  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8643  VkImage* pImage,
8644  VmaAllocation* pAllocation,
8645  VmaAllocationInfo* pAllocationInfo)
8646 {
8647  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
8648 
8649  VMA_DEBUG_LOG("vmaCreateImage");
8650 
8651  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8652 
8653  *pImage = VK_NULL_HANDLE;
8654  *pAllocation = VK_NULL_HANDLE;
8655 
8656  // 1. Create VkImage.
8657  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
8658  allocator->m_hDevice,
8659  pImageCreateInfo,
8660  allocator->GetAllocationCallbacks(),
8661  pImage);
8662  if(res >= 0)
8663  {
8664  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
8665  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
8666  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
8667 
8668  // 2. Allocate memory using allocator.
8669  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
8670  if(res >= 0)
8671  {
8672  // 3. Bind image with memory.
8673  res = (*allocator->GetVulkanFunctions().vkBindImageMemory)(
8674  allocator->m_hDevice,
8675  *pImage,
8676  (*pAllocation)->GetMemory(),
8677  (*pAllocation)->GetOffset());
8678  if(res >= 0)
8679  {
8680  // All steps succeeded.
8681  if(pAllocationInfo != VMA_NULL)
8682  {
8683  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8684  }
8685  return VK_SUCCESS;
8686  }
8687  allocator->FreeMemory(*pAllocation);
8688  *pAllocation = VK_NULL_HANDLE;
8689  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
8690  *pImage = VK_NULL_HANDLE;
8691  return res;
8692  }
8693  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
8694  *pImage = VK_NULL_HANDLE;
8695  return res;
8696  }
8697  return res;
8698 }
8699 
8700 void vmaDestroyImage(
8701  VmaAllocator allocator,
8702  VkImage image,
8703  VmaAllocation allocation)
8704 {
8705  if(image != VK_NULL_HANDLE)
8706  {
8707  VMA_ASSERT(allocator);
8708 
8709  VMA_DEBUG_LOG("vmaDestroyImage");
8710 
8711  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8712 
8713  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
8714 
8715  allocator->FreeMemory(allocation);
8716  }
8717 }
8718 
8719 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:847
-
VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:1101
+Go to the documentation of this file.
1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
823 #include <vulkan/vulkan.h>
824 
825 VK_DEFINE_HANDLE(VmaAllocator)
826 
827 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
829  VmaAllocator allocator,
830  uint32_t memoryType,
831  VkDeviceMemory memory,
832  VkDeviceSize size);
834 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
835  VmaAllocator allocator,
836  uint32_t memoryType,
837  VkDeviceMemory memory,
838  VkDeviceSize size);
839 
847 typedef struct VmaDeviceMemoryCallbacks {
849  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
851  PFN_vmaFreeDeviceMemoryFunction pfnFree;
852 } VmaDeviceMemoryCallbacks;
853 
855 typedef enum VmaAllocatorCreateFlagBits {
860  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
882  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
883 
884  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
885 } VmaAllocatorCreateFlagBits;
886 typedef VkFlags VmaAllocatorCreateFlags;
887 
892 typedef struct VmaVulkanFunctions {
893  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
894  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
895  PFN_vkAllocateMemory vkAllocateMemory;
896  PFN_vkFreeMemory vkFreeMemory;
897  PFN_vkMapMemory vkMapMemory;
898  PFN_vkUnmapMemory vkUnmapMemory;
899  PFN_vkBindBufferMemory vkBindBufferMemory;
900  PFN_vkBindImageMemory vkBindImageMemory;
901  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
902  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
903  PFN_vkCreateBuffer vkCreateBuffer;
904  PFN_vkDestroyBuffer vkDestroyBuffer;
905  PFN_vkCreateImage vkCreateImage;
906  PFN_vkDestroyImage vkDestroyImage;
907  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
908  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
909 } VmaVulkanFunctions;
910 
912 typedef struct VmaAllocatorCreateInfo
913 {
915  VmaAllocatorCreateFlags flags;
917 
918  VkPhysicalDevice physicalDevice;
920 
921  VkDevice device;
923 
924  VkDeviceSize preferredLargeHeapBlockSize;
926 
927  const VkAllocationCallbacks* pAllocationCallbacks;
929 
930  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
944  uint32_t frameInUseCount;
968  const VkDeviceSize* pHeapSizeLimit;
980  const VmaVulkanFunctions* pVulkanFunctions;
981 } VmaAllocatorCreateInfo;
982 
984 VkResult vmaCreateAllocator(
985  const VmaAllocatorCreateInfo* pCreateInfo,
986  VmaAllocator* pAllocator);
987 
989 void vmaDestroyAllocator(
990  VmaAllocator allocator);
991 
996 void vmaGetPhysicalDeviceProperties(
997  VmaAllocator allocator,
998  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
999 
1004 void vmaGetMemoryProperties(
1005  VmaAllocator allocator,
1006  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
1007 
1014 void vmaGetMemoryTypeProperties(
1015  VmaAllocator allocator,
1016  uint32_t memoryTypeIndex,
1017  VkMemoryPropertyFlags* pFlags);
1018 
1027 void vmaSetCurrentFrameIndex(
1028  VmaAllocator allocator,
1029  uint32_t frameIndex);
1030 
1033 typedef struct VmaStatInfo
1034 {
1036  uint32_t blockCount;
1038  uint32_t allocationCount;
1040  uint32_t unusedRangeCount;
1042  VkDeviceSize usedBytes;
1044  VkDeviceSize unusedBytes;
1045  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1046  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1047 } VmaStatInfo;
1048 
1050 typedef struct VmaStats
1051 {
1052  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1053  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1054  VmaStatInfo total;
1055 } VmaStats;
1056 
1058 void vmaCalculateStats(
1059  VmaAllocator allocator,
1060  VmaStats* pStats);
1061 
1062 #define VMA_STATS_STRING_ENABLED 1
1063 
1064 #if VMA_STATS_STRING_ENABLED
1065 
1067 
1069 void vmaBuildStatsString(
1070  VmaAllocator allocator,
1071  char** ppStatsString,
1072  VkBool32 detailedMap);
1073 
1074 void vmaFreeStatsString(
1075  VmaAllocator allocator,
1076  char* pStatsString);
1077 
1078 #endif // #if VMA_STATS_STRING_ENABLED
1079 
1080 VK_DEFINE_HANDLE(VmaPool)
1081 
1082 typedef enum VmaMemoryUsage
1083 {
1087  VMA_MEMORY_USAGE_UNKNOWN = 0,
1104  VMA_MEMORY_USAGE_GPU_ONLY = 1,
1114  VMA_MEMORY_USAGE_CPU_ONLY = 2,
1121  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1130  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1131  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1132 } VmaMemoryUsage;
1133 
1135 typedef enum VmaAllocationCreateFlagBits {
1147  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1148 
1158  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1171  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1184  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1191  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1197  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1198 
1199  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1200 } VmaAllocationCreateFlagBits;
1201 typedef VkFlags VmaAllocationCreateFlags;
1202 
1203 typedef struct VmaAllocationCreateInfo
1204 {
1206  VmaAllocationCreateFlags flags;
1212  VmaMemoryUsage usage;
1217  VkMemoryPropertyFlags requiredFlags;
1222  VkMemoryPropertyFlags preferredFlags;
1230  uint32_t memoryTypeBits;
1236  VmaPool pool;
1243  void* pUserData;
1244 } VmaAllocationCreateInfo;
1245 
1260 VkResult vmaFindMemoryTypeIndex(
1261  VmaAllocator allocator,
1262  uint32_t memoryTypeBits,
1263  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1264  uint32_t* pMemoryTypeIndex);
1265 
1267 typedef enum VmaPoolCreateFlagBits {
1285  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1286 
1287  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1288 } VmaPoolCreateFlagBits;
1289 typedef VkFlags VmaPoolCreateFlags;
1290 
1293 typedef struct VmaPoolCreateInfo {
1296  uint32_t memoryTypeIndex;
1299  VmaPoolCreateFlags flags;
1304  VkDeviceSize blockSize;
1309  size_t minBlockCount;
1317  size_t maxBlockCount;
1331  uint32_t frameInUseCount;
1332 } VmaPoolCreateInfo;
1333 
1336 typedef struct VmaPoolStats {
1339  VkDeviceSize size;
1342  VkDeviceSize unusedSize;
1345  size_t allocationCount;
1348  size_t unusedRangeCount;
1355  VkDeviceSize unusedRangeSizeMax;
1356 } VmaPoolStats;
1357 
1364 VkResult vmaCreatePool(
1365  VmaAllocator allocator,
1366  const VmaPoolCreateInfo* pCreateInfo,
1367  VmaPool* pPool);
1368 
1371 void vmaDestroyPool(
1372  VmaAllocator allocator,
1373  VmaPool pool);
1374 
1381 void vmaGetPoolStats(
1382  VmaAllocator allocator,
1383  VmaPool pool,
1384  VmaPoolStats* pPoolStats);
1385 
1392 void vmaMakePoolAllocationsLost(
1393  VmaAllocator allocator,
1394  VmaPool pool,
1395  size_t* pLostAllocationCount);
1396 
1397 VK_DEFINE_HANDLE(VmaAllocation)
1398 
1399 
1401 typedef struct VmaAllocationInfo {
1406  uint32_t memoryType;
1415  VkDeviceMemory deviceMemory;
1420  VkDeviceSize offset;
1425  VkDeviceSize size;
1434  void* pMappedData;
1439  void* pUserData;
1440 } VmaAllocationInfo;
1441 
1452 VkResult vmaAllocateMemory(
1453  VmaAllocator allocator,
1454  const VkMemoryRequirements* pVkMemoryRequirements,
1455  const VmaAllocationCreateInfo* pCreateInfo,
1456  VmaAllocation* pAllocation,
1457  VmaAllocationInfo* pAllocationInfo);
1458 
1465 VkResult vmaAllocateMemoryForBuffer(
1466  VmaAllocator allocator,
1467  VkBuffer buffer,
1468  const VmaAllocationCreateInfo* pCreateInfo,
1469  VmaAllocation* pAllocation,
1470  VmaAllocationInfo* pAllocationInfo);
1471 
1473 VkResult vmaAllocateMemoryForImage(
1474  VmaAllocator allocator,
1475  VkImage image,
1476  const VmaAllocationCreateInfo* pCreateInfo,
1477  VmaAllocation* pAllocation,
1478  VmaAllocationInfo* pAllocationInfo);
1479 
1481 void vmaFreeMemory(
1482  VmaAllocator allocator,
1483  VmaAllocation allocation);
1484 
1486 void vmaGetAllocationInfo(
1487  VmaAllocator allocator,
1488  VmaAllocation allocation,
1489  VmaAllocationInfo* pAllocationInfo);
1490 
1504 void vmaSetAllocationUserData(
1505  VmaAllocator allocator,
1506  VmaAllocation allocation,
1507  void* pUserData);
1508 
1519 void vmaCreateLostAllocation(
1520  VmaAllocator allocator,
1521  VmaAllocation* pAllocation);
1522 
1557 VkResult vmaMapMemory(
1558  VmaAllocator allocator,
1559  VmaAllocation allocation,
1560  void** ppData);
1561 
1566 void vmaUnmapMemory(
1567  VmaAllocator allocator,
1568  VmaAllocation allocation);
1569 
1571 typedef struct VmaDefragmentationInfo {
1576  VkDeviceSize maxBytesToMove;
1581  uint32_t maxAllocationsToMove;
1582 } VmaDefragmentationInfo;
1583 
1585 typedef struct VmaDefragmentationStats {
1587  VkDeviceSize bytesMoved;
1589  VkDeviceSize bytesFreed;
1591  uint32_t allocationsMoved;
1593  uint32_t deviceMemoryBlocksFreed;
1594 } VmaDefragmentationStats;
1595 
1678 VkResult vmaDefragment(
1679  VmaAllocator allocator,
1680  VmaAllocation* pAllocations,
1681  size_t allocationCount,
1682  VkBool32* pAllocationsChanged,
1683  const VmaDefragmentationInfo *pDefragmentationInfo,
1684  VmaDefragmentationStats* pDefragmentationStats);
1685 
1712 VkResult vmaCreateBuffer(
1713  VmaAllocator allocator,
1714  const VkBufferCreateInfo* pBufferCreateInfo,
1715  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1716  VkBuffer* pBuffer,
1717  VmaAllocation* pAllocation,
1718  VmaAllocationInfo* pAllocationInfo);
1719 
1731 void vmaDestroyBuffer(
1732  VmaAllocator allocator,
1733  VkBuffer buffer,
1734  VmaAllocation allocation);
1735 
1737 VkResult vmaCreateImage(
1738  VmaAllocator allocator,
1739  const VkImageCreateInfo* pImageCreateInfo,
1740  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1741  VkImage* pImage,
1742  VmaAllocation* pAllocation,
1743  VmaAllocationInfo* pAllocationInfo);
1744 
1756 void vmaDestroyImage(
1757  VmaAllocator allocator,
1758  VkImage image,
1759  VmaAllocation allocation);
1760 
1761 #ifdef __cplusplus
1762 }
1763 #endif
1764 
1765 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
1766 
1767 // For Visual Studio IntelliSense.
1768 #ifdef __INTELLISENSE__
1769 #define VMA_IMPLEMENTATION
1770 #endif
1771 
1772 #ifdef VMA_IMPLEMENTATION
1773 #undef VMA_IMPLEMENTATION
1774 
1775 #include <cstdint>
1776 #include <cstdlib>
1777 #include <cstring>
1778 
1779 /*******************************************************************************
1780 CONFIGURATION SECTION
1781 
1782 Define some of these macros before each #include of this header or change them
1783 here if you need other then default behavior depending on your environment.
1784 */
1785 
1786 /*
1787 Define this macro to 1 to make the library fetch pointers to Vulkan functions
1788 internally, like:
1789 
1790  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1791 
1792 Define to 0 if you are going to provide you own pointers to Vulkan functions via
1793 VmaAllocatorCreateInfo::pVulkanFunctions.
1794 */
1795 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
1796 #define VMA_STATIC_VULKAN_FUNCTIONS 1
1797 #endif
1798 
1799 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
1800 //#define VMA_USE_STL_CONTAINERS 1
1801 
1802 /* Set this macro to 1 to make the library including and using STL containers:
1803 std::pair, std::vector, std::list, std::unordered_map.
1804 
1805 Set it to 0 or undefined to make the library using its own implementation of
1806 the containers.
1807 */
1808 #if VMA_USE_STL_CONTAINERS
1809  #define VMA_USE_STL_VECTOR 1
1810  #define VMA_USE_STL_UNORDERED_MAP 1
1811  #define VMA_USE_STL_LIST 1
1812 #endif
1813 
1814 #if VMA_USE_STL_VECTOR
1815  #include <vector>
1816 #endif
1817 
1818 #if VMA_USE_STL_UNORDERED_MAP
1819  #include <unordered_map>
1820 #endif
1821 
1822 #if VMA_USE_STL_LIST
1823  #include <list>
1824 #endif
1825 
1826 /*
1827 Following headers are used in this CONFIGURATION section only, so feel free to
1828 remove them if not needed.
1829 */
1830 #include <cassert> // for assert
1831 #include <algorithm> // for min, max
1832 #include <mutex> // for std::mutex
1833 #include <atomic> // for std::atomic
1834 
1835 #if !defined(_WIN32) && !defined(__APPLE__)
1836  #include <malloc.h> // for aligned_alloc()
1837 #endif
1838 
1839 #if defined(__APPLE__)
1840 #include <cstdlib>
1841 void *aligned_alloc(size_t alignment, size_t size)
1842 {
1843  // alignment must be >= sizeof(void*)
1844  if(alignment < sizeof(void*))
1845  {
1846  alignment = sizeof(void*);
1847  }
1848 
1849  void *pointer;
1850  if(posix_memalign(&pointer, alignment, size) == 0)
1851  return pointer;
1852  return VMA_NULL;
1853 }
1854 #endif
1855 
1856 // Normal assert to check for programmer's errors, especially in Debug configuration.
1857 #ifndef VMA_ASSERT
1858  #ifdef _DEBUG
1859  #define VMA_ASSERT(expr) assert(expr)
1860  #else
1861  #define VMA_ASSERT(expr)
1862  #endif
1863 #endif
1864 
1865 // Assert that will be called very often, like inside data structures e.g. operator[].
1866 // Making it non-empty can make program slow.
1867 #ifndef VMA_HEAVY_ASSERT
1868  #ifdef _DEBUG
1869  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
1870  #else
1871  #define VMA_HEAVY_ASSERT(expr)
1872  #endif
1873 #endif
1874 
1875 #ifndef VMA_NULL
1876  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
1877  #define VMA_NULL nullptr
1878 #endif
1879 
1880 #ifndef VMA_ALIGN_OF
1881  #define VMA_ALIGN_OF(type) (__alignof(type))
1882 #endif
1883 
1884 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
1885  #if defined(_WIN32)
1886  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
1887  #else
1888  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
1889  #endif
1890 #endif
1891 
1892 #ifndef VMA_SYSTEM_FREE
1893  #if defined(_WIN32)
1894  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
1895  #else
1896  #define VMA_SYSTEM_FREE(ptr) free(ptr)
1897  #endif
1898 #endif
1899 
1900 #ifndef VMA_MIN
1901  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
1902 #endif
1903 
1904 #ifndef VMA_MAX
1905  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
1906 #endif
1907 
1908 #ifndef VMA_SWAP
1909  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
1910 #endif
1911 
1912 #ifndef VMA_SORT
1913  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
1914 #endif
1915 
1916 #ifndef VMA_DEBUG_LOG
1917  #define VMA_DEBUG_LOG(format, ...)
1918  /*
1919  #define VMA_DEBUG_LOG(format, ...) do { \
1920  printf(format, __VA_ARGS__); \
1921  printf("\n"); \
1922  } while(false)
1923  */
1924 #endif
1925 
1926 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
1927 #if VMA_STATS_STRING_ENABLED
1928  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
1929  {
1930  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
1931  }
1932  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
1933  {
1934  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
1935  }
1936  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
1937  {
1938  snprintf(outStr, strLen, "%p", ptr);
1939  }
1940 #endif
1941 
1942 #ifndef VMA_MUTEX
1943  class VmaMutex
1944  {
1945  public:
1946  VmaMutex() { }
1947  ~VmaMutex() { }
1948  void Lock() { m_Mutex.lock(); }
1949  void Unlock() { m_Mutex.unlock(); }
1950  private:
1951  std::mutex m_Mutex;
1952  };
1953  #define VMA_MUTEX VmaMutex
1954 #endif
1955 
1956 /*
1957 If providing your own implementation, you need to implement a subset of std::atomic:
1958 
1959 - Constructor(uint32_t desired)
1960 - uint32_t load() const
1961 - void store(uint32_t desired)
1962 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
1963 */
1964 #ifndef VMA_ATOMIC_UINT32
1965  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
1966 #endif
1967 
1968 #ifndef VMA_BEST_FIT
1969 
1981  #define VMA_BEST_FIT (1)
1982 #endif
1983 
1984 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
1985 
1989  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
1990 #endif
1991 
1992 #ifndef VMA_DEBUG_ALIGNMENT
1993 
1997  #define VMA_DEBUG_ALIGNMENT (1)
1998 #endif
1999 
2000 #ifndef VMA_DEBUG_MARGIN
2001 
2005  #define VMA_DEBUG_MARGIN (0)
2006 #endif
2007 
2008 #ifndef VMA_DEBUG_GLOBAL_MUTEX
2009 
2013  #define VMA_DEBUG_GLOBAL_MUTEX (0)
2014 #endif
2015 
2016 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
2017 
2021  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
2022 #endif
2023 
2024 #ifndef VMA_SMALL_HEAP_MAX_SIZE
2025  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
2027 #endif
2028 
2029 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
2030  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
2032 #endif
2033 
2034 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
2035 
2036 /*******************************************************************************
2037 END OF CONFIGURATION
2038 */
2039 
2040 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
2041  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
2042 
2043 // Returns number of bits set to 1 in (v).
2044 static inline uint32_t VmaCountBitsSet(uint32_t v)
2045 {
2046  uint32_t c = v - ((v >> 1) & 0x55555555);
2047  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
2048  c = ((c >> 4) + c) & 0x0F0F0F0F;
2049  c = ((c >> 8) + c) & 0x00FF00FF;
2050  c = ((c >> 16) + c) & 0x0000FFFF;
2051  return c;
2052 }
2053 
2054 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2055 // Use types like uint32_t, uint64_t as T.
2056 template <typename T>
2057 static inline T VmaAlignUp(T val, T align)
2058 {
2059  return (val + align - 1) / align * align;
2060 }
2061 
2062 // Division with mathematical rounding to nearest number.
2063 template <typename T>
2064 inline T VmaRoundDiv(T x, T y)
2065 {
2066  return (x + (y / (T)2)) / y;
2067 }
2068 
2069 #ifndef VMA_SORT
2070 
2071 template<typename Iterator, typename Compare>
2072 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2073 {
2074  Iterator centerValue = end; --centerValue;
2075  Iterator insertIndex = beg;
2076  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2077  {
2078  if(cmp(*memTypeIndex, *centerValue))
2079  {
2080  if(insertIndex != memTypeIndex)
2081  {
2082  VMA_SWAP(*memTypeIndex, *insertIndex);
2083  }
2084  ++insertIndex;
2085  }
2086  }
2087  if(insertIndex != centerValue)
2088  {
2089  VMA_SWAP(*insertIndex, *centerValue);
2090  }
2091  return insertIndex;
2092 }
2093 
2094 template<typename Iterator, typename Compare>
2095 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2096 {
2097  if(beg < end)
2098  {
2099  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2100  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2101  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2102  }
2103 }
2104 
2105 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2106 
2107 #endif // #ifndef VMA_SORT
2108 
2109 /*
2110 Returns true if two memory blocks occupy overlapping pages.
2111 ResourceA must be in less memory offset than ResourceB.
2112 
2113 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2114 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2115 */
2116 static inline bool VmaBlocksOnSamePage(
2117  VkDeviceSize resourceAOffset,
2118  VkDeviceSize resourceASize,
2119  VkDeviceSize resourceBOffset,
2120  VkDeviceSize pageSize)
2121 {
2122  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2123  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2124  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2125  VkDeviceSize resourceBStart = resourceBOffset;
2126  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2127  return resourceAEndPage == resourceBStartPage;
2128 }
2129 
2130 enum VmaSuballocationType
2131 {
2132  VMA_SUBALLOCATION_TYPE_FREE = 0,
2133  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2134  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2135  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2136  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2137  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2138  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2139 };
2140 
2141 /*
2142 Returns true if given suballocation types could conflict and must respect
2143 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2144 or linear image and another one is optimal image. If type is unknown, behave
2145 conservatively.
2146 */
2147 static inline bool VmaIsBufferImageGranularityConflict(
2148  VmaSuballocationType suballocType1,
2149  VmaSuballocationType suballocType2)
2150 {
2151  if(suballocType1 > suballocType2)
2152  {
2153  VMA_SWAP(suballocType1, suballocType2);
2154  }
2155 
2156  switch(suballocType1)
2157  {
2158  case VMA_SUBALLOCATION_TYPE_FREE:
2159  return false;
2160  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
2161  return true;
2162  case VMA_SUBALLOCATION_TYPE_BUFFER:
2163  return
2164  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2165  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2166  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
2167  return
2168  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2169  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
2170  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2171  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
2172  return
2173  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2174  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
2175  return false;
2176  default:
2177  VMA_ASSERT(0);
2178  return true;
2179  }
2180 }
2181 
2182 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
2183 struct VmaMutexLock
2184 {
2185 public:
2186  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
2187  m_pMutex(useMutex ? &mutex : VMA_NULL)
2188  {
2189  if(m_pMutex)
2190  {
2191  m_pMutex->Lock();
2192  }
2193  }
2194 
2195  ~VmaMutexLock()
2196  {
2197  if(m_pMutex)
2198  {
2199  m_pMutex->Unlock();
2200  }
2201  }
2202 
2203 private:
2204  VMA_MUTEX* m_pMutex;
2205 };
2206 
2207 #if VMA_DEBUG_GLOBAL_MUTEX
2208  static VMA_MUTEX gDebugGlobalMutex;
2209  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
2210 #else
2211  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
2212 #endif
2213 
2214 // Minimum size of a free suballocation to register it in the free suballocation collection.
2215 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
2216 
2217 /*
2218 Performs binary search and returns iterator to first element that is greater or
2219 equal to (key), according to comparison (cmp).
2220 
2221 Cmp should return true if first argument is less than second argument.
2222 
2223 Returned value is the found element, if present in the collection or place where
2224 new element with value (key) should be inserted.
2225 */
2226 template <typename IterT, typename KeyT, typename CmpT>
2227 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
2228 {
2229  size_t down = 0, up = (end - beg);
2230  while(down < up)
2231  {
2232  const size_t mid = (down + up) / 2;
2233  if(cmp(*(beg+mid), key))
2234  {
2235  down = mid + 1;
2236  }
2237  else
2238  {
2239  up = mid;
2240  }
2241  }
2242  return beg + down;
2243 }
2244 
2246 // Memory allocation
2247 
2248 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
2249 {
2250  if((pAllocationCallbacks != VMA_NULL) &&
2251  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
2252  {
2253  return (*pAllocationCallbacks->pfnAllocation)(
2254  pAllocationCallbacks->pUserData,
2255  size,
2256  alignment,
2257  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2258  }
2259  else
2260  {
2261  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
2262  }
2263 }
2264 
2265 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
2266 {
2267  if((pAllocationCallbacks != VMA_NULL) &&
2268  (pAllocationCallbacks->pfnFree != VMA_NULL))
2269  {
2270  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
2271  }
2272  else
2273  {
2274  VMA_SYSTEM_FREE(ptr);
2275  }
2276 }
2277 
2278 template<typename T>
2279 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
2280 {
2281  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
2282 }
2283 
2284 template<typename T>
2285 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
2286 {
2287  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
2288 }
2289 
2290 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
2291 
2292 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
2293 
2294 template<typename T>
2295 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
2296 {
2297  ptr->~T();
2298  VmaFree(pAllocationCallbacks, ptr);
2299 }
2300 
2301 template<typename T>
2302 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
2303 {
2304  if(ptr != VMA_NULL)
2305  {
2306  for(size_t i = count; i--; )
2307  {
2308  ptr[i].~T();
2309  }
2310  VmaFree(pAllocationCallbacks, ptr);
2311  }
2312 }
2313 
2314 // STL-compatible allocator.
2315 template<typename T>
2316 class VmaStlAllocator
2317 {
2318 public:
2319  const VkAllocationCallbacks* const m_pCallbacks;
2320  typedef T value_type;
2321 
2322  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
2323  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
2324 
2325  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
2326  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
2327 
2328  template<typename U>
2329  bool operator==(const VmaStlAllocator<U>& rhs) const
2330  {
2331  return m_pCallbacks == rhs.m_pCallbacks;
2332  }
2333  template<typename U>
2334  bool operator!=(const VmaStlAllocator<U>& rhs) const
2335  {
2336  return m_pCallbacks != rhs.m_pCallbacks;
2337  }
2338 
2339  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
2340 };
2341 
2342 #if VMA_USE_STL_VECTOR
2343 
2344 #define VmaVector std::vector
2345 
2346 template<typename T, typename allocatorT>
2347 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
2348 {
2349  vec.insert(vec.begin() + index, item);
2350 }
2351 
2352 template<typename T, typename allocatorT>
2353 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
2354 {
2355  vec.erase(vec.begin() + index);
2356 }
2357 
2358 #else // #if VMA_USE_STL_VECTOR
2359 
2360 /* Class with interface compatible with subset of std::vector.
2361 T must be POD because constructors and destructors are not called and memcpy is
2362 used for these objects. */
2363 template<typename T, typename AllocatorT>
2364 class VmaVector
2365 {
2366 public:
2367  typedef T value_type;
2368 
2369  VmaVector(const AllocatorT& allocator) :
2370  m_Allocator(allocator),
2371  m_pArray(VMA_NULL),
2372  m_Count(0),
2373  m_Capacity(0)
2374  {
2375  }
2376 
2377  VmaVector(size_t count, const AllocatorT& allocator) :
2378  m_Allocator(allocator),
2379  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
2380  m_Count(count),
2381  m_Capacity(count)
2382  {
2383  }
2384 
2385  VmaVector(const VmaVector<T, AllocatorT>& src) :
2386  m_Allocator(src.m_Allocator),
2387  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
2388  m_Count(src.m_Count),
2389  m_Capacity(src.m_Count)
2390  {
2391  if(m_Count != 0)
2392  {
2393  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
2394  }
2395  }
2396 
2397  ~VmaVector()
2398  {
2399  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2400  }
2401 
2402  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
2403  {
2404  if(&rhs != this)
2405  {
2406  resize(rhs.m_Count);
2407  if(m_Count != 0)
2408  {
2409  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
2410  }
2411  }
2412  return *this;
2413  }
2414 
2415  bool empty() const { return m_Count == 0; }
2416  size_t size() const { return m_Count; }
2417  T* data() { return m_pArray; }
2418  const T* data() const { return m_pArray; }
2419 
2420  T& operator[](size_t index)
2421  {
2422  VMA_HEAVY_ASSERT(index < m_Count);
2423  return m_pArray[index];
2424  }
2425  const T& operator[](size_t index) const
2426  {
2427  VMA_HEAVY_ASSERT(index < m_Count);
2428  return m_pArray[index];
2429  }
2430 
2431  T& front()
2432  {
2433  VMA_HEAVY_ASSERT(m_Count > 0);
2434  return m_pArray[0];
2435  }
2436  const T& front() const
2437  {
2438  VMA_HEAVY_ASSERT(m_Count > 0);
2439  return m_pArray[0];
2440  }
2441  T& back()
2442  {
2443  VMA_HEAVY_ASSERT(m_Count > 0);
2444  return m_pArray[m_Count - 1];
2445  }
2446  const T& back() const
2447  {
2448  VMA_HEAVY_ASSERT(m_Count > 0);
2449  return m_pArray[m_Count - 1];
2450  }
2451 
2452  void reserve(size_t newCapacity, bool freeMemory = false)
2453  {
2454  newCapacity = VMA_MAX(newCapacity, m_Count);
2455 
2456  if((newCapacity < m_Capacity) && !freeMemory)
2457  {
2458  newCapacity = m_Capacity;
2459  }
2460 
2461  if(newCapacity != m_Capacity)
2462  {
2463  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
2464  if(m_Count != 0)
2465  {
2466  memcpy(newArray, m_pArray, m_Count * sizeof(T));
2467  }
2468  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2469  m_Capacity = newCapacity;
2470  m_pArray = newArray;
2471  }
2472  }
2473 
2474  void resize(size_t newCount, bool freeMemory = false)
2475  {
2476  size_t newCapacity = m_Capacity;
2477  if(newCount > m_Capacity)
2478  {
2479  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
2480  }
2481  else if(freeMemory)
2482  {
2483  newCapacity = newCount;
2484  }
2485 
2486  if(newCapacity != m_Capacity)
2487  {
2488  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
2489  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
2490  if(elementsToCopy != 0)
2491  {
2492  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
2493  }
2494  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2495  m_Capacity = newCapacity;
2496  m_pArray = newArray;
2497  }
2498 
2499  m_Count = newCount;
2500  }
2501 
2502  void clear(bool freeMemory = false)
2503  {
2504  resize(0, freeMemory);
2505  }
2506 
2507  void insert(size_t index, const T& src)
2508  {
2509  VMA_HEAVY_ASSERT(index <= m_Count);
2510  const size_t oldCount = size();
2511  resize(oldCount + 1);
2512  if(index < oldCount)
2513  {
2514  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
2515  }
2516  m_pArray[index] = src;
2517  }
2518 
2519  void remove(size_t index)
2520  {
2521  VMA_HEAVY_ASSERT(index < m_Count);
2522  const size_t oldCount = size();
2523  if(index < oldCount - 1)
2524  {
2525  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
2526  }
2527  resize(oldCount - 1);
2528  }
2529 
2530  void push_back(const T& src)
2531  {
2532  const size_t newIndex = size();
2533  resize(newIndex + 1);
2534  m_pArray[newIndex] = src;
2535  }
2536 
2537  void pop_back()
2538  {
2539  VMA_HEAVY_ASSERT(m_Count > 0);
2540  resize(size() - 1);
2541  }
2542 
2543  void push_front(const T& src)
2544  {
2545  insert(0, src);
2546  }
2547 
2548  void pop_front()
2549  {
2550  VMA_HEAVY_ASSERT(m_Count > 0);
2551  remove(0);
2552  }
2553 
2554  typedef T* iterator;
2555 
2556  iterator begin() { return m_pArray; }
2557  iterator end() { return m_pArray + m_Count; }
2558 
2559 private:
2560  AllocatorT m_Allocator;
2561  T* m_pArray;
2562  size_t m_Count;
2563  size_t m_Capacity;
2564 };
2565 
2566 template<typename T, typename allocatorT>
2567 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2568 {
2569  vec.insert(index, item);
2570 }
2571 
2572 template<typename T, typename allocatorT>
2573 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2574 {
2575  vec.remove(index);
2576 }
2577 
2578 #endif // #if VMA_USE_STL_VECTOR
2579 
2580 template<typename CmpLess, typename VectorT>
2581 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2582 {
2583  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2584  vector.data(),
2585  vector.data() + vector.size(),
2586  value,
2587  CmpLess()) - vector.data();
2588  VmaVectorInsert(vector, indexToInsert, value);
2589  return indexToInsert;
2590 }
2591 
2592 template<typename CmpLess, typename VectorT>
2593 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2594 {
2595  CmpLess comparator;
2596  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2597  vector.begin(),
2598  vector.end(),
2599  value,
2600  comparator);
2601  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
2602  {
2603  size_t indexToRemove = it - vector.begin();
2604  VmaVectorRemove(vector, indexToRemove);
2605  return true;
2606  }
2607  return false;
2608 }
2609 
2610 template<typename CmpLess, typename VectorT>
2611 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
2612 {
2613  CmpLess comparator;
2614  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2615  vector.data(),
2616  vector.data() + vector.size(),
2617  value,
2618  comparator);
2619  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
2620  {
2621  return it - vector.begin();
2622  }
2623  else
2624  {
2625  return vector.size();
2626  }
2627 }
2628 
2630 // class VmaPoolAllocator
2631 
2632 /*
2633 Allocator for objects of type T using a list of arrays (pools) to speed up
2634 allocation. Number of elements that can be allocated is not bounded because
2635 allocator can create multiple blocks.
2636 */
2637 template<typename T>
2638 class VmaPoolAllocator
2639 {
2640 public:
2641  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
2642  ~VmaPoolAllocator();
2643  void Clear();
2644  T* Alloc();
2645  void Free(T* ptr);
2646 
2647 private:
2648  union Item
2649  {
2650  uint32_t NextFreeIndex;
2651  T Value;
2652  };
2653 
2654  struct ItemBlock
2655  {
2656  Item* pItems;
2657  uint32_t FirstFreeIndex;
2658  };
2659 
2660  const VkAllocationCallbacks* m_pAllocationCallbacks;
2661  size_t m_ItemsPerBlock;
2662  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
2663 
2664  ItemBlock& CreateNewBlock();
2665 };
2666 
2667 template<typename T>
2668 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
2669  m_pAllocationCallbacks(pAllocationCallbacks),
2670  m_ItemsPerBlock(itemsPerBlock),
2671  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
2672 {
2673  VMA_ASSERT(itemsPerBlock > 0);
2674 }
2675 
2676 template<typename T>
2677 VmaPoolAllocator<T>::~VmaPoolAllocator()
2678 {
2679  Clear();
2680 }
2681 
2682 template<typename T>
2683 void VmaPoolAllocator<T>::Clear()
2684 {
2685  for(size_t i = m_ItemBlocks.size(); i--; )
2686  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
2687  m_ItemBlocks.clear();
2688 }
2689 
2690 template<typename T>
2691 T* VmaPoolAllocator<T>::Alloc()
2692 {
2693  for(size_t i = m_ItemBlocks.size(); i--; )
2694  {
2695  ItemBlock& block = m_ItemBlocks[i];
2696  // This block has some free items: Use first one.
2697  if(block.FirstFreeIndex != UINT32_MAX)
2698  {
2699  Item* const pItem = &block.pItems[block.FirstFreeIndex];
2700  block.FirstFreeIndex = pItem->NextFreeIndex;
2701  return &pItem->Value;
2702  }
2703  }
2704 
2705  // No block has free item: Create new one and use it.
2706  ItemBlock& newBlock = CreateNewBlock();
2707  Item* const pItem = &newBlock.pItems[0];
2708  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
2709  return &pItem->Value;
2710 }
2711 
2712 template<typename T>
2713 void VmaPoolAllocator<T>::Free(T* ptr)
2714 {
2715  // Search all memory blocks to find ptr.
2716  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
2717  {
2718  ItemBlock& block = m_ItemBlocks[i];
2719 
2720  // Casting to union.
2721  Item* pItemPtr;
2722  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
2723 
2724  // Check if pItemPtr is in address range of this block.
2725  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
2726  {
2727  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
2728  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
2729  block.FirstFreeIndex = index;
2730  return;
2731  }
2732  }
2733  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
2734 }
2735 
2736 template<typename T>
2737 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
2738 {
2739  ItemBlock newBlock = {
2740  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
2741 
2742  m_ItemBlocks.push_back(newBlock);
2743 
2744  // Setup singly-linked list of all free items in this block.
2745  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
2746  newBlock.pItems[i].NextFreeIndex = i + 1;
2747  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
2748  return m_ItemBlocks.back();
2749 }
2750 
2752 // class VmaRawList, VmaList
2753 
2754 #if VMA_USE_STL_LIST
2755 
2756 #define VmaList std::list
2757 
2758 #else // #if VMA_USE_STL_LIST
2759 
2760 template<typename T>
2761 struct VmaListItem
2762 {
2763  VmaListItem* pPrev;
2764  VmaListItem* pNext;
2765  T Value;
2766 };
2767 
2768 // Doubly linked list.
2769 template<typename T>
2770 class VmaRawList
2771 {
2772 public:
2773  typedef VmaListItem<T> ItemType;
2774 
2775  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
2776  ~VmaRawList();
2777  void Clear();
2778 
2779  size_t GetCount() const { return m_Count; }
2780  bool IsEmpty() const { return m_Count == 0; }
2781 
2782  ItemType* Front() { return m_pFront; }
2783  const ItemType* Front() const { return m_pFront; }
2784  ItemType* Back() { return m_pBack; }
2785  const ItemType* Back() const { return m_pBack; }
2786 
2787  ItemType* PushBack();
2788  ItemType* PushFront();
2789  ItemType* PushBack(const T& value);
2790  ItemType* PushFront(const T& value);
2791  void PopBack();
2792  void PopFront();
2793 
2794  // Item can be null - it means PushBack.
2795  ItemType* InsertBefore(ItemType* pItem);
2796  // Item can be null - it means PushFront.
2797  ItemType* InsertAfter(ItemType* pItem);
2798 
2799  ItemType* InsertBefore(ItemType* pItem, const T& value);
2800  ItemType* InsertAfter(ItemType* pItem, const T& value);
2801 
2802  void Remove(ItemType* pItem);
2803 
2804 private:
2805  const VkAllocationCallbacks* const m_pAllocationCallbacks;
2806  VmaPoolAllocator<ItemType> m_ItemAllocator;
2807  ItemType* m_pFront;
2808  ItemType* m_pBack;
2809  size_t m_Count;
2810 
2811  // Declared not defined, to block copy constructor and assignment operator.
2812  VmaRawList(const VmaRawList<T>& src);
2813  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
2814 };
2815 
2816 template<typename T>
2817 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
2818  m_pAllocationCallbacks(pAllocationCallbacks),
2819  m_ItemAllocator(pAllocationCallbacks, 128),
2820  m_pFront(VMA_NULL),
2821  m_pBack(VMA_NULL),
2822  m_Count(0)
2823 {
2824 }
2825 
2826 template<typename T>
2827 VmaRawList<T>::~VmaRawList()
2828 {
2829  // Intentionally not calling Clear, because that would be unnecessary
2830  // computations to return all items to m_ItemAllocator as free.
2831 }
2832 
2833 template<typename T>
2834 void VmaRawList<T>::Clear()
2835 {
2836  if(IsEmpty() == false)
2837  {
2838  ItemType* pItem = m_pBack;
2839  while(pItem != VMA_NULL)
2840  {
2841  ItemType* const pPrevItem = pItem->pPrev;
2842  m_ItemAllocator.Free(pItem);
2843  pItem = pPrevItem;
2844  }
2845  m_pFront = VMA_NULL;
2846  m_pBack = VMA_NULL;
2847  m_Count = 0;
2848  }
2849 }
2850 
2851 template<typename T>
2852 VmaListItem<T>* VmaRawList<T>::PushBack()
2853 {
2854  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2855  pNewItem->pNext = VMA_NULL;
2856  if(IsEmpty())
2857  {
2858  pNewItem->pPrev = VMA_NULL;
2859  m_pFront = pNewItem;
2860  m_pBack = pNewItem;
2861  m_Count = 1;
2862  }
2863  else
2864  {
2865  pNewItem->pPrev = m_pBack;
2866  m_pBack->pNext = pNewItem;
2867  m_pBack = pNewItem;
2868  ++m_Count;
2869  }
2870  return pNewItem;
2871 }
2872 
2873 template<typename T>
2874 VmaListItem<T>* VmaRawList<T>::PushFront()
2875 {
2876  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2877  pNewItem->pPrev = VMA_NULL;
2878  if(IsEmpty())
2879  {
2880  pNewItem->pNext = VMA_NULL;
2881  m_pFront = pNewItem;
2882  m_pBack = pNewItem;
2883  m_Count = 1;
2884  }
2885  else
2886  {
2887  pNewItem->pNext = m_pFront;
2888  m_pFront->pPrev = pNewItem;
2889  m_pFront = pNewItem;
2890  ++m_Count;
2891  }
2892  return pNewItem;
2893 }
2894 
2895 template<typename T>
2896 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
2897 {
2898  ItemType* const pNewItem = PushBack();
2899  pNewItem->Value = value;
2900  return pNewItem;
2901 }
2902 
2903 template<typename T>
2904 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
2905 {
2906  ItemType* const pNewItem = PushFront();
2907  pNewItem->Value = value;
2908  return pNewItem;
2909 }
2910 
2911 template<typename T>
2912 void VmaRawList<T>::PopBack()
2913 {
2914  VMA_HEAVY_ASSERT(m_Count > 0);
2915  ItemType* const pBackItem = m_pBack;
2916  ItemType* const pPrevItem = pBackItem->pPrev;
2917  if(pPrevItem != VMA_NULL)
2918  {
2919  pPrevItem->pNext = VMA_NULL;
2920  }
2921  m_pBack = pPrevItem;
2922  m_ItemAllocator.Free(pBackItem);
2923  --m_Count;
2924 }
2925 
2926 template<typename T>
2927 void VmaRawList<T>::PopFront()
2928 {
2929  VMA_HEAVY_ASSERT(m_Count > 0);
2930  ItemType* const pFrontItem = m_pFront;
2931  ItemType* const pNextItem = pFrontItem->pNext;
2932  if(pNextItem != VMA_NULL)
2933  {
2934  pNextItem->pPrev = VMA_NULL;
2935  }
2936  m_pFront = pNextItem;
2937  m_ItemAllocator.Free(pFrontItem);
2938  --m_Count;
2939 }
2940 
2941 template<typename T>
2942 void VmaRawList<T>::Remove(ItemType* pItem)
2943 {
2944  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
2945  VMA_HEAVY_ASSERT(m_Count > 0);
2946 
2947  if(pItem->pPrev != VMA_NULL)
2948  {
2949  pItem->pPrev->pNext = pItem->pNext;
2950  }
2951  else
2952  {
2953  VMA_HEAVY_ASSERT(m_pFront == pItem);
2954  m_pFront = pItem->pNext;
2955  }
2956 
2957  if(pItem->pNext != VMA_NULL)
2958  {
2959  pItem->pNext->pPrev = pItem->pPrev;
2960  }
2961  else
2962  {
2963  VMA_HEAVY_ASSERT(m_pBack == pItem);
2964  m_pBack = pItem->pPrev;
2965  }
2966 
2967  m_ItemAllocator.Free(pItem);
2968  --m_Count;
2969 }
2970 
2971 template<typename T>
2972 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
2973 {
2974  if(pItem != VMA_NULL)
2975  {
2976  ItemType* const prevItem = pItem->pPrev;
2977  ItemType* const newItem = m_ItemAllocator.Alloc();
2978  newItem->pPrev = prevItem;
2979  newItem->pNext = pItem;
2980  pItem->pPrev = newItem;
2981  if(prevItem != VMA_NULL)
2982  {
2983  prevItem->pNext = newItem;
2984  }
2985  else
2986  {
2987  VMA_HEAVY_ASSERT(m_pFront == pItem);
2988  m_pFront = newItem;
2989  }
2990  ++m_Count;
2991  return newItem;
2992  }
2993  else
2994  return PushBack();
2995 }
2996 
2997 template<typename T>
2998 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
2999 {
3000  if(pItem != VMA_NULL)
3001  {
3002  ItemType* const nextItem = pItem->pNext;
3003  ItemType* const newItem = m_ItemAllocator.Alloc();
3004  newItem->pNext = nextItem;
3005  newItem->pPrev = pItem;
3006  pItem->pNext = newItem;
3007  if(nextItem != VMA_NULL)
3008  {
3009  nextItem->pPrev = newItem;
3010  }
3011  else
3012  {
3013  VMA_HEAVY_ASSERT(m_pBack == pItem);
3014  m_pBack = newItem;
3015  }
3016  ++m_Count;
3017  return newItem;
3018  }
3019  else
3020  return PushFront();
3021 }
3022 
3023 template<typename T>
3024 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
3025 {
3026  ItemType* const newItem = InsertBefore(pItem);
3027  newItem->Value = value;
3028  return newItem;
3029 }
3030 
3031 template<typename T>
3032 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
3033 {
3034  ItemType* const newItem = InsertAfter(pItem);
3035  newItem->Value = value;
3036  return newItem;
3037 }
3038 
3039 template<typename T, typename AllocatorT>
3040 class VmaList
3041 {
3042 public:
3043  class iterator
3044  {
3045  public:
3046  iterator() :
3047  m_pList(VMA_NULL),
3048  m_pItem(VMA_NULL)
3049  {
3050  }
3051 
3052  T& operator*() const
3053  {
3054  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3055  return m_pItem->Value;
3056  }
3057  T* operator->() const
3058  {
3059  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3060  return &m_pItem->Value;
3061  }
3062 
3063  iterator& operator++()
3064  {
3065  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3066  m_pItem = m_pItem->pNext;
3067  return *this;
3068  }
3069  iterator& operator--()
3070  {
3071  if(m_pItem != VMA_NULL)
3072  {
3073  m_pItem = m_pItem->pPrev;
3074  }
3075  else
3076  {
3077  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3078  m_pItem = m_pList->Back();
3079  }
3080  return *this;
3081  }
3082 
3083  iterator operator++(int)
3084  {
3085  iterator result = *this;
3086  ++*this;
3087  return result;
3088  }
3089  iterator operator--(int)
3090  {
3091  iterator result = *this;
3092  --*this;
3093  return result;
3094  }
3095 
3096  bool operator==(const iterator& rhs) const
3097  {
3098  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3099  return m_pItem == rhs.m_pItem;
3100  }
3101  bool operator!=(const iterator& rhs) const
3102  {
3103  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3104  return m_pItem != rhs.m_pItem;
3105  }
3106 
3107  private:
3108  VmaRawList<T>* m_pList;
3109  VmaListItem<T>* m_pItem;
3110 
3111  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3112  m_pList(pList),
3113  m_pItem(pItem)
3114  {
3115  }
3116 
3117  friend class VmaList<T, AllocatorT>;
3118  };
3119 
3120  class const_iterator
3121  {
3122  public:
3123  const_iterator() :
3124  m_pList(VMA_NULL),
3125  m_pItem(VMA_NULL)
3126  {
3127  }
3128 
3129  const_iterator(const iterator& src) :
3130  m_pList(src.m_pList),
3131  m_pItem(src.m_pItem)
3132  {
3133  }
3134 
3135  const T& operator*() const
3136  {
3137  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3138  return m_pItem->Value;
3139  }
3140  const T* operator->() const
3141  {
3142  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3143  return &m_pItem->Value;
3144  }
3145 
3146  const_iterator& operator++()
3147  {
3148  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3149  m_pItem = m_pItem->pNext;
3150  return *this;
3151  }
3152  const_iterator& operator--()
3153  {
3154  if(m_pItem != VMA_NULL)
3155  {
3156  m_pItem = m_pItem->pPrev;
3157  }
3158  else
3159  {
3160  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3161  m_pItem = m_pList->Back();
3162  }
3163  return *this;
3164  }
3165 
3166  const_iterator operator++(int)
3167  {
3168  const_iterator result = *this;
3169  ++*this;
3170  return result;
3171  }
3172  const_iterator operator--(int)
3173  {
3174  const_iterator result = *this;
3175  --*this;
3176  return result;
3177  }
3178 
3179  bool operator==(const const_iterator& rhs) const
3180  {
3181  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3182  return m_pItem == rhs.m_pItem;
3183  }
3184  bool operator!=(const const_iterator& rhs) const
3185  {
3186  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3187  return m_pItem != rhs.m_pItem;
3188  }
3189 
3190  private:
3191  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
3192  m_pList(pList),
3193  m_pItem(pItem)
3194  {
3195  }
3196 
3197  const VmaRawList<T>* m_pList;
3198  const VmaListItem<T>* m_pItem;
3199 
3200  friend class VmaList<T, AllocatorT>;
3201  };
3202 
3203  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
3204 
3205  bool empty() const { return m_RawList.IsEmpty(); }
3206  size_t size() const { return m_RawList.GetCount(); }
3207 
3208  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
3209  iterator end() { return iterator(&m_RawList, VMA_NULL); }
3210 
3211  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
3212  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
3213 
3214  void clear() { m_RawList.Clear(); }
3215  void push_back(const T& value) { m_RawList.PushBack(value); }
3216  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
3217  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
3218 
3219 private:
3220  VmaRawList<T> m_RawList;
3221 };
3222 
3223 #endif // #if VMA_USE_STL_LIST
3224 
3226 // class VmaMap
3227 
3228 // Unused in this version.
3229 #if 0
3230 
3231 #if VMA_USE_STL_UNORDERED_MAP
3232 
3233 #define VmaPair std::pair
3234 
3235 #define VMA_MAP_TYPE(KeyT, ValueT) \
3236  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
3237 
3238 #else // #if VMA_USE_STL_UNORDERED_MAP
3239 
3240 template<typename T1, typename T2>
3241 struct VmaPair
3242 {
3243  T1 first;
3244  T2 second;
3245 
3246  VmaPair() : first(), second() { }
3247  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
3248 };
3249 
3250 /* Class compatible with subset of interface of std::unordered_map.
3251 KeyT, ValueT must be POD because they will be stored in VmaVector.
3252 */
3253 template<typename KeyT, typename ValueT>
3254 class VmaMap
3255 {
3256 public:
3257  typedef VmaPair<KeyT, ValueT> PairType;
3258  typedef PairType* iterator;
3259 
3260  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
3261 
3262  iterator begin() { return m_Vector.begin(); }
3263  iterator end() { return m_Vector.end(); }
3264 
3265  void insert(const PairType& pair);
3266  iterator find(const KeyT& key);
3267  void erase(iterator it);
3268 
3269 private:
3270  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
3271 };
3272 
3273 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
3274 
3275 template<typename FirstT, typename SecondT>
3276 struct VmaPairFirstLess
3277 {
3278  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
3279  {
3280  return lhs.first < rhs.first;
3281  }
3282  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
3283  {
3284  return lhs.first < rhsFirst;
3285  }
3286 };
3287 
3288 template<typename KeyT, typename ValueT>
3289 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
3290 {
3291  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3292  m_Vector.data(),
3293  m_Vector.data() + m_Vector.size(),
3294  pair,
3295  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
3296  VmaVectorInsert(m_Vector, indexToInsert, pair);
3297 }
3298 
3299 template<typename KeyT, typename ValueT>
3300 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
3301 {
3302  PairType* it = VmaBinaryFindFirstNotLess(
3303  m_Vector.data(),
3304  m_Vector.data() + m_Vector.size(),
3305  key,
3306  VmaPairFirstLess<KeyT, ValueT>());
3307  if((it != m_Vector.end()) && (it->first == key))
3308  {
3309  return it;
3310  }
3311  else
3312  {
3313  return m_Vector.end();
3314  }
3315 }
3316 
3317 template<typename KeyT, typename ValueT>
3318 void VmaMap<KeyT, ValueT>::erase(iterator it)
3319 {
3320  VmaVectorRemove(m_Vector, it - m_Vector.begin());
3321 }
3322 
3323 #endif // #if VMA_USE_STL_UNORDERED_MAP
3324 
3325 #endif // #if 0
3326 
3328 
3329 class VmaDeviceMemoryBlock;
3330 
3331 struct VmaAllocation_T
3332 {
3333 private:
3334  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
3335 
3336  enum FLAGS
3337  {
3338  FLAG_USER_DATA_STRING = 0x01,
3339  };
3340 
3341 public:
3342  enum ALLOCATION_TYPE
3343  {
3344  ALLOCATION_TYPE_NONE,
3345  ALLOCATION_TYPE_BLOCK,
3346  ALLOCATION_TYPE_DEDICATED,
3347  };
3348 
3349  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
3350  m_Alignment(1),
3351  m_Size(0),
3352  m_pUserData(VMA_NULL),
3353  m_LastUseFrameIndex(currentFrameIndex),
3354  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
3355  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
3356  m_MapCount(0),
3357  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
3358  {
3359  }
3360 
3361  ~VmaAllocation_T()
3362  {
3363  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
3364 
3365  // Check if owned string was freed.
3366  VMA_ASSERT(m_pUserData == VMA_NULL);
3367  }
3368 
3369  void InitBlockAllocation(
3370  VmaPool hPool,
3371  VmaDeviceMemoryBlock* block,
3372  VkDeviceSize offset,
3373  VkDeviceSize alignment,
3374  VkDeviceSize size,
3375  VmaSuballocationType suballocationType,
3376  bool mapped,
3377  bool canBecomeLost)
3378  {
3379  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3380  VMA_ASSERT(block != VMA_NULL);
3381  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3382  m_Alignment = alignment;
3383  m_Size = size;
3384  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3385  m_SuballocationType = (uint8_t)suballocationType;
3386  m_BlockAllocation.m_hPool = hPool;
3387  m_BlockAllocation.m_Block = block;
3388  m_BlockAllocation.m_Offset = offset;
3389  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
3390  }
3391 
3392  void InitLost()
3393  {
3394  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3395  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
3396  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3397  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
3398  m_BlockAllocation.m_Block = VMA_NULL;
3399  m_BlockAllocation.m_Offset = 0;
3400  m_BlockAllocation.m_CanBecomeLost = true;
3401  }
3402 
3403  void ChangeBlockAllocation(
3404  VmaAllocator hAllocator,
3405  VmaDeviceMemoryBlock* block,
3406  VkDeviceSize offset);
3407 
3408  // pMappedData not null means allocation is created with MAPPED flag.
3409  void InitDedicatedAllocation(
3410  uint32_t memoryTypeIndex,
3411  VkDeviceMemory hMemory,
3412  VmaSuballocationType suballocationType,
3413  void* pMappedData,
3414  VkDeviceSize size)
3415  {
3416  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3417  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
3418  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
3419  m_Alignment = 0;
3420  m_Size = size;
3421  m_SuballocationType = (uint8_t)suballocationType;
3422  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3423  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
3424  m_DedicatedAllocation.m_hMemory = hMemory;
3425  m_DedicatedAllocation.m_pMappedData = pMappedData;
3426  }
3427 
3428  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
3429  VkDeviceSize GetAlignment() const { return m_Alignment; }
3430  VkDeviceSize GetSize() const { return m_Size; }
3431  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
3432  void* GetUserData() const { return m_pUserData; }
3433  void SetUserData(VmaAllocator hAllocator, void* pUserData);
3434  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
3435 
3436  VmaDeviceMemoryBlock* GetBlock() const
3437  {
3438  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
3439  return m_BlockAllocation.m_Block;
3440  }
3441  VkDeviceSize GetOffset() const;
3442  VkDeviceMemory GetMemory() const;
3443  uint32_t GetMemoryTypeIndex() const;
3444  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
3445  void* GetMappedData() const;
3446  bool CanBecomeLost() const;
3447  VmaPool GetPool() const;
3448 
3449  uint32_t GetLastUseFrameIndex() const
3450  {
3451  return m_LastUseFrameIndex.load();
3452  }
3453  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
3454  {
3455  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
3456  }
3457  /*
3458  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
3459  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
3460  - Else, returns false.
3461 
3462  If hAllocation is already lost, assert - you should not call it then.
3463  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
3464  */
3465  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3466 
3467  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
3468  {
3469  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
3470  outInfo.blockCount = 1;
3471  outInfo.allocationCount = 1;
3472  outInfo.unusedRangeCount = 0;
3473  outInfo.usedBytes = m_Size;
3474  outInfo.unusedBytes = 0;
3475  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
3476  outInfo.unusedRangeSizeMin = UINT64_MAX;
3477  outInfo.unusedRangeSizeMax = 0;
3478  }
3479 
3480  void BlockAllocMap();
3481  void BlockAllocUnmap();
3482  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
3483  void DedicatedAllocUnmap(VmaAllocator hAllocator);
3484 
3485 private:
3486  VkDeviceSize m_Alignment;
3487  VkDeviceSize m_Size;
3488  void* m_pUserData;
3489  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
3490  uint8_t m_Type; // ALLOCATION_TYPE
3491  uint8_t m_SuballocationType; // VmaSuballocationType
3492  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
3493  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
3494  uint8_t m_MapCount;
3495  uint8_t m_Flags; // enum FLAGS
3496 
3497  // Allocation out of VmaDeviceMemoryBlock.
3498  struct BlockAllocation
3499  {
3500  VmaPool m_hPool; // Null if belongs to general memory.
3501  VmaDeviceMemoryBlock* m_Block;
3502  VkDeviceSize m_Offset;
3503  bool m_CanBecomeLost;
3504  };
3505 
3506  // Allocation for an object that has its own private VkDeviceMemory.
3507  struct DedicatedAllocation
3508  {
3509  uint32_t m_MemoryTypeIndex;
3510  VkDeviceMemory m_hMemory;
3511  void* m_pMappedData; // Not null means memory is mapped.
3512  };
3513 
3514  union
3515  {
3516  // Allocation out of VmaDeviceMemoryBlock.
3517  BlockAllocation m_BlockAllocation;
3518  // Allocation for an object that has its own private VkDeviceMemory.
3519  DedicatedAllocation m_DedicatedAllocation;
3520  };
3521 
3522  void FreeUserDataString(VmaAllocator hAllocator);
3523 };
3524 
3525 /*
3526 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
3527 allocated memory block or free.
3528 */
3529 struct VmaSuballocation
3530 {
3531  VkDeviceSize offset;
3532  VkDeviceSize size;
3533  VmaAllocation hAllocation;
3534  VmaSuballocationType type;
3535 };
3536 
3537 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
3538 
3539 // Cost of one additional allocation lost, as equivalent in bytes.
3540 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
3541 
3542 /*
3543 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3544 
3545 If canMakeOtherLost was false:
3546 - item points to a FREE suballocation.
3547 - itemsToMakeLostCount is 0.
3548 
3549 If canMakeOtherLost was true:
3550 - item points to first of sequence of suballocations, which are either FREE,
3551  or point to VmaAllocations that can become lost.
3552 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3553  the requested allocation to succeed.
3554 */
3555 struct VmaAllocationRequest
3556 {
3557  VkDeviceSize offset;
3558  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3559  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3560  VmaSuballocationList::iterator item;
3561  size_t itemsToMakeLostCount;
3562 
3563  VkDeviceSize CalcCost() const
3564  {
3565  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3566  }
3567 };
3568 
3569 /*
3570 Data structure used for bookkeeping of allocations and unused ranges of memory
3571 in a single VkDeviceMemory block.
3572 */
3573 class VmaBlockMetadata
3574 {
3575 public:
3576  VmaBlockMetadata(VmaAllocator hAllocator);
3577  ~VmaBlockMetadata();
3578  void Init(VkDeviceSize size);
3579 
3580  // Validates all data structures inside this object. If not valid, returns false.
3581  bool Validate() const;
3582  VkDeviceSize GetSize() const { return m_Size; }
3583  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
3584  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3585  VkDeviceSize GetUnusedRangeSizeMax() const;
3586  // Returns true if this block is empty - contains only single free suballocation.
3587  bool IsEmpty() const;
3588 
3589  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3590  void AddPoolStats(VmaPoolStats& inoutStats) const;
3591 
3592 #if VMA_STATS_STRING_ENABLED
3593  void PrintDetailedMap(class VmaJsonWriter& json) const;
3594 #endif
3595 
3596  // Creates trivial request for case when block is empty.
3597  void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3598 
3599  // Tries to find a place for suballocation with given parameters inside this block.
3600  // If succeeded, fills pAllocationRequest and returns true.
3601  // If failed, returns false.
3602  bool CreateAllocationRequest(
3603  uint32_t currentFrameIndex,
3604  uint32_t frameInUseCount,
3605  VkDeviceSize bufferImageGranularity,
3606  VkDeviceSize allocSize,
3607  VkDeviceSize allocAlignment,
3608  VmaSuballocationType allocType,
3609  bool canMakeOtherLost,
3610  VmaAllocationRequest* pAllocationRequest);
3611 
3612  bool MakeRequestedAllocationsLost(
3613  uint32_t currentFrameIndex,
3614  uint32_t frameInUseCount,
3615  VmaAllocationRequest* pAllocationRequest);
3616 
3617  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3618 
3619  // Makes actual allocation based on request. Request must already be checked and valid.
3620  void Alloc(
3621  const VmaAllocationRequest& request,
3622  VmaSuballocationType type,
3623  VkDeviceSize allocSize,
3624  VmaAllocation hAllocation);
3625 
3626  // Frees suballocation assigned to given memory region.
3627  void Free(const VmaAllocation allocation);
3628  void FreeAtOffset(VkDeviceSize offset);
3629 
3630 private:
3631  VkDeviceSize m_Size;
3632  uint32_t m_FreeCount;
3633  VkDeviceSize m_SumFreeSize;
3634  VmaSuballocationList m_Suballocations;
3635  // Suballocations that are free and have size greater than certain threshold.
3636  // Sorted by size, ascending.
3637  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
3638 
3639  bool ValidateFreeSuballocationList() const;
3640 
3641  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
3642  // If yes, fills pOffset and returns true. If no, returns false.
3643  bool CheckAllocation(
3644  uint32_t currentFrameIndex,
3645  uint32_t frameInUseCount,
3646  VkDeviceSize bufferImageGranularity,
3647  VkDeviceSize allocSize,
3648  VkDeviceSize allocAlignment,
3649  VmaSuballocationType allocType,
3650  VmaSuballocationList::const_iterator suballocItem,
3651  bool canMakeOtherLost,
3652  VkDeviceSize* pOffset,
3653  size_t* itemsToMakeLostCount,
3654  VkDeviceSize* pSumFreeSize,
3655  VkDeviceSize* pSumItemSize) const;
3656  // Given free suballocation, it merges it with following one, which must also be free.
3657  void MergeFreeWithNext(VmaSuballocationList::iterator item);
3658  // Releases given suballocation, making it free.
3659  // Merges it with adjacent free suballocations if applicable.
3660  // Returns iterator to new free suballocation at this place.
3661  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
3662  // Given free suballocation, it inserts it into sorted list of
3663  // m_FreeSuballocationsBySize if it's suitable.
3664  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
3665  // Given free suballocation, it removes it from sorted list of
3666  // m_FreeSuballocationsBySize if it's suitable.
3667  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
3668 };
3669 
3670 // Helper class that represents mapped memory. Synchronized internally.
3671 class VmaDeviceMemoryMapping
3672 {
3673 public:
3674  VmaDeviceMemoryMapping();
3675  ~VmaDeviceMemoryMapping();
3676 
3677  void* GetMappedData() const { return m_pMappedData; }
3678 
3679  // ppData can be null.
3680  VkResult Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData);
3681  void Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count);
3682 
3683 private:
3684  VMA_MUTEX m_Mutex;
3685  uint32_t m_MapCount;
3686  void* m_pMappedData;
3687 };
3688 
3689 /*
3690 Represents a single block of device memory (`VkDeviceMemory`) with all the
3691 data about its regions (aka suballocations, `VmaAllocation`), assigned and free.
3692 
3693 Thread-safety: This class must be externally synchronized.
3694 */
3695 class VmaDeviceMemoryBlock
3696 {
3697 public:
3698  uint32_t m_MemoryTypeIndex;
3699  VkDeviceMemory m_hMemory;
3700  VmaDeviceMemoryMapping m_Mapping;
3701  VmaBlockMetadata m_Metadata;
3702 
3703  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
3704 
3705  ~VmaDeviceMemoryBlock()
3706  {
3707  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
3708  }
3709 
3710  // Always call after construction.
3711  void Init(
3712  uint32_t newMemoryTypeIndex,
3713  VkDeviceMemory newMemory,
3714  VkDeviceSize newSize);
3715  // Always call before destruction.
3716  void Destroy(VmaAllocator allocator);
3717 
3718  // Validates all data structures inside this object. If not valid, returns false.
3719  bool Validate() const;
3720 
3721  // ppData can be null.
3722  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
3723  void Unmap(VmaAllocator hAllocator, uint32_t count);
3724 };
3725 
3726 struct VmaPointerLess
3727 {
3728  bool operator()(const void* lhs, const void* rhs) const
3729  {
3730  return lhs < rhs;
3731  }
3732 };
3733 
3734 class VmaDefragmentator;
3735 
3736 /*
3737 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
3738 Vulkan memory type.
3739 
3740 Synchronized internally with a mutex.
3741 */
3742 struct VmaBlockVector
3743 {
3744  VmaBlockVector(
3745  VmaAllocator hAllocator,
3746  uint32_t memoryTypeIndex,
3747  VkDeviceSize preferredBlockSize,
3748  size_t minBlockCount,
3749  size_t maxBlockCount,
3750  VkDeviceSize bufferImageGranularity,
3751  uint32_t frameInUseCount,
3752  bool isCustomPool);
3753  ~VmaBlockVector();
3754 
3755  VkResult CreateMinBlocks();
3756 
3757  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
3758  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
3759  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
3760  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
3761 
3762  void GetPoolStats(VmaPoolStats* pStats);
3763 
3764  bool IsEmpty() const { return m_Blocks.empty(); }
3765 
3766  VkResult Allocate(
3767  VmaPool hCurrentPool,
3768  uint32_t currentFrameIndex,
3769  const VkMemoryRequirements& vkMemReq,
3770  const VmaAllocationCreateInfo& createInfo,
3771  VmaSuballocationType suballocType,
3772  VmaAllocation* pAllocation);
3773 
3774  void Free(
3775  VmaAllocation hAllocation);
3776 
3777  // Adds statistics of this BlockVector to pStats.
3778  void AddStats(VmaStats* pStats);
3779 
3780 #if VMA_STATS_STRING_ENABLED
3781  void PrintDetailedMap(class VmaJsonWriter& json);
3782 #endif
3783 
3784  void MakePoolAllocationsLost(
3785  uint32_t currentFrameIndex,
3786  size_t* pLostAllocationCount);
3787 
3788  VmaDefragmentator* EnsureDefragmentator(
3789  VmaAllocator hAllocator,
3790  uint32_t currentFrameIndex);
3791 
3792  VkResult Defragment(
3793  VmaDefragmentationStats* pDefragmentationStats,
3794  VkDeviceSize& maxBytesToMove,
3795  uint32_t& maxAllocationsToMove);
3796 
3797  void DestroyDefragmentator();
3798 
3799 private:
3800  friend class VmaDefragmentator;
3801 
3802  const VmaAllocator m_hAllocator;
3803  const uint32_t m_MemoryTypeIndex;
3804  const VkDeviceSize m_PreferredBlockSize;
3805  const size_t m_MinBlockCount;
3806  const size_t m_MaxBlockCount;
3807  const VkDeviceSize m_BufferImageGranularity;
3808  const uint32_t m_FrameInUseCount;
3809  const bool m_IsCustomPool;
3810  VMA_MUTEX m_Mutex;
3811  // Incrementally sorted by sumFreeSize, ascending.
3812  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
3813  /* There can be at most one allocation that is completely empty - a
3814  hysteresis to avoid pessimistic case of alternating creation and destruction
3815  of a VkDeviceMemory. */
3816  bool m_HasEmptyBlock;
3817  VmaDefragmentator* m_pDefragmentator;
3818 
3819  size_t CalcMaxBlockSize() const;
3820 
3821  // Finds and removes given block from vector.
3822  void Remove(VmaDeviceMemoryBlock* pBlock);
3823 
3824  // Performs single step in sorting m_Blocks. They may not be fully sorted
3825  // after this call.
3826  void IncrementallySortBlocks();
3827 
3828  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
3829 };
3830 
3831 struct VmaPool_T
3832 {
3833 public:
3834  VmaBlockVector m_BlockVector;
3835 
3836  // Takes ownership.
3837  VmaPool_T(
3838  VmaAllocator hAllocator,
3839  const VmaPoolCreateInfo& createInfo);
3840  ~VmaPool_T();
3841 
3842  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
3843 
3844 #if VMA_STATS_STRING_ENABLED
3845  //void PrintDetailedMap(class VmaStringBuilder& sb);
3846 #endif
3847 };
3848 
3849 class VmaDefragmentator
3850 {
3851  const VmaAllocator m_hAllocator;
3852  VmaBlockVector* const m_pBlockVector;
3853  uint32_t m_CurrentFrameIndex;
3854  VkDeviceSize m_BytesMoved;
3855  uint32_t m_AllocationsMoved;
3856 
3857  struct AllocationInfo
3858  {
3859  VmaAllocation m_hAllocation;
3860  VkBool32* m_pChanged;
3861 
3862  AllocationInfo() :
3863  m_hAllocation(VK_NULL_HANDLE),
3864  m_pChanged(VMA_NULL)
3865  {
3866  }
3867  };
3868 
3869  struct AllocationInfoSizeGreater
3870  {
3871  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
3872  {
3873  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
3874  }
3875  };
3876 
3877  // Used between AddAllocation and Defragment.
3878  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3879 
3880  struct BlockInfo
3881  {
3882  VmaDeviceMemoryBlock* m_pBlock;
3883  bool m_HasNonMovableAllocations;
3884  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3885 
3886  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
3887  m_pBlock(VMA_NULL),
3888  m_HasNonMovableAllocations(true),
3889  m_Allocations(pAllocationCallbacks),
3890  m_pMappedDataForDefragmentation(VMA_NULL)
3891  {
3892  }
3893 
3894  void CalcHasNonMovableAllocations()
3895  {
3896  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
3897  const size_t defragmentAllocCount = m_Allocations.size();
3898  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
3899  }
3900 
3901  void SortAllocationsBySizeDescecnding()
3902  {
3903  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
3904  }
3905 
3906  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
3907  void Unmap(VmaAllocator hAllocator);
3908 
3909  private:
3910  // Not null if mapped for defragmentation only, not originally mapped.
3911  void* m_pMappedDataForDefragmentation;
3912  };
3913 
3914  struct BlockPointerLess
3915  {
3916  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
3917  {
3918  return pLhsBlockInfo->m_pBlock < pRhsBlock;
3919  }
3920  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3921  {
3922  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
3923  }
3924  };
3925 
3926  // 1. Blocks with some non-movable allocations go first.
3927  // 2. Blocks with smaller sumFreeSize go first.
3928  struct BlockInfoCompareMoveDestination
3929  {
3930  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3931  {
3932  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
3933  {
3934  return true;
3935  }
3936  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
3937  {
3938  return false;
3939  }
3940  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
3941  {
3942  return true;
3943  }
3944  return false;
3945  }
3946  };
3947 
3948  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
3949  BlockInfoVector m_Blocks;
3950 
3951  VkResult DefragmentRound(
3952  VkDeviceSize maxBytesToMove,
3953  uint32_t maxAllocationsToMove);
3954 
3955  static bool MoveMakesSense(
3956  size_t dstBlockIndex, VkDeviceSize dstOffset,
3957  size_t srcBlockIndex, VkDeviceSize srcOffset);
3958 
3959 public:
3960  VmaDefragmentator(
3961  VmaAllocator hAllocator,
3962  VmaBlockVector* pBlockVector,
3963  uint32_t currentFrameIndex);
3964 
3965  ~VmaDefragmentator();
3966 
3967  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
3968  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
3969 
3970  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
3971 
3972  VkResult Defragment(
3973  VkDeviceSize maxBytesToMove,
3974  uint32_t maxAllocationsToMove);
3975 };
3976 
3977 // Main allocator object.
3978 struct VmaAllocator_T
3979 {
3980  bool m_UseMutex;
3981  bool m_UseKhrDedicatedAllocation;
3982  VkDevice m_hDevice;
3983  bool m_AllocationCallbacksSpecified;
3984  VkAllocationCallbacks m_AllocationCallbacks;
3985  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
3986 
3987  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
3988  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
3989  VMA_MUTEX m_HeapSizeLimitMutex;
3990 
3991  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
3992  VkPhysicalDeviceMemoryProperties m_MemProps;
3993 
3994  // Default pools.
3995  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
3996 
3997  // Each vector is sorted by memory (handle value).
3998  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
3999  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
4000  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
4001 
4002  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
4003  ~VmaAllocator_T();
4004 
4005  const VkAllocationCallbacks* GetAllocationCallbacks() const
4006  {
4007  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
4008  }
4009  const VmaVulkanFunctions& GetVulkanFunctions() const
4010  {
4011  return m_VulkanFunctions;
4012  }
4013 
4014  VkDeviceSize GetBufferImageGranularity() const
4015  {
4016  return VMA_MAX(
4017  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
4018  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
4019  }
4020 
4021  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
4022  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
4023 
4024  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
4025  {
4026  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
4027  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
4028  }
4029 
4030  void GetBufferMemoryRequirements(
4031  VkBuffer hBuffer,
4032  VkMemoryRequirements& memReq,
4033  bool& requiresDedicatedAllocation,
4034  bool& prefersDedicatedAllocation) const;
4035  void GetImageMemoryRequirements(
4036  VkImage hImage,
4037  VkMemoryRequirements& memReq,
4038  bool& requiresDedicatedAllocation,
4039  bool& prefersDedicatedAllocation) const;
4040 
4041  // Main allocation function.
4042  VkResult AllocateMemory(
4043  const VkMemoryRequirements& vkMemReq,
4044  bool requiresDedicatedAllocation,
4045  bool prefersDedicatedAllocation,
4046  VkBuffer dedicatedBuffer,
4047  VkImage dedicatedImage,
4048  const VmaAllocationCreateInfo& createInfo,
4049  VmaSuballocationType suballocType,
4050  VmaAllocation* pAllocation);
4051 
4052  // Main deallocation function.
4053  void FreeMemory(const VmaAllocation allocation);
4054 
4055  void CalculateStats(VmaStats* pStats);
4056 
4057 #if VMA_STATS_STRING_ENABLED
4058  void PrintDetailedMap(class VmaJsonWriter& json);
4059 #endif
4060 
4061  VkResult Defragment(
4062  VmaAllocation* pAllocations,
4063  size_t allocationCount,
4064  VkBool32* pAllocationsChanged,
4065  const VmaDefragmentationInfo* pDefragmentationInfo,
4066  VmaDefragmentationStats* pDefragmentationStats);
4067 
4068  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
4069 
4070  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
4071  void DestroyPool(VmaPool pool);
4072  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
4073 
4074  void SetCurrentFrameIndex(uint32_t frameIndex);
4075 
4076  void MakePoolAllocationsLost(
4077  VmaPool hPool,
4078  size_t* pLostAllocationCount);
4079 
4080  void CreateLostAllocation(VmaAllocation* pAllocation);
4081 
4082  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
4083  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
4084 
4085  VkResult Map(VmaAllocation hAllocation, void** ppData);
4086  void Unmap(VmaAllocation hAllocation);
4087 
4088 private:
4089  VkDeviceSize m_PreferredLargeHeapBlockSize;
4090 
4091  VkPhysicalDevice m_PhysicalDevice;
4092  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
4093 
4094  VMA_MUTEX m_PoolsMutex;
4095  // Protected by m_PoolsMutex. Sorted by pointer value.
4096  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
4097 
4098  VmaVulkanFunctions m_VulkanFunctions;
4099 
4100  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
4101 
4102  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
4103 
4104  VkResult AllocateMemoryOfType(
4105  const VkMemoryRequirements& vkMemReq,
4106  bool dedicatedAllocation,
4107  VkBuffer dedicatedBuffer,
4108  VkImage dedicatedImage,
4109  const VmaAllocationCreateInfo& createInfo,
4110  uint32_t memTypeIndex,
4111  VmaSuballocationType suballocType,
4112  VmaAllocation* pAllocation);
4113 
4114  // Allocates and registers new VkDeviceMemory specifically for single allocation.
4115  VkResult AllocateDedicatedMemory(
4116  VkDeviceSize size,
4117  VmaSuballocationType suballocType,
4118  uint32_t memTypeIndex,
4119  bool map,
4120  bool isUserDataString,
4121  void* pUserData,
4122  VkBuffer dedicatedBuffer,
4123  VkImage dedicatedImage,
4124  VmaAllocation* pAllocation);
4125 
4126  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
4127  void FreeDedicatedMemory(VmaAllocation allocation);
4128 };
4129 
4131 // Memory allocation #2 after VmaAllocator_T definition
4132 
4133 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
4134 {
4135  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
4136 }
4137 
4138 static void VmaFree(VmaAllocator hAllocator, void* ptr)
4139 {
4140  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
4141 }
4142 
4143 template<typename T>
4144 static T* VmaAllocate(VmaAllocator hAllocator)
4145 {
4146  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
4147 }
4148 
4149 template<typename T>
4150 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
4151 {
4152  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
4153 }
4154 
4155 template<typename T>
4156 static void vma_delete(VmaAllocator hAllocator, T* ptr)
4157 {
4158  if(ptr != VMA_NULL)
4159  {
4160  ptr->~T();
4161  VmaFree(hAllocator, ptr);
4162  }
4163 }
4164 
4165 template<typename T>
4166 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
4167 {
4168  if(ptr != VMA_NULL)
4169  {
4170  for(size_t i = count; i--; )
4171  ptr[i].~T();
4172  VmaFree(hAllocator, ptr);
4173  }
4174 }
4175 
4177 // VmaStringBuilder
4178 
4179 #if VMA_STATS_STRING_ENABLED
4180 
4181 class VmaStringBuilder
4182 {
4183 public:
4184  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
4185  size_t GetLength() const { return m_Data.size(); }
4186  const char* GetData() const { return m_Data.data(); }
4187 
4188  void Add(char ch) { m_Data.push_back(ch); }
4189  void Add(const char* pStr);
4190  void AddNewLine() { Add('\n'); }
4191  void AddNumber(uint32_t num);
4192  void AddNumber(uint64_t num);
4193  void AddPointer(const void* ptr);
4194 
4195 private:
4196  VmaVector< char, VmaStlAllocator<char> > m_Data;
4197 };
4198 
4199 void VmaStringBuilder::Add(const char* pStr)
4200 {
4201  const size_t strLen = strlen(pStr);
4202  if(strLen > 0)
4203  {
4204  const size_t oldCount = m_Data.size();
4205  m_Data.resize(oldCount + strLen);
4206  memcpy(m_Data.data() + oldCount, pStr, strLen);
4207  }
4208 }
4209 
4210 void VmaStringBuilder::AddNumber(uint32_t num)
4211 {
4212  char buf[11];
4213  VmaUint32ToStr(buf, sizeof(buf), num);
4214  Add(buf);
4215 }
4216 
4217 void VmaStringBuilder::AddNumber(uint64_t num)
4218 {
4219  char buf[21];
4220  VmaUint64ToStr(buf, sizeof(buf), num);
4221  Add(buf);
4222 }
4223 
4224 void VmaStringBuilder::AddPointer(const void* ptr)
4225 {
4226  char buf[21];
4227  VmaPtrToStr(buf, sizeof(buf), ptr);
4228  Add(buf);
4229 }
4230 
4231 #endif // #if VMA_STATS_STRING_ENABLED
4232 
4234 // VmaJsonWriter
4235 
4236 #if VMA_STATS_STRING_ENABLED
4237 
4238 class VmaJsonWriter
4239 {
4240 public:
4241  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
4242  ~VmaJsonWriter();
4243 
4244  void BeginObject(bool singleLine = false);
4245  void EndObject();
4246 
4247  void BeginArray(bool singleLine = false);
4248  void EndArray();
4249 
4250  void WriteString(const char* pStr);
4251  void BeginString(const char* pStr = VMA_NULL);
4252  void ContinueString(const char* pStr);
4253  void ContinueString(uint32_t n);
4254  void ContinueString(uint64_t n);
4255  void ContinueString_Pointer(const void* ptr);
4256  void EndString(const char* pStr = VMA_NULL);
4257 
4258  void WriteNumber(uint32_t n);
4259  void WriteNumber(uint64_t n);
4260  void WriteBool(bool b);
4261  void WriteNull();
4262 
4263 private:
4264  static const char* const INDENT;
4265 
4266  enum COLLECTION_TYPE
4267  {
4268  COLLECTION_TYPE_OBJECT,
4269  COLLECTION_TYPE_ARRAY,
4270  };
4271  struct StackItem
4272  {
4273  COLLECTION_TYPE type;
4274  uint32_t valueCount;
4275  bool singleLineMode;
4276  };
4277 
4278  VmaStringBuilder& m_SB;
4279  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
4280  bool m_InsideString;
4281 
4282  void BeginValue(bool isString);
4283  void WriteIndent(bool oneLess = false);
4284 };
4285 
4286 const char* const VmaJsonWriter::INDENT = " ";
4287 
4288 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
4289  m_SB(sb),
4290  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
4291  m_InsideString(false)
4292 {
4293 }
4294 
4295 VmaJsonWriter::~VmaJsonWriter()
4296 {
4297  VMA_ASSERT(!m_InsideString);
4298  VMA_ASSERT(m_Stack.empty());
4299 }
4300 
4301 void VmaJsonWriter::BeginObject(bool singleLine)
4302 {
4303  VMA_ASSERT(!m_InsideString);
4304 
4305  BeginValue(false);
4306  m_SB.Add('{');
4307 
4308  StackItem item;
4309  item.type = COLLECTION_TYPE_OBJECT;
4310  item.valueCount = 0;
4311  item.singleLineMode = singleLine;
4312  m_Stack.push_back(item);
4313 }
4314 
4315 void VmaJsonWriter::EndObject()
4316 {
4317  VMA_ASSERT(!m_InsideString);
4318 
4319  WriteIndent(true);
4320  m_SB.Add('}');
4321 
4322  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
4323  m_Stack.pop_back();
4324 }
4325 
4326 void VmaJsonWriter::BeginArray(bool singleLine)
4327 {
4328  VMA_ASSERT(!m_InsideString);
4329 
4330  BeginValue(false);
4331  m_SB.Add('[');
4332 
4333  StackItem item;
4334  item.type = COLLECTION_TYPE_ARRAY;
4335  item.valueCount = 0;
4336  item.singleLineMode = singleLine;
4337  m_Stack.push_back(item);
4338 }
4339 
4340 void VmaJsonWriter::EndArray()
4341 {
4342  VMA_ASSERT(!m_InsideString);
4343 
4344  WriteIndent(true);
4345  m_SB.Add(']');
4346 
4347  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
4348  m_Stack.pop_back();
4349 }
4350 
4351 void VmaJsonWriter::WriteString(const char* pStr)
4352 {
4353  BeginString(pStr);
4354  EndString();
4355 }
4356 
4357 void VmaJsonWriter::BeginString(const char* pStr)
4358 {
4359  VMA_ASSERT(!m_InsideString);
4360 
4361  BeginValue(true);
4362  m_SB.Add('"');
4363  m_InsideString = true;
4364  if(pStr != VMA_NULL && pStr[0] != '\0')
4365  {
4366  ContinueString(pStr);
4367  }
4368 }
4369 
4370 void VmaJsonWriter::ContinueString(const char* pStr)
4371 {
4372  VMA_ASSERT(m_InsideString);
4373 
4374  const size_t strLen = strlen(pStr);
4375  for(size_t i = 0; i < strLen; ++i)
4376  {
4377  char ch = pStr[i];
4378  if(ch == '\'')
4379  {
4380  m_SB.Add("\\\\");
4381  }
4382  else if(ch == '"')
4383  {
4384  m_SB.Add("\\\"");
4385  }
4386  else if(ch >= 32)
4387  {
4388  m_SB.Add(ch);
4389  }
4390  else switch(ch)
4391  {
4392  case '\b':
4393  m_SB.Add("\\b");
4394  break;
4395  case '\f':
4396  m_SB.Add("\\f");
4397  break;
4398  case '\n':
4399  m_SB.Add("\\n");
4400  break;
4401  case '\r':
4402  m_SB.Add("\\r");
4403  break;
4404  case '\t':
4405  m_SB.Add("\\t");
4406  break;
4407  default:
4408  VMA_ASSERT(0 && "Character not currently supported.");
4409  break;
4410  }
4411  }
4412 }
4413 
4414 void VmaJsonWriter::ContinueString(uint32_t n)
4415 {
4416  VMA_ASSERT(m_InsideString);
4417  m_SB.AddNumber(n);
4418 }
4419 
4420 void VmaJsonWriter::ContinueString(uint64_t n)
4421 {
4422  VMA_ASSERT(m_InsideString);
4423  m_SB.AddNumber(n);
4424 }
4425 
4426 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
4427 {
4428  VMA_ASSERT(m_InsideString);
4429  m_SB.AddPointer(ptr);
4430 }
4431 
4432 void VmaJsonWriter::EndString(const char* pStr)
4433 {
4434  VMA_ASSERT(m_InsideString);
4435  if(pStr != VMA_NULL && pStr[0] != '\0')
4436  {
4437  ContinueString(pStr);
4438  }
4439  m_SB.Add('"');
4440  m_InsideString = false;
4441 }
4442 
4443 void VmaJsonWriter::WriteNumber(uint32_t n)
4444 {
4445  VMA_ASSERT(!m_InsideString);
4446  BeginValue(false);
4447  m_SB.AddNumber(n);
4448 }
4449 
4450 void VmaJsonWriter::WriteNumber(uint64_t n)
4451 {
4452  VMA_ASSERT(!m_InsideString);
4453  BeginValue(false);
4454  m_SB.AddNumber(n);
4455 }
4456 
4457 void VmaJsonWriter::WriteBool(bool b)
4458 {
4459  VMA_ASSERT(!m_InsideString);
4460  BeginValue(false);
4461  m_SB.Add(b ? "true" : "false");
4462 }
4463 
4464 void VmaJsonWriter::WriteNull()
4465 {
4466  VMA_ASSERT(!m_InsideString);
4467  BeginValue(false);
4468  m_SB.Add("null");
4469 }
4470 
4471 void VmaJsonWriter::BeginValue(bool isString)
4472 {
4473  if(!m_Stack.empty())
4474  {
4475  StackItem& currItem = m_Stack.back();
4476  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4477  currItem.valueCount % 2 == 0)
4478  {
4479  VMA_ASSERT(isString);
4480  }
4481 
4482  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4483  currItem.valueCount % 2 != 0)
4484  {
4485  m_SB.Add(": ");
4486  }
4487  else if(currItem.valueCount > 0)
4488  {
4489  m_SB.Add(", ");
4490  WriteIndent();
4491  }
4492  else
4493  {
4494  WriteIndent();
4495  }
4496  ++currItem.valueCount;
4497  }
4498 }
4499 
4500 void VmaJsonWriter::WriteIndent(bool oneLess)
4501 {
4502  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
4503  {
4504  m_SB.AddNewLine();
4505 
4506  size_t count = m_Stack.size();
4507  if(count > 0 && oneLess)
4508  {
4509  --count;
4510  }
4511  for(size_t i = 0; i < count; ++i)
4512  {
4513  m_SB.Add(INDENT);
4514  }
4515  }
4516 }
4517 
4518 #endif // #if VMA_STATS_STRING_ENABLED
4519 
4521 
4522 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
4523 {
4524  if(IsUserDataString())
4525  {
4526  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
4527 
4528  FreeUserDataString(hAllocator);
4529 
4530  if(pUserData != VMA_NULL)
4531  {
4532  const char* const newStrSrc = (char*)pUserData;
4533  const size_t newStrLen = strlen(newStrSrc);
4534  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
4535  memcpy(newStrDst, newStrSrc, newStrLen + 1);
4536  m_pUserData = newStrDst;
4537  }
4538  }
4539  else
4540  {
4541  m_pUserData = pUserData;
4542  }
4543 }
4544 
4545 void VmaAllocation_T::ChangeBlockAllocation(
4546  VmaAllocator hAllocator,
4547  VmaDeviceMemoryBlock* block,
4548  VkDeviceSize offset)
4549 {
4550  VMA_ASSERT(block != VMA_NULL);
4551  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4552 
4553  // Move mapping reference counter from old block to new block.
4554  if(block != m_BlockAllocation.m_Block)
4555  {
4556  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
4557  if(IsPersistentMap())
4558  ++mapRefCount;
4559  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
4560  block->Map(hAllocator, mapRefCount, VMA_NULL);
4561  }
4562 
4563  m_BlockAllocation.m_Block = block;
4564  m_BlockAllocation.m_Offset = offset;
4565 }
4566 
4567 VkDeviceSize VmaAllocation_T::GetOffset() const
4568 {
4569  switch(m_Type)
4570  {
4571  case ALLOCATION_TYPE_BLOCK:
4572  return m_BlockAllocation.m_Offset;
4573  case ALLOCATION_TYPE_DEDICATED:
4574  return 0;
4575  default:
4576  VMA_ASSERT(0);
4577  return 0;
4578  }
4579 }
4580 
4581 VkDeviceMemory VmaAllocation_T::GetMemory() const
4582 {
4583  switch(m_Type)
4584  {
4585  case ALLOCATION_TYPE_BLOCK:
4586  return m_BlockAllocation.m_Block->m_hMemory;
4587  case ALLOCATION_TYPE_DEDICATED:
4588  return m_DedicatedAllocation.m_hMemory;
4589  default:
4590  VMA_ASSERT(0);
4591  return VK_NULL_HANDLE;
4592  }
4593 }
4594 
4595 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
4596 {
4597  switch(m_Type)
4598  {
4599  case ALLOCATION_TYPE_BLOCK:
4600  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
4601  case ALLOCATION_TYPE_DEDICATED:
4602  return m_DedicatedAllocation.m_MemoryTypeIndex;
4603  default:
4604  VMA_ASSERT(0);
4605  return UINT32_MAX;
4606  }
4607 }
4608 
4609 void* VmaAllocation_T::GetMappedData() const
4610 {
4611  switch(m_Type)
4612  {
4613  case ALLOCATION_TYPE_BLOCK:
4614  if(m_MapCount != 0)
4615  {
4616  void* pBlockData = m_BlockAllocation.m_Block->m_Mapping.GetMappedData();
4617  VMA_ASSERT(pBlockData != VMA_NULL);
4618  return (char*)pBlockData + m_BlockAllocation.m_Offset;
4619  }
4620  else
4621  {
4622  return VMA_NULL;
4623  }
4624  break;
4625  case ALLOCATION_TYPE_DEDICATED:
4626  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
4627  return m_DedicatedAllocation.m_pMappedData;
4628  default:
4629  VMA_ASSERT(0);
4630  return VMA_NULL;
4631  }
4632 }
4633 
4634 bool VmaAllocation_T::CanBecomeLost() const
4635 {
4636  switch(m_Type)
4637  {
4638  case ALLOCATION_TYPE_BLOCK:
4639  return m_BlockAllocation.m_CanBecomeLost;
4640  case ALLOCATION_TYPE_DEDICATED:
4641  return false;
4642  default:
4643  VMA_ASSERT(0);
4644  return false;
4645  }
4646 }
4647 
4648 VmaPool VmaAllocation_T::GetPool() const
4649 {
4650  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4651  return m_BlockAllocation.m_hPool;
4652 }
4653 
4654 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4655 {
4656  VMA_ASSERT(CanBecomeLost());
4657 
4658  /*
4659  Warning: This is a carefully designed algorithm.
4660  Do not modify unless you really know what you're doing :)
4661  */
4662  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
4663  for(;;)
4664  {
4665  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
4666  {
4667  VMA_ASSERT(0);
4668  return false;
4669  }
4670  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
4671  {
4672  return false;
4673  }
4674  else // Last use time earlier than current time.
4675  {
4676  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
4677  {
4678  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
4679  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
4680  return true;
4681  }
4682  }
4683  }
4684 }
4685 
4686 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
4687 {
4688  VMA_ASSERT(IsUserDataString());
4689  if(m_pUserData != VMA_NULL)
4690  {
4691  char* const oldStr = (char*)m_pUserData;
4692  const size_t oldStrLen = strlen(oldStr);
4693  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
4694  m_pUserData = VMA_NULL;
4695  }
4696 }
4697 
4698 void VmaAllocation_T::BlockAllocMap()
4699 {
4700  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4701 
4702  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4703  {
4704  ++m_MapCount;
4705  }
4706  else
4707  {
4708  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
4709  }
4710 }
4711 
4712 void VmaAllocation_T::BlockAllocUnmap()
4713 {
4714  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4715 
4716  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4717  {
4718  --m_MapCount;
4719  }
4720  else
4721  {
4722  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
4723  }
4724 }
4725 
4726 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
4727 {
4728  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4729 
4730  if(m_MapCount != 0)
4731  {
4732  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4733  {
4734  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
4735  *ppData = m_DedicatedAllocation.m_pMappedData;
4736  ++m_MapCount;
4737  return VK_SUCCESS;
4738  }
4739  else
4740  {
4741  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
4742  return VK_ERROR_MEMORY_MAP_FAILED;
4743  }
4744  }
4745  else
4746  {
4747  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
4748  hAllocator->m_hDevice,
4749  m_DedicatedAllocation.m_hMemory,
4750  0, // offset
4751  VK_WHOLE_SIZE,
4752  0, // flags
4753  ppData);
4754  if(result == VK_SUCCESS)
4755  {
4756  m_DedicatedAllocation.m_pMappedData = *ppData;
4757  m_MapCount = 1;
4758  }
4759  return result;
4760  }
4761 }
4762 
4763 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
4764 {
4765  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4766 
4767  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4768  {
4769  --m_MapCount;
4770  if(m_MapCount == 0)
4771  {
4772  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
4773  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
4774  hAllocator->m_hDevice,
4775  m_DedicatedAllocation.m_hMemory);
4776  }
4777  }
4778  else
4779  {
4780  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
4781  }
4782 }
4783 
4784 #if VMA_STATS_STRING_ENABLED
4785 
4786 // Correspond to values of enum VmaSuballocationType.
4787 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
4788  "FREE",
4789  "UNKNOWN",
4790  "BUFFER",
4791  "IMAGE_UNKNOWN",
4792  "IMAGE_LINEAR",
4793  "IMAGE_OPTIMAL",
4794 };
4795 
4796 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
4797 {
4798  json.BeginObject();
4799 
4800  json.WriteString("Blocks");
4801  json.WriteNumber(stat.blockCount);
4802 
4803  json.WriteString("Allocations");
4804  json.WriteNumber(stat.allocationCount);
4805 
4806  json.WriteString("UnusedRanges");
4807  json.WriteNumber(stat.unusedRangeCount);
4808 
4809  json.WriteString("UsedBytes");
4810  json.WriteNumber(stat.usedBytes);
4811 
4812  json.WriteString("UnusedBytes");
4813  json.WriteNumber(stat.unusedBytes);
4814 
4815  if(stat.allocationCount > 1)
4816  {
4817  json.WriteString("AllocationSize");
4818  json.BeginObject(true);
4819  json.WriteString("Min");
4820  json.WriteNumber(stat.allocationSizeMin);
4821  json.WriteString("Avg");
4822  json.WriteNumber(stat.allocationSizeAvg);
4823  json.WriteString("Max");
4824  json.WriteNumber(stat.allocationSizeMax);
4825  json.EndObject();
4826  }
4827 
4828  if(stat.unusedRangeCount > 1)
4829  {
4830  json.WriteString("UnusedRangeSize");
4831  json.BeginObject(true);
4832  json.WriteString("Min");
4833  json.WriteNumber(stat.unusedRangeSizeMin);
4834  json.WriteString("Avg");
4835  json.WriteNumber(stat.unusedRangeSizeAvg);
4836  json.WriteString("Max");
4837  json.WriteNumber(stat.unusedRangeSizeMax);
4838  json.EndObject();
4839  }
4840 
4841  json.EndObject();
4842 }
4843 
4844 #endif // #if VMA_STATS_STRING_ENABLED
4845 
4846 struct VmaSuballocationItemSizeLess
4847 {
4848  bool operator()(
4849  const VmaSuballocationList::iterator lhs,
4850  const VmaSuballocationList::iterator rhs) const
4851  {
4852  return lhs->size < rhs->size;
4853  }
4854  bool operator()(
4855  const VmaSuballocationList::iterator lhs,
4856  VkDeviceSize rhsSize) const
4857  {
4858  return lhs->size < rhsSize;
4859  }
4860 };
4861 
4863 // class VmaBlockMetadata
4864 
4865 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
4866  m_Size(0),
4867  m_FreeCount(0),
4868  m_SumFreeSize(0),
4869  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
4870  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
4871 {
4872 }
4873 
4874 VmaBlockMetadata::~VmaBlockMetadata()
4875 {
4876 }
4877 
4878 void VmaBlockMetadata::Init(VkDeviceSize size)
4879 {
4880  m_Size = size;
4881  m_FreeCount = 1;
4882  m_SumFreeSize = size;
4883 
4884  VmaSuballocation suballoc = {};
4885  suballoc.offset = 0;
4886  suballoc.size = size;
4887  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4888  suballoc.hAllocation = VK_NULL_HANDLE;
4889 
4890  m_Suballocations.push_back(suballoc);
4891  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
4892  --suballocItem;
4893  m_FreeSuballocationsBySize.push_back(suballocItem);
4894 }
4895 
4896 bool VmaBlockMetadata::Validate() const
4897 {
4898  if(m_Suballocations.empty())
4899  {
4900  return false;
4901  }
4902 
4903  // Expected offset of new suballocation as calculates from previous ones.
4904  VkDeviceSize calculatedOffset = 0;
4905  // Expected number of free suballocations as calculated from traversing their list.
4906  uint32_t calculatedFreeCount = 0;
4907  // Expected sum size of free suballocations as calculated from traversing their list.
4908  VkDeviceSize calculatedSumFreeSize = 0;
4909  // Expected number of free suballocations that should be registered in
4910  // m_FreeSuballocationsBySize calculated from traversing their list.
4911  size_t freeSuballocationsToRegister = 0;
4912  // True if previous visisted suballocation was free.
4913  bool prevFree = false;
4914 
4915  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4916  suballocItem != m_Suballocations.cend();
4917  ++suballocItem)
4918  {
4919  const VmaSuballocation& subAlloc = *suballocItem;
4920 
4921  // Actual offset of this suballocation doesn't match expected one.
4922  if(subAlloc.offset != calculatedOffset)
4923  {
4924  return false;
4925  }
4926 
4927  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
4928  // Two adjacent free suballocations are invalid. They should be merged.
4929  if(prevFree && currFree)
4930  {
4931  return false;
4932  }
4933 
4934  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
4935  {
4936  return false;
4937  }
4938 
4939  if(currFree)
4940  {
4941  calculatedSumFreeSize += subAlloc.size;
4942  ++calculatedFreeCount;
4943  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4944  {
4945  ++freeSuballocationsToRegister;
4946  }
4947  }
4948  else
4949  {
4950  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
4951  {
4952  return false;
4953  }
4954  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
4955  {
4956  return false;
4957  }
4958  }
4959 
4960  calculatedOffset += subAlloc.size;
4961  prevFree = currFree;
4962  }
4963 
4964  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
4965  // match expected one.
4966  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
4967  {
4968  return false;
4969  }
4970 
4971  VkDeviceSize lastSize = 0;
4972  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
4973  {
4974  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
4975 
4976  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
4977  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
4978  {
4979  return false;
4980  }
4981  // They must be sorted by size ascending.
4982  if(suballocItem->size < lastSize)
4983  {
4984  return false;
4985  }
4986 
4987  lastSize = suballocItem->size;
4988  }
4989 
4990  // Check if totals match calculacted values.
4991  if(!ValidateFreeSuballocationList() ||
4992  (calculatedOffset != m_Size) ||
4993  (calculatedSumFreeSize != m_SumFreeSize) ||
4994  (calculatedFreeCount != m_FreeCount))
4995  {
4996  return false;
4997  }
4998 
4999  return true;
5000 }
5001 
5002 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
5003 {
5004  if(!m_FreeSuballocationsBySize.empty())
5005  {
5006  return m_FreeSuballocationsBySize.back()->size;
5007  }
5008  else
5009  {
5010  return 0;
5011  }
5012 }
5013 
5014 bool VmaBlockMetadata::IsEmpty() const
5015 {
5016  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
5017 }
5018 
5019 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
5020 {
5021  outInfo.blockCount = 1;
5022 
5023  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5024  outInfo.allocationCount = rangeCount - m_FreeCount;
5025  outInfo.unusedRangeCount = m_FreeCount;
5026 
5027  outInfo.unusedBytes = m_SumFreeSize;
5028  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
5029 
5030  outInfo.allocationSizeMin = UINT64_MAX;
5031  outInfo.allocationSizeMax = 0;
5032  outInfo.unusedRangeSizeMin = UINT64_MAX;
5033  outInfo.unusedRangeSizeMax = 0;
5034 
5035  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5036  suballocItem != m_Suballocations.cend();
5037  ++suballocItem)
5038  {
5039  const VmaSuballocation& suballoc = *suballocItem;
5040  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
5041  {
5042  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
5043  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
5044  }
5045  else
5046  {
5047  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
5048  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
5049  }
5050  }
5051 }
5052 
5053 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
5054 {
5055  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5056 
5057  inoutStats.size += m_Size;
5058  inoutStats.unusedSize += m_SumFreeSize;
5059  inoutStats.allocationCount += rangeCount - m_FreeCount;
5060  inoutStats.unusedRangeCount += m_FreeCount;
5061  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
5062 }
5063 
5064 #if VMA_STATS_STRING_ENABLED
5065 
5066 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
5067 {
5068  json.BeginObject();
5069 
5070  json.WriteString("TotalBytes");
5071  json.WriteNumber(m_Size);
5072 
5073  json.WriteString("UnusedBytes");
5074  json.WriteNumber(m_SumFreeSize);
5075 
5076  json.WriteString("Allocations");
5077  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
5078 
5079  json.WriteString("UnusedRanges");
5080  json.WriteNumber(m_FreeCount);
5081 
5082  json.WriteString("Suballocations");
5083  json.BeginArray();
5084  size_t i = 0;
5085  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5086  suballocItem != m_Suballocations.cend();
5087  ++suballocItem, ++i)
5088  {
5089  json.BeginObject(true);
5090 
5091  json.WriteString("Type");
5092  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
5093 
5094  json.WriteString("Size");
5095  json.WriteNumber(suballocItem->size);
5096 
5097  json.WriteString("Offset");
5098  json.WriteNumber(suballocItem->offset);
5099 
5100  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5101  {
5102  const void* pUserData = suballocItem->hAllocation->GetUserData();
5103  if(pUserData != VMA_NULL)
5104  {
5105  json.WriteString("UserData");
5106  if(suballocItem->hAllocation->IsUserDataString())
5107  {
5108  json.WriteString((const char*)pUserData);
5109  }
5110  else
5111  {
5112  json.BeginString();
5113  json.ContinueString_Pointer(pUserData);
5114  json.EndString();
5115  }
5116  }
5117  }
5118 
5119  json.EndObject();
5120  }
5121  json.EndArray();
5122 
5123  json.EndObject();
5124 }
5125 
5126 #endif // #if VMA_STATS_STRING_ENABLED
5127 
5128 /*
5129 How many suitable free suballocations to analyze before choosing best one.
5130 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
5131  be chosen.
5132 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
5133  suballocations will be analized and best one will be chosen.
5134 - Any other value is also acceptable.
5135 */
5136 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
5137 
5138 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
5139 {
5140  VMA_ASSERT(IsEmpty());
5141  pAllocationRequest->offset = 0;
5142  pAllocationRequest->sumFreeSize = m_SumFreeSize;
5143  pAllocationRequest->sumItemSize = 0;
5144  pAllocationRequest->item = m_Suballocations.begin();
5145  pAllocationRequest->itemsToMakeLostCount = 0;
5146 }
5147 
5148 bool VmaBlockMetadata::CreateAllocationRequest(
5149  uint32_t currentFrameIndex,
5150  uint32_t frameInUseCount,
5151  VkDeviceSize bufferImageGranularity,
5152  VkDeviceSize allocSize,
5153  VkDeviceSize allocAlignment,
5154  VmaSuballocationType allocType,
5155  bool canMakeOtherLost,
5156  VmaAllocationRequest* pAllocationRequest)
5157 {
5158  VMA_ASSERT(allocSize > 0);
5159  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5160  VMA_ASSERT(pAllocationRequest != VMA_NULL);
5161  VMA_HEAVY_ASSERT(Validate());
5162 
5163  // There is not enough total free space in this block to fullfill the request: Early return.
5164  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
5165  {
5166  return false;
5167  }
5168 
5169  // New algorithm, efficiently searching freeSuballocationsBySize.
5170  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
5171  if(freeSuballocCount > 0)
5172  {
5173  if(VMA_BEST_FIT)
5174  {
5175  // Find first free suballocation with size not less than allocSize.
5176  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5177  m_FreeSuballocationsBySize.data(),
5178  m_FreeSuballocationsBySize.data() + freeSuballocCount,
5179  allocSize,
5180  VmaSuballocationItemSizeLess());
5181  size_t index = it - m_FreeSuballocationsBySize.data();
5182  for(; index < freeSuballocCount; ++index)
5183  {
5184  if(CheckAllocation(
5185  currentFrameIndex,
5186  frameInUseCount,
5187  bufferImageGranularity,
5188  allocSize,
5189  allocAlignment,
5190  allocType,
5191  m_FreeSuballocationsBySize[index],
5192  false, // canMakeOtherLost
5193  &pAllocationRequest->offset,
5194  &pAllocationRequest->itemsToMakeLostCount,
5195  &pAllocationRequest->sumFreeSize,
5196  &pAllocationRequest->sumItemSize))
5197  {
5198  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5199  return true;
5200  }
5201  }
5202  }
5203  else
5204  {
5205  // Search staring from biggest suballocations.
5206  for(size_t index = freeSuballocCount; index--; )
5207  {
5208  if(CheckAllocation(
5209  currentFrameIndex,
5210  frameInUseCount,
5211  bufferImageGranularity,
5212  allocSize,
5213  allocAlignment,
5214  allocType,
5215  m_FreeSuballocationsBySize[index],
5216  false, // canMakeOtherLost
5217  &pAllocationRequest->offset,
5218  &pAllocationRequest->itemsToMakeLostCount,
5219  &pAllocationRequest->sumFreeSize,
5220  &pAllocationRequest->sumItemSize))
5221  {
5222  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5223  return true;
5224  }
5225  }
5226  }
5227  }
5228 
5229  if(canMakeOtherLost)
5230  {
5231  // Brute-force algorithm. TODO: Come up with something better.
5232 
5233  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
5234  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
5235 
5236  VmaAllocationRequest tmpAllocRequest = {};
5237  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
5238  suballocIt != m_Suballocations.end();
5239  ++suballocIt)
5240  {
5241  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
5242  suballocIt->hAllocation->CanBecomeLost())
5243  {
5244  if(CheckAllocation(
5245  currentFrameIndex,
5246  frameInUseCount,
5247  bufferImageGranularity,
5248  allocSize,
5249  allocAlignment,
5250  allocType,
5251  suballocIt,
5252  canMakeOtherLost,
5253  &tmpAllocRequest.offset,
5254  &tmpAllocRequest.itemsToMakeLostCount,
5255  &tmpAllocRequest.sumFreeSize,
5256  &tmpAllocRequest.sumItemSize))
5257  {
5258  tmpAllocRequest.item = suballocIt;
5259 
5260  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
5261  {
5262  *pAllocationRequest = tmpAllocRequest;
5263  }
5264  }
5265  }
5266  }
5267 
5268  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
5269  {
5270  return true;
5271  }
5272  }
5273 
5274  return false;
5275 }
5276 
5277 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
5278  uint32_t currentFrameIndex,
5279  uint32_t frameInUseCount,
5280  VmaAllocationRequest* pAllocationRequest)
5281 {
5282  while(pAllocationRequest->itemsToMakeLostCount > 0)
5283  {
5284  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
5285  {
5286  ++pAllocationRequest->item;
5287  }
5288  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5289  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
5290  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
5291  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5292  {
5293  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
5294  --pAllocationRequest->itemsToMakeLostCount;
5295  }
5296  else
5297  {
5298  return false;
5299  }
5300  }
5301 
5302  VMA_HEAVY_ASSERT(Validate());
5303  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5304  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
5305 
5306  return true;
5307 }
5308 
5309 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5310 {
5311  uint32_t lostAllocationCount = 0;
5312  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
5313  it != m_Suballocations.end();
5314  ++it)
5315  {
5316  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
5317  it->hAllocation->CanBecomeLost() &&
5318  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5319  {
5320  it = FreeSuballocation(it);
5321  ++lostAllocationCount;
5322  }
5323  }
5324  return lostAllocationCount;
5325 }
5326 
5327 void VmaBlockMetadata::Alloc(
5328  const VmaAllocationRequest& request,
5329  VmaSuballocationType type,
5330  VkDeviceSize allocSize,
5331  VmaAllocation hAllocation)
5332 {
5333  VMA_ASSERT(request.item != m_Suballocations.end());
5334  VmaSuballocation& suballoc = *request.item;
5335  // Given suballocation is a free block.
5336  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5337  // Given offset is inside this suballocation.
5338  VMA_ASSERT(request.offset >= suballoc.offset);
5339  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
5340  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
5341  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
5342 
5343  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
5344  // it to become used.
5345  UnregisterFreeSuballocation(request.item);
5346 
5347  suballoc.offset = request.offset;
5348  suballoc.size = allocSize;
5349  suballoc.type = type;
5350  suballoc.hAllocation = hAllocation;
5351 
5352  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
5353  if(paddingEnd)
5354  {
5355  VmaSuballocation paddingSuballoc = {};
5356  paddingSuballoc.offset = request.offset + allocSize;
5357  paddingSuballoc.size = paddingEnd;
5358  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5359  VmaSuballocationList::iterator next = request.item;
5360  ++next;
5361  const VmaSuballocationList::iterator paddingEndItem =
5362  m_Suballocations.insert(next, paddingSuballoc);
5363  RegisterFreeSuballocation(paddingEndItem);
5364  }
5365 
5366  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
5367  if(paddingBegin)
5368  {
5369  VmaSuballocation paddingSuballoc = {};
5370  paddingSuballoc.offset = request.offset - paddingBegin;
5371  paddingSuballoc.size = paddingBegin;
5372  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5373  const VmaSuballocationList::iterator paddingBeginItem =
5374  m_Suballocations.insert(request.item, paddingSuballoc);
5375  RegisterFreeSuballocation(paddingBeginItem);
5376  }
5377 
5378  // Update totals.
5379  m_FreeCount = m_FreeCount - 1;
5380  if(paddingBegin > 0)
5381  {
5382  ++m_FreeCount;
5383  }
5384  if(paddingEnd > 0)
5385  {
5386  ++m_FreeCount;
5387  }
5388  m_SumFreeSize -= allocSize;
5389 }
5390 
5391 void VmaBlockMetadata::Free(const VmaAllocation allocation)
5392 {
5393  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5394  suballocItem != m_Suballocations.end();
5395  ++suballocItem)
5396  {
5397  VmaSuballocation& suballoc = *suballocItem;
5398  if(suballoc.hAllocation == allocation)
5399  {
5400  FreeSuballocation(suballocItem);
5401  VMA_HEAVY_ASSERT(Validate());
5402  return;
5403  }
5404  }
5405  VMA_ASSERT(0 && "Not found!");
5406 }
5407 
5408 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
5409 {
5410  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5411  suballocItem != m_Suballocations.end();
5412  ++suballocItem)
5413  {
5414  VmaSuballocation& suballoc = *suballocItem;
5415  if(suballoc.offset == offset)
5416  {
5417  FreeSuballocation(suballocItem);
5418  return;
5419  }
5420  }
5421  VMA_ASSERT(0 && "Not found!");
5422 }
5423 
5424 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
5425 {
5426  VkDeviceSize lastSize = 0;
5427  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
5428  {
5429  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
5430 
5431  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
5432  {
5433  VMA_ASSERT(0);
5434  return false;
5435  }
5436  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5437  {
5438  VMA_ASSERT(0);
5439  return false;
5440  }
5441  if(it->size < lastSize)
5442  {
5443  VMA_ASSERT(0);
5444  return false;
5445  }
5446 
5447  lastSize = it->size;
5448  }
5449  return true;
5450 }
5451 
5452 bool VmaBlockMetadata::CheckAllocation(
5453  uint32_t currentFrameIndex,
5454  uint32_t frameInUseCount,
5455  VkDeviceSize bufferImageGranularity,
5456  VkDeviceSize allocSize,
5457  VkDeviceSize allocAlignment,
5458  VmaSuballocationType allocType,
5459  VmaSuballocationList::const_iterator suballocItem,
5460  bool canMakeOtherLost,
5461  VkDeviceSize* pOffset,
5462  size_t* itemsToMakeLostCount,
5463  VkDeviceSize* pSumFreeSize,
5464  VkDeviceSize* pSumItemSize) const
5465 {
5466  VMA_ASSERT(allocSize > 0);
5467  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5468  VMA_ASSERT(suballocItem != m_Suballocations.cend());
5469  VMA_ASSERT(pOffset != VMA_NULL);
5470 
5471  *itemsToMakeLostCount = 0;
5472  *pSumFreeSize = 0;
5473  *pSumItemSize = 0;
5474 
5475  if(canMakeOtherLost)
5476  {
5477  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5478  {
5479  *pSumFreeSize = suballocItem->size;
5480  }
5481  else
5482  {
5483  if(suballocItem->hAllocation->CanBecomeLost() &&
5484  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5485  {
5486  ++*itemsToMakeLostCount;
5487  *pSumItemSize = suballocItem->size;
5488  }
5489  else
5490  {
5491  return false;
5492  }
5493  }
5494 
5495  // Remaining size is too small for this request: Early return.
5496  if(m_Size - suballocItem->offset < allocSize)
5497  {
5498  return false;
5499  }
5500 
5501  // Start from offset equal to beginning of this suballocation.
5502  *pOffset = suballocItem->offset;
5503 
5504  // Apply VMA_DEBUG_MARGIN at the beginning.
5505  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5506  {
5507  *pOffset += VMA_DEBUG_MARGIN;
5508  }
5509 
5510  // Apply alignment.
5511  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5512  *pOffset = VmaAlignUp(*pOffset, alignment);
5513 
5514  // Check previous suballocations for BufferImageGranularity conflicts.
5515  // Make bigger alignment if necessary.
5516  if(bufferImageGranularity > 1)
5517  {
5518  bool bufferImageGranularityConflict = false;
5519  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5520  while(prevSuballocItem != m_Suballocations.cbegin())
5521  {
5522  --prevSuballocItem;
5523  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5524  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5525  {
5526  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5527  {
5528  bufferImageGranularityConflict = true;
5529  break;
5530  }
5531  }
5532  else
5533  // Already on previous page.
5534  break;
5535  }
5536  if(bufferImageGranularityConflict)
5537  {
5538  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5539  }
5540  }
5541 
5542  // Now that we have final *pOffset, check if we are past suballocItem.
5543  // If yes, return false - this function should be called for another suballocItem as starting point.
5544  if(*pOffset >= suballocItem->offset + suballocItem->size)
5545  {
5546  return false;
5547  }
5548 
5549  // Calculate padding at the beginning based on current offset.
5550  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
5551 
5552  // Calculate required margin at the end if this is not last suballocation.
5553  VmaSuballocationList::const_iterator next = suballocItem;
5554  ++next;
5555  const VkDeviceSize requiredEndMargin =
5556  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5557 
5558  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
5559  // Another early return check.
5560  if(suballocItem->offset + totalSize > m_Size)
5561  {
5562  return false;
5563  }
5564 
5565  // Advance lastSuballocItem until desired size is reached.
5566  // Update itemsToMakeLostCount.
5567  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
5568  if(totalSize > suballocItem->size)
5569  {
5570  VkDeviceSize remainingSize = totalSize - suballocItem->size;
5571  while(remainingSize > 0)
5572  {
5573  ++lastSuballocItem;
5574  if(lastSuballocItem == m_Suballocations.cend())
5575  {
5576  return false;
5577  }
5578  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5579  {
5580  *pSumFreeSize += lastSuballocItem->size;
5581  }
5582  else
5583  {
5584  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
5585  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
5586  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5587  {
5588  ++*itemsToMakeLostCount;
5589  *pSumItemSize += lastSuballocItem->size;
5590  }
5591  else
5592  {
5593  return false;
5594  }
5595  }
5596  remainingSize = (lastSuballocItem->size < remainingSize) ?
5597  remainingSize - lastSuballocItem->size : 0;
5598  }
5599  }
5600 
5601  // Check next suballocations for BufferImageGranularity conflicts.
5602  // If conflict exists, we must mark more allocations lost or fail.
5603  if(bufferImageGranularity > 1)
5604  {
5605  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
5606  ++nextSuballocItem;
5607  while(nextSuballocItem != m_Suballocations.cend())
5608  {
5609  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5610  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5611  {
5612  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5613  {
5614  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
5615  if(nextSuballoc.hAllocation->CanBecomeLost() &&
5616  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5617  {
5618  ++*itemsToMakeLostCount;
5619  }
5620  else
5621  {
5622  return false;
5623  }
5624  }
5625  }
5626  else
5627  {
5628  // Already on next page.
5629  break;
5630  }
5631  ++nextSuballocItem;
5632  }
5633  }
5634  }
5635  else
5636  {
5637  const VmaSuballocation& suballoc = *suballocItem;
5638  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5639 
5640  *pSumFreeSize = suballoc.size;
5641 
5642  // Size of this suballocation is too small for this request: Early return.
5643  if(suballoc.size < allocSize)
5644  {
5645  return false;
5646  }
5647 
5648  // Start from offset equal to beginning of this suballocation.
5649  *pOffset = suballoc.offset;
5650 
5651  // Apply VMA_DEBUG_MARGIN at the beginning.
5652  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5653  {
5654  *pOffset += VMA_DEBUG_MARGIN;
5655  }
5656 
5657  // Apply alignment.
5658  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5659  *pOffset = VmaAlignUp(*pOffset, alignment);
5660 
5661  // Check previous suballocations for BufferImageGranularity conflicts.
5662  // Make bigger alignment if necessary.
5663  if(bufferImageGranularity > 1)
5664  {
5665  bool bufferImageGranularityConflict = false;
5666  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5667  while(prevSuballocItem != m_Suballocations.cbegin())
5668  {
5669  --prevSuballocItem;
5670  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5671  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5672  {
5673  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5674  {
5675  bufferImageGranularityConflict = true;
5676  break;
5677  }
5678  }
5679  else
5680  // Already on previous page.
5681  break;
5682  }
5683  if(bufferImageGranularityConflict)
5684  {
5685  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5686  }
5687  }
5688 
5689  // Calculate padding at the beginning based on current offset.
5690  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
5691 
5692  // Calculate required margin at the end if this is not last suballocation.
5693  VmaSuballocationList::const_iterator next = suballocItem;
5694  ++next;
5695  const VkDeviceSize requiredEndMargin =
5696  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5697 
5698  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
5699  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
5700  {
5701  return false;
5702  }
5703 
5704  // Check next suballocations for BufferImageGranularity conflicts.
5705  // If conflict exists, allocation cannot be made here.
5706  if(bufferImageGranularity > 1)
5707  {
5708  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
5709  ++nextSuballocItem;
5710  while(nextSuballocItem != m_Suballocations.cend())
5711  {
5712  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5713  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5714  {
5715  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5716  {
5717  return false;
5718  }
5719  }
5720  else
5721  {
5722  // Already on next page.
5723  break;
5724  }
5725  ++nextSuballocItem;
5726  }
5727  }
5728  }
5729 
5730  // All tests passed: Success. pOffset is already filled.
5731  return true;
5732 }
5733 
5734 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
5735 {
5736  VMA_ASSERT(item != m_Suballocations.end());
5737  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5738 
5739  VmaSuballocationList::iterator nextItem = item;
5740  ++nextItem;
5741  VMA_ASSERT(nextItem != m_Suballocations.end());
5742  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
5743 
5744  item->size += nextItem->size;
5745  --m_FreeCount;
5746  m_Suballocations.erase(nextItem);
5747 }
5748 
5749 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
5750 {
5751  // Change this suballocation to be marked as free.
5752  VmaSuballocation& suballoc = *suballocItem;
5753  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5754  suballoc.hAllocation = VK_NULL_HANDLE;
5755 
5756  // Update totals.
5757  ++m_FreeCount;
5758  m_SumFreeSize += suballoc.size;
5759 
5760  // Merge with previous and/or next suballocation if it's also free.
5761  bool mergeWithNext = false;
5762  bool mergeWithPrev = false;
5763 
5764  VmaSuballocationList::iterator nextItem = suballocItem;
5765  ++nextItem;
5766  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
5767  {
5768  mergeWithNext = true;
5769  }
5770 
5771  VmaSuballocationList::iterator prevItem = suballocItem;
5772  if(suballocItem != m_Suballocations.begin())
5773  {
5774  --prevItem;
5775  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5776  {
5777  mergeWithPrev = true;
5778  }
5779  }
5780 
5781  if(mergeWithNext)
5782  {
5783  UnregisterFreeSuballocation(nextItem);
5784  MergeFreeWithNext(suballocItem);
5785  }
5786 
5787  if(mergeWithPrev)
5788  {
5789  UnregisterFreeSuballocation(prevItem);
5790  MergeFreeWithNext(prevItem);
5791  RegisterFreeSuballocation(prevItem);
5792  return prevItem;
5793  }
5794  else
5795  {
5796  RegisterFreeSuballocation(suballocItem);
5797  return suballocItem;
5798  }
5799 }
5800 
5801 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
5802 {
5803  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5804  VMA_ASSERT(item->size > 0);
5805 
5806  // You may want to enable this validation at the beginning or at the end of
5807  // this function, depending on what do you want to check.
5808  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5809 
5810  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5811  {
5812  if(m_FreeSuballocationsBySize.empty())
5813  {
5814  m_FreeSuballocationsBySize.push_back(item);
5815  }
5816  else
5817  {
5818  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
5819  }
5820  }
5821 
5822  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5823 }
5824 
5825 
5826 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
5827 {
5828  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5829  VMA_ASSERT(item->size > 0);
5830 
5831  // You may want to enable this validation at the beginning or at the end of
5832  // this function, depending on what do you want to check.
5833  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5834 
5835  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5836  {
5837  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5838  m_FreeSuballocationsBySize.data(),
5839  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
5840  item,
5841  VmaSuballocationItemSizeLess());
5842  for(size_t index = it - m_FreeSuballocationsBySize.data();
5843  index < m_FreeSuballocationsBySize.size();
5844  ++index)
5845  {
5846  if(m_FreeSuballocationsBySize[index] == item)
5847  {
5848  VmaVectorRemove(m_FreeSuballocationsBySize, index);
5849  return;
5850  }
5851  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
5852  }
5853  VMA_ASSERT(0 && "Not found.");
5854  }
5855 
5856  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5857 }
5858 
5860 // class VmaDeviceMemoryMapping
5861 
5862 VmaDeviceMemoryMapping::VmaDeviceMemoryMapping() :
5863  m_MapCount(0),
5864  m_pMappedData(VMA_NULL)
5865 {
5866 }
5867 
5868 VmaDeviceMemoryMapping::~VmaDeviceMemoryMapping()
5869 {
5870  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
5871 }
5872 
5873 VkResult VmaDeviceMemoryMapping::Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData)
5874 {
5875  if(count == 0)
5876  {
5877  return VK_SUCCESS;
5878  }
5879 
5880  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
5881  if(m_MapCount != 0)
5882  {
5883  m_MapCount += count;
5884  VMA_ASSERT(m_pMappedData != VMA_NULL);
5885  if(ppData != VMA_NULL)
5886  {
5887  *ppData = m_pMappedData;
5888  }
5889  return VK_SUCCESS;
5890  }
5891  else
5892  {
5893  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5894  hAllocator->m_hDevice,
5895  hMemory,
5896  0, // offset
5897  VK_WHOLE_SIZE,
5898  0, // flags
5899  &m_pMappedData);
5900  if(result == VK_SUCCESS)
5901  {
5902  if(ppData != VMA_NULL)
5903  {
5904  *ppData = m_pMappedData;
5905  }
5906  m_MapCount = count;
5907  }
5908  return result;
5909  }
5910 }
5911 
5912 void VmaDeviceMemoryMapping::Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count)
5913 {
5914  if(count == 0)
5915  {
5916  return;
5917  }
5918 
5919  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
5920  if(m_MapCount >= count)
5921  {
5922  m_MapCount -= count;
5923  if(m_MapCount == 0)
5924  {
5925  m_pMappedData = VMA_NULL;
5926  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, hMemory);
5927  }
5928  }
5929  else
5930  {
5931  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
5932  }
5933 }
5934 
5936 // class VmaDeviceMemoryBlock
5937 
5938 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
5939  m_MemoryTypeIndex(UINT32_MAX),
5940  m_hMemory(VK_NULL_HANDLE),
5941  m_Metadata(hAllocator)
5942 {
5943 }
5944 
5945 void VmaDeviceMemoryBlock::Init(
5946  uint32_t newMemoryTypeIndex,
5947  VkDeviceMemory newMemory,
5948  VkDeviceSize newSize)
5949 {
5950  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
5951 
5952  m_MemoryTypeIndex = newMemoryTypeIndex;
5953  m_hMemory = newMemory;
5954 
5955  m_Metadata.Init(newSize);
5956 }
5957 
5958 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
5959 {
5960  // This is the most important assert in the entire library.
5961  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
5962  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
5963 
5964  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
5965  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
5966  m_hMemory = VK_NULL_HANDLE;
5967 }
5968 
5969 bool VmaDeviceMemoryBlock::Validate() const
5970 {
5971  if((m_hMemory == VK_NULL_HANDLE) ||
5972  (m_Metadata.GetSize() == 0))
5973  {
5974  return false;
5975  }
5976 
5977  return m_Metadata.Validate();
5978 }
5979 
5980 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
5981 {
5982  return m_Mapping.Map(hAllocator, m_hMemory, count, ppData);
5983 }
5984 
5985 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
5986 {
5987  m_Mapping.Unmap(hAllocator, m_hMemory, count);
5988 }
5989 
5990 static void InitStatInfo(VmaStatInfo& outInfo)
5991 {
5992  memset(&outInfo, 0, sizeof(outInfo));
5993  outInfo.allocationSizeMin = UINT64_MAX;
5994  outInfo.unusedRangeSizeMin = UINT64_MAX;
5995 }
5996 
5997 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
5998 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
5999 {
6000  inoutInfo.blockCount += srcInfo.blockCount;
6001  inoutInfo.allocationCount += srcInfo.allocationCount;
6002  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
6003  inoutInfo.usedBytes += srcInfo.usedBytes;
6004  inoutInfo.unusedBytes += srcInfo.unusedBytes;
6005  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
6006  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
6007  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
6008  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
6009 }
6010 
6011 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
6012 {
6013  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
6014  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
6015  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
6016  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
6017 }
6018 
6019 VmaPool_T::VmaPool_T(
6020  VmaAllocator hAllocator,
6021  const VmaPoolCreateInfo& createInfo) :
6022  m_BlockVector(
6023  hAllocator,
6024  createInfo.memoryTypeIndex,
6025  createInfo.blockSize,
6026  createInfo.minBlockCount,
6027  createInfo.maxBlockCount,
6028  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
6029  createInfo.frameInUseCount,
6030  true) // isCustomPool
6031 {
6032 }
6033 
6034 VmaPool_T::~VmaPool_T()
6035 {
6036 }
6037 
6038 #if VMA_STATS_STRING_ENABLED
6039 
6040 #endif // #if VMA_STATS_STRING_ENABLED
6041 
6042 VmaBlockVector::VmaBlockVector(
6043  VmaAllocator hAllocator,
6044  uint32_t memoryTypeIndex,
6045  VkDeviceSize preferredBlockSize,
6046  size_t minBlockCount,
6047  size_t maxBlockCount,
6048  VkDeviceSize bufferImageGranularity,
6049  uint32_t frameInUseCount,
6050  bool isCustomPool) :
6051  m_hAllocator(hAllocator),
6052  m_MemoryTypeIndex(memoryTypeIndex),
6053  m_PreferredBlockSize(preferredBlockSize),
6054  m_MinBlockCount(minBlockCount),
6055  m_MaxBlockCount(maxBlockCount),
6056  m_BufferImageGranularity(bufferImageGranularity),
6057  m_FrameInUseCount(frameInUseCount),
6058  m_IsCustomPool(isCustomPool),
6059  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
6060  m_HasEmptyBlock(false),
6061  m_pDefragmentator(VMA_NULL)
6062 {
6063 }
6064 
6065 VmaBlockVector::~VmaBlockVector()
6066 {
6067  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
6068 
6069  for(size_t i = m_Blocks.size(); i--; )
6070  {
6071  m_Blocks[i]->Destroy(m_hAllocator);
6072  vma_delete(m_hAllocator, m_Blocks[i]);
6073  }
6074 }
6075 
6076 VkResult VmaBlockVector::CreateMinBlocks()
6077 {
6078  for(size_t i = 0; i < m_MinBlockCount; ++i)
6079  {
6080  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
6081  if(res != VK_SUCCESS)
6082  {
6083  return res;
6084  }
6085  }
6086  return VK_SUCCESS;
6087 }
6088 
6089 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
6090 {
6091  pStats->size = 0;
6092  pStats->unusedSize = 0;
6093  pStats->allocationCount = 0;
6094  pStats->unusedRangeCount = 0;
6095  pStats->unusedRangeSizeMax = 0;
6096 
6097  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6098 
6099  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6100  {
6101  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6102  VMA_ASSERT(pBlock);
6103  VMA_HEAVY_ASSERT(pBlock->Validate());
6104  pBlock->m_Metadata.AddPoolStats(*pStats);
6105  }
6106 }
6107 
6108 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
6109 
6110 VkResult VmaBlockVector::Allocate(
6111  VmaPool hCurrentPool,
6112  uint32_t currentFrameIndex,
6113  const VkMemoryRequirements& vkMemReq,
6114  const VmaAllocationCreateInfo& createInfo,
6115  VmaSuballocationType suballocType,
6116  VmaAllocation* pAllocation)
6117 {
6118  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
6119  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
6120 
6121  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6122 
6123  // 1. Search existing allocations. Try to allocate without making other allocations lost.
6124  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6125  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6126  {
6127  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6128  VMA_ASSERT(pCurrBlock);
6129  VmaAllocationRequest currRequest = {};
6130  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6131  currentFrameIndex,
6132  m_FrameInUseCount,
6133  m_BufferImageGranularity,
6134  vkMemReq.size,
6135  vkMemReq.alignment,
6136  suballocType,
6137  false, // canMakeOtherLost
6138  &currRequest))
6139  {
6140  // Allocate from pCurrBlock.
6141  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
6142 
6143  if(mapped)
6144  {
6145  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
6146  if(res != VK_SUCCESS)
6147  {
6148  return res;
6149  }
6150  }
6151 
6152  // We no longer have an empty Allocation.
6153  if(pCurrBlock->m_Metadata.IsEmpty())
6154  {
6155  m_HasEmptyBlock = false;
6156  }
6157 
6158  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6159  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
6160  (*pAllocation)->InitBlockAllocation(
6161  hCurrentPool,
6162  pCurrBlock,
6163  currRequest.offset,
6164  vkMemReq.alignment,
6165  vkMemReq.size,
6166  suballocType,
6167  mapped,
6168  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6169  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
6170  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6171  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6172  return VK_SUCCESS;
6173  }
6174  }
6175 
6176  const bool canCreateNewBlock =
6177  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
6178  (m_Blocks.size() < m_MaxBlockCount);
6179 
6180  // 2. Try to create new block.
6181  if(canCreateNewBlock)
6182  {
6183  // Calculate optimal size for new block.
6184  VkDeviceSize newBlockSize = m_PreferredBlockSize;
6185  uint32_t newBlockSizeShift = 0;
6186  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
6187 
6188  // Allocating blocks of other sizes is allowed only in default pools.
6189  // In custom pools block size is fixed.
6190  if(m_IsCustomPool == false)
6191  {
6192  // Allocate 1/8, 1/4, 1/2 as first blocks.
6193  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
6194  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
6195  {
6196  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6197  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= vkMemReq.size * 2)
6198  {
6199  newBlockSize = smallerNewBlockSize;
6200  ++newBlockSizeShift;
6201  }
6202  else
6203  {
6204  break;
6205  }
6206  }
6207  }
6208 
6209  size_t newBlockIndex = 0;
6210  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
6211  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
6212  if(m_IsCustomPool == false)
6213  {
6214  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
6215  {
6216  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6217  if(smallerNewBlockSize >= vkMemReq.size)
6218  {
6219  newBlockSize = smallerNewBlockSize;
6220  ++newBlockSizeShift;
6221  res = CreateBlock(newBlockSize, &newBlockIndex);
6222  }
6223  else
6224  {
6225  break;
6226  }
6227  }
6228  }
6229 
6230  if(res == VK_SUCCESS)
6231  {
6232  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
6233  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= vkMemReq.size);
6234 
6235  if(mapped)
6236  {
6237  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
6238  if(res != VK_SUCCESS)
6239  {
6240  return res;
6241  }
6242  }
6243 
6244  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
6245  VmaAllocationRequest allocRequest;
6246  pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
6247  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6248  pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
6249  (*pAllocation)->InitBlockAllocation(
6250  hCurrentPool,
6251  pBlock,
6252  allocRequest.offset,
6253  vkMemReq.alignment,
6254  vkMemReq.size,
6255  suballocType,
6256  mapped,
6257  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6258  VMA_HEAVY_ASSERT(pBlock->Validate());
6259  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
6260  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6261  return VK_SUCCESS;
6262  }
6263  }
6264 
6265  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
6266 
6267  // 3. Try to allocate from existing blocks with making other allocations lost.
6268  if(canMakeOtherLost)
6269  {
6270  uint32_t tryIndex = 0;
6271  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
6272  {
6273  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
6274  VmaAllocationRequest bestRequest = {};
6275  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
6276 
6277  // 1. Search existing allocations.
6278  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6279  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6280  {
6281  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6282  VMA_ASSERT(pCurrBlock);
6283  VmaAllocationRequest currRequest = {};
6284  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6285  currentFrameIndex,
6286  m_FrameInUseCount,
6287  m_BufferImageGranularity,
6288  vkMemReq.size,
6289  vkMemReq.alignment,
6290  suballocType,
6291  canMakeOtherLost,
6292  &currRequest))
6293  {
6294  const VkDeviceSize currRequestCost = currRequest.CalcCost();
6295  if(pBestRequestBlock == VMA_NULL ||
6296  currRequestCost < bestRequestCost)
6297  {
6298  pBestRequestBlock = pCurrBlock;
6299  bestRequest = currRequest;
6300  bestRequestCost = currRequestCost;
6301 
6302  if(bestRequestCost == 0)
6303  {
6304  break;
6305  }
6306  }
6307  }
6308  }
6309 
6310  if(pBestRequestBlock != VMA_NULL)
6311  {
6312  if(mapped)
6313  {
6314  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
6315  if(res != VK_SUCCESS)
6316  {
6317  return res;
6318  }
6319  }
6320 
6321  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
6322  currentFrameIndex,
6323  m_FrameInUseCount,
6324  &bestRequest))
6325  {
6326  // We no longer have an empty Allocation.
6327  if(pBestRequestBlock->m_Metadata.IsEmpty())
6328  {
6329  m_HasEmptyBlock = false;
6330  }
6331  // Allocate from this pBlock.
6332  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6333  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
6334  (*pAllocation)->InitBlockAllocation(
6335  hCurrentPool,
6336  pBestRequestBlock,
6337  bestRequest.offset,
6338  vkMemReq.alignment,
6339  vkMemReq.size,
6340  suballocType,
6341  mapped,
6342  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6343  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
6344  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6345  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6346  return VK_SUCCESS;
6347  }
6348  // else: Some allocations must have been touched while we are here. Next try.
6349  }
6350  else
6351  {
6352  // Could not find place in any of the blocks - break outer loop.
6353  break;
6354  }
6355  }
6356  /* Maximum number of tries exceeded - a very unlike event when many other
6357  threads are simultaneously touching allocations making it impossible to make
6358  lost at the same time as we try to allocate. */
6359  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
6360  {
6361  return VK_ERROR_TOO_MANY_OBJECTS;
6362  }
6363  }
6364 
6365  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6366 }
6367 
6368 void VmaBlockVector::Free(
6369  VmaAllocation hAllocation)
6370 {
6371  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
6372 
6373  // Scope for lock.
6374  {
6375  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6376 
6377  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
6378 
6379  if(hAllocation->IsPersistentMap())
6380  {
6381  pBlock->m_Mapping.Unmap(m_hAllocator, pBlock->m_hMemory, 1);
6382  }
6383 
6384  pBlock->m_Metadata.Free(hAllocation);
6385  VMA_HEAVY_ASSERT(pBlock->Validate());
6386 
6387  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
6388 
6389  // pBlock became empty after this deallocation.
6390  if(pBlock->m_Metadata.IsEmpty())
6391  {
6392  // Already has empty Allocation. We don't want to have two, so delete this one.
6393  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
6394  {
6395  pBlockToDelete = pBlock;
6396  Remove(pBlock);
6397  }
6398  // We now have first empty Allocation.
6399  else
6400  {
6401  m_HasEmptyBlock = true;
6402  }
6403  }
6404  // pBlock didn't become empty, but we have another empty block - find and free that one.
6405  // (This is optional, heuristics.)
6406  else if(m_HasEmptyBlock)
6407  {
6408  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
6409  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
6410  {
6411  pBlockToDelete = pLastBlock;
6412  m_Blocks.pop_back();
6413  m_HasEmptyBlock = false;
6414  }
6415  }
6416 
6417  IncrementallySortBlocks();
6418  }
6419 
6420  // Destruction of a free Allocation. Deferred until this point, outside of mutex
6421  // lock, for performance reason.
6422  if(pBlockToDelete != VMA_NULL)
6423  {
6424  VMA_DEBUG_LOG(" Deleted empty allocation");
6425  pBlockToDelete->Destroy(m_hAllocator);
6426  vma_delete(m_hAllocator, pBlockToDelete);
6427  }
6428 }
6429 
6430 size_t VmaBlockVector::CalcMaxBlockSize() const
6431 {
6432  size_t result = 0;
6433  for(size_t i = m_Blocks.size(); i--; )
6434  {
6435  result = VMA_MAX((uint64_t)result, (uint64_t)m_Blocks[i]->m_Metadata.GetSize());
6436  if(result >= m_PreferredBlockSize)
6437  {
6438  break;
6439  }
6440  }
6441  return result;
6442 }
6443 
6444 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
6445 {
6446  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6447  {
6448  if(m_Blocks[blockIndex] == pBlock)
6449  {
6450  VmaVectorRemove(m_Blocks, blockIndex);
6451  return;
6452  }
6453  }
6454  VMA_ASSERT(0);
6455 }
6456 
6457 void VmaBlockVector::IncrementallySortBlocks()
6458 {
6459  // Bubble sort only until first swap.
6460  for(size_t i = 1; i < m_Blocks.size(); ++i)
6461  {
6462  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
6463  {
6464  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
6465  return;
6466  }
6467  }
6468 }
6469 
6470 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
6471 {
6472  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6473  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
6474  allocInfo.allocationSize = blockSize;
6475  VkDeviceMemory mem = VK_NULL_HANDLE;
6476  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
6477  if(res < 0)
6478  {
6479  return res;
6480  }
6481 
6482  // New VkDeviceMemory successfully created.
6483 
6484  // Create new Allocation for it.
6485  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
6486  pBlock->Init(
6487  m_MemoryTypeIndex,
6488  mem,
6489  allocInfo.allocationSize);
6490 
6491  m_Blocks.push_back(pBlock);
6492  if(pNewBlockIndex != VMA_NULL)
6493  {
6494  *pNewBlockIndex = m_Blocks.size() - 1;
6495  }
6496 
6497  return VK_SUCCESS;
6498 }
6499 
6500 #if VMA_STATS_STRING_ENABLED
6501 
6502 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
6503 {
6504  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6505 
6506  json.BeginObject();
6507 
6508  if(m_IsCustomPool)
6509  {
6510  json.WriteString("MemoryTypeIndex");
6511  json.WriteNumber(m_MemoryTypeIndex);
6512 
6513  json.WriteString("BlockSize");
6514  json.WriteNumber(m_PreferredBlockSize);
6515 
6516  json.WriteString("BlockCount");
6517  json.BeginObject(true);
6518  if(m_MinBlockCount > 0)
6519  {
6520  json.WriteString("Min");
6521  json.WriteNumber((uint64_t)m_MinBlockCount);
6522  }
6523  if(m_MaxBlockCount < SIZE_MAX)
6524  {
6525  json.WriteString("Max");
6526  json.WriteNumber((uint64_t)m_MaxBlockCount);
6527  }
6528  json.WriteString("Cur");
6529  json.WriteNumber((uint64_t)m_Blocks.size());
6530  json.EndObject();
6531 
6532  if(m_FrameInUseCount > 0)
6533  {
6534  json.WriteString("FrameInUseCount");
6535  json.WriteNumber(m_FrameInUseCount);
6536  }
6537  }
6538  else
6539  {
6540  json.WriteString("PreferredBlockSize");
6541  json.WriteNumber(m_PreferredBlockSize);
6542  }
6543 
6544  json.WriteString("Blocks");
6545  json.BeginArray();
6546  for(size_t i = 0; i < m_Blocks.size(); ++i)
6547  {
6548  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
6549  }
6550  json.EndArray();
6551 
6552  json.EndObject();
6553 }
6554 
6555 #endif // #if VMA_STATS_STRING_ENABLED
6556 
6557 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
6558  VmaAllocator hAllocator,
6559  uint32_t currentFrameIndex)
6560 {
6561  if(m_pDefragmentator == VMA_NULL)
6562  {
6563  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
6564  hAllocator,
6565  this,
6566  currentFrameIndex);
6567  }
6568 
6569  return m_pDefragmentator;
6570 }
6571 
6572 VkResult VmaBlockVector::Defragment(
6573  VmaDefragmentationStats* pDefragmentationStats,
6574  VkDeviceSize& maxBytesToMove,
6575  uint32_t& maxAllocationsToMove)
6576 {
6577  if(m_pDefragmentator == VMA_NULL)
6578  {
6579  return VK_SUCCESS;
6580  }
6581 
6582  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6583 
6584  // Defragment.
6585  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
6586 
6587  // Accumulate statistics.
6588  if(pDefragmentationStats != VMA_NULL)
6589  {
6590  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
6591  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
6592  pDefragmentationStats->bytesMoved += bytesMoved;
6593  pDefragmentationStats->allocationsMoved += allocationsMoved;
6594  VMA_ASSERT(bytesMoved <= maxBytesToMove);
6595  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
6596  maxBytesToMove -= bytesMoved;
6597  maxAllocationsToMove -= allocationsMoved;
6598  }
6599 
6600  // Free empty blocks.
6601  m_HasEmptyBlock = false;
6602  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
6603  {
6604  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
6605  if(pBlock->m_Metadata.IsEmpty())
6606  {
6607  if(m_Blocks.size() > m_MinBlockCount)
6608  {
6609  if(pDefragmentationStats != VMA_NULL)
6610  {
6611  ++pDefragmentationStats->deviceMemoryBlocksFreed;
6612  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
6613  }
6614 
6615  VmaVectorRemove(m_Blocks, blockIndex);
6616  pBlock->Destroy(m_hAllocator);
6617  vma_delete(m_hAllocator, pBlock);
6618  }
6619  else
6620  {
6621  m_HasEmptyBlock = true;
6622  }
6623  }
6624  }
6625 
6626  return result;
6627 }
6628 
6629 void VmaBlockVector::DestroyDefragmentator()
6630 {
6631  if(m_pDefragmentator != VMA_NULL)
6632  {
6633  vma_delete(m_hAllocator, m_pDefragmentator);
6634  m_pDefragmentator = VMA_NULL;
6635  }
6636 }
6637 
6638 void VmaBlockVector::MakePoolAllocationsLost(
6639  uint32_t currentFrameIndex,
6640  size_t* pLostAllocationCount)
6641 {
6642  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6643  size_t lostAllocationCount = 0;
6644  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6645  {
6646  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6647  VMA_ASSERT(pBlock);
6648  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
6649  }
6650  if(pLostAllocationCount != VMA_NULL)
6651  {
6652  *pLostAllocationCount = lostAllocationCount;
6653  }
6654 }
6655 
6656 void VmaBlockVector::AddStats(VmaStats* pStats)
6657 {
6658  const uint32_t memTypeIndex = m_MemoryTypeIndex;
6659  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
6660 
6661  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6662 
6663  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6664  {
6665  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6666  VMA_ASSERT(pBlock);
6667  VMA_HEAVY_ASSERT(pBlock->Validate());
6668  VmaStatInfo allocationStatInfo;
6669  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
6670  VmaAddStatInfo(pStats->total, allocationStatInfo);
6671  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
6672  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
6673  }
6674 }
6675 
6677 // VmaDefragmentator members definition
6678 
6679 VmaDefragmentator::VmaDefragmentator(
6680  VmaAllocator hAllocator,
6681  VmaBlockVector* pBlockVector,
6682  uint32_t currentFrameIndex) :
6683  m_hAllocator(hAllocator),
6684  m_pBlockVector(pBlockVector),
6685  m_CurrentFrameIndex(currentFrameIndex),
6686  m_BytesMoved(0),
6687  m_AllocationsMoved(0),
6688  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
6689  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
6690 {
6691 }
6692 
6693 VmaDefragmentator::~VmaDefragmentator()
6694 {
6695  for(size_t i = m_Blocks.size(); i--; )
6696  {
6697  vma_delete(m_hAllocator, m_Blocks[i]);
6698  }
6699 }
6700 
6701 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
6702 {
6703  AllocationInfo allocInfo;
6704  allocInfo.m_hAllocation = hAlloc;
6705  allocInfo.m_pChanged = pChanged;
6706  m_Allocations.push_back(allocInfo);
6707 }
6708 
6709 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
6710 {
6711  // It has already been mapped for defragmentation.
6712  if(m_pMappedDataForDefragmentation)
6713  {
6714  *ppMappedData = m_pMappedDataForDefragmentation;
6715  return VK_SUCCESS;
6716  }
6717 
6718  // It is originally mapped.
6719  if(m_pBlock->m_Mapping.GetMappedData())
6720  {
6721  *ppMappedData = m_pBlock->m_Mapping.GetMappedData();
6722  return VK_SUCCESS;
6723  }
6724 
6725  // Map on first usage.
6726  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
6727  *ppMappedData = m_pMappedDataForDefragmentation;
6728  return res;
6729 }
6730 
6731 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
6732 {
6733  if(m_pMappedDataForDefragmentation != VMA_NULL)
6734  {
6735  m_pBlock->Unmap(hAllocator, 1);
6736  }
6737 }
6738 
6739 VkResult VmaDefragmentator::DefragmentRound(
6740  VkDeviceSize maxBytesToMove,
6741  uint32_t maxAllocationsToMove)
6742 {
6743  if(m_Blocks.empty())
6744  {
6745  return VK_SUCCESS;
6746  }
6747 
6748  size_t srcBlockIndex = m_Blocks.size() - 1;
6749  size_t srcAllocIndex = SIZE_MAX;
6750  for(;;)
6751  {
6752  // 1. Find next allocation to move.
6753  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
6754  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
6755  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
6756  {
6757  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
6758  {
6759  // Finished: no more allocations to process.
6760  if(srcBlockIndex == 0)
6761  {
6762  return VK_SUCCESS;
6763  }
6764  else
6765  {
6766  --srcBlockIndex;
6767  srcAllocIndex = SIZE_MAX;
6768  }
6769  }
6770  else
6771  {
6772  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
6773  }
6774  }
6775 
6776  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
6777  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
6778 
6779  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
6780  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
6781  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
6782  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
6783 
6784  // 2. Try to find new place for this allocation in preceding or current block.
6785  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
6786  {
6787  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
6788  VmaAllocationRequest dstAllocRequest;
6789  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
6790  m_CurrentFrameIndex,
6791  m_pBlockVector->GetFrameInUseCount(),
6792  m_pBlockVector->GetBufferImageGranularity(),
6793  size,
6794  alignment,
6795  suballocType,
6796  false, // canMakeOtherLost
6797  &dstAllocRequest) &&
6798  MoveMakesSense(
6799  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
6800  {
6801  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
6802 
6803  // Reached limit on number of allocations or bytes to move.
6804  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
6805  (m_BytesMoved + size > maxBytesToMove))
6806  {
6807  return VK_INCOMPLETE;
6808  }
6809 
6810  void* pDstMappedData = VMA_NULL;
6811  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
6812  if(res != VK_SUCCESS)
6813  {
6814  return res;
6815  }
6816 
6817  void* pSrcMappedData = VMA_NULL;
6818  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
6819  if(res != VK_SUCCESS)
6820  {
6821  return res;
6822  }
6823 
6824  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
6825  memcpy(
6826  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
6827  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
6828  static_cast<size_t>(size));
6829 
6830  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
6831  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
6832 
6833  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
6834 
6835  if(allocInfo.m_pChanged != VMA_NULL)
6836  {
6837  *allocInfo.m_pChanged = VK_TRUE;
6838  }
6839 
6840  ++m_AllocationsMoved;
6841  m_BytesMoved += size;
6842 
6843  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
6844 
6845  break;
6846  }
6847  }
6848 
6849  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
6850 
6851  if(srcAllocIndex > 0)
6852  {
6853  --srcAllocIndex;
6854  }
6855  else
6856  {
6857  if(srcBlockIndex > 0)
6858  {
6859  --srcBlockIndex;
6860  srcAllocIndex = SIZE_MAX;
6861  }
6862  else
6863  {
6864  return VK_SUCCESS;
6865  }
6866  }
6867  }
6868 }
6869 
6870 VkResult VmaDefragmentator::Defragment(
6871  VkDeviceSize maxBytesToMove,
6872  uint32_t maxAllocationsToMove)
6873 {
6874  if(m_Allocations.empty())
6875  {
6876  return VK_SUCCESS;
6877  }
6878 
6879  // Create block info for each block.
6880  const size_t blockCount = m_pBlockVector->m_Blocks.size();
6881  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6882  {
6883  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
6884  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
6885  m_Blocks.push_back(pBlockInfo);
6886  }
6887 
6888  // Sort them by m_pBlock pointer value.
6889  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
6890 
6891  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
6892  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
6893  {
6894  AllocationInfo& allocInfo = m_Allocations[blockIndex];
6895  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
6896  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6897  {
6898  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
6899  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
6900  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
6901  {
6902  (*it)->m_Allocations.push_back(allocInfo);
6903  }
6904  else
6905  {
6906  VMA_ASSERT(0);
6907  }
6908  }
6909  }
6910  m_Allocations.clear();
6911 
6912  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6913  {
6914  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
6915  pBlockInfo->CalcHasNonMovableAllocations();
6916  pBlockInfo->SortAllocationsBySizeDescecnding();
6917  }
6918 
6919  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
6920  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
6921 
6922  // Execute defragmentation rounds (the main part).
6923  VkResult result = VK_SUCCESS;
6924  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
6925  {
6926  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
6927  }
6928 
6929  // Unmap blocks that were mapped for defragmentation.
6930  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6931  {
6932  m_Blocks[blockIndex]->Unmap(m_hAllocator);
6933  }
6934 
6935  return result;
6936 }
6937 
6938 bool VmaDefragmentator::MoveMakesSense(
6939  size_t dstBlockIndex, VkDeviceSize dstOffset,
6940  size_t srcBlockIndex, VkDeviceSize srcOffset)
6941 {
6942  if(dstBlockIndex < srcBlockIndex)
6943  {
6944  return true;
6945  }
6946  if(dstBlockIndex > srcBlockIndex)
6947  {
6948  return false;
6949  }
6950  if(dstOffset < srcOffset)
6951  {
6952  return true;
6953  }
6954  return false;
6955 }
6956 
6958 // VmaAllocator_T
6959 
6960 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
6961  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
6962  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
6963  m_hDevice(pCreateInfo->device),
6964  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
6965  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
6966  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
6967  m_PreferredLargeHeapBlockSize(0),
6968  m_PhysicalDevice(pCreateInfo->physicalDevice),
6969  m_CurrentFrameIndex(0),
6970  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
6971 {
6972  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
6973 
6974  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
6975  memset(&m_MemProps, 0, sizeof(m_MemProps));
6976  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
6977 
6978  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
6979  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
6980 
6981  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6982  {
6983  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
6984  }
6985 
6986  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
6987  {
6988  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
6989  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
6990  }
6991 
6992  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
6993 
6994  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
6995  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
6996 
6997  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
6998  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
6999 
7000  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
7001  {
7002  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
7003  {
7004  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
7005  if(limit != VK_WHOLE_SIZE)
7006  {
7007  m_HeapSizeLimit[heapIndex] = limit;
7008  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
7009  {
7010  m_MemProps.memoryHeaps[heapIndex].size = limit;
7011  }
7012  }
7013  }
7014  }
7015 
7016  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7017  {
7018  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
7019 
7020  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
7021  this,
7022  memTypeIndex,
7023  preferredBlockSize,
7024  0,
7025  SIZE_MAX,
7026  GetBufferImageGranularity(),
7027  pCreateInfo->frameInUseCount,
7028  false); // isCustomPool
7029  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
7030  // becase minBlockCount is 0.
7031  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
7032  }
7033 }
7034 
7035 VmaAllocator_T::~VmaAllocator_T()
7036 {
7037  VMA_ASSERT(m_Pools.empty());
7038 
7039  for(size_t i = GetMemoryTypeCount(); i--; )
7040  {
7041  vma_delete(this, m_pDedicatedAllocations[i]);
7042  vma_delete(this, m_pBlockVectors[i]);
7043  }
7044 }
7045 
7046 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
7047 {
7048 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7049  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
7050  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
7051  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
7052  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
7053  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
7054  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
7055  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
7056  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
7057  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
7058  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
7059  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
7060  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
7061  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
7062  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
7063  if(m_UseKhrDedicatedAllocation)
7064  {
7065  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
7066  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
7067  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
7068  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
7069  }
7070 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7071 
7072 #define VMA_COPY_IF_NOT_NULL(funcName) \
7073  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
7074 
7075  if(pVulkanFunctions != VMA_NULL)
7076  {
7077  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
7078  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
7079  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
7080  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
7081  VMA_COPY_IF_NOT_NULL(vkMapMemory);
7082  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
7083  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
7084  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
7085  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
7086  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
7087  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
7088  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
7089  VMA_COPY_IF_NOT_NULL(vkCreateImage);
7090  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
7091  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
7092  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
7093  }
7094 
7095 #undef VMA_COPY_IF_NOT_NULL
7096 
7097  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
7098  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
7099  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
7100  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
7101  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
7102  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
7103  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
7104  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
7105  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
7106  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
7107  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
7108  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
7109  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
7110  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
7111  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
7112  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
7113  if(m_UseKhrDedicatedAllocation)
7114  {
7115  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
7116  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
7117  }
7118 }
7119 
7120 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
7121 {
7122  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7123  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
7124  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
7125  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
7126 }
7127 
7128 VkResult VmaAllocator_T::AllocateMemoryOfType(
7129  const VkMemoryRequirements& vkMemReq,
7130  bool dedicatedAllocation,
7131  VkBuffer dedicatedBuffer,
7132  VkImage dedicatedImage,
7133  const VmaAllocationCreateInfo& createInfo,
7134  uint32_t memTypeIndex,
7135  VmaSuballocationType suballocType,
7136  VmaAllocation* pAllocation)
7137 {
7138  VMA_ASSERT(pAllocation != VMA_NULL);
7139  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
7140 
7141  VmaAllocationCreateInfo finalCreateInfo = createInfo;
7142 
7143  // If memory type is not HOST_VISIBLE, disable MAPPED.
7144  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7145  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
7146  {
7147  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
7148  }
7149 
7150  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
7151  VMA_ASSERT(blockVector);
7152 
7153  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
7154  bool preferDedicatedMemory =
7155  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
7156  dedicatedAllocation ||
7157  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
7158  vkMemReq.size > preferredBlockSize / 2;
7159 
7160  if(preferDedicatedMemory &&
7161  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
7162  finalCreateInfo.pool == VK_NULL_HANDLE)
7163  {
7164  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
7165  }
7166 
7167  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
7168  {
7169  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7170  {
7171  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7172  }
7173  else
7174  {
7175  return AllocateDedicatedMemory(
7176  vkMemReq.size,
7177  suballocType,
7178  memTypeIndex,
7179  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7180  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7181  finalCreateInfo.pUserData,
7182  dedicatedBuffer,
7183  dedicatedImage,
7184  pAllocation);
7185  }
7186  }
7187  else
7188  {
7189  VkResult res = blockVector->Allocate(
7190  VK_NULL_HANDLE, // hCurrentPool
7191  m_CurrentFrameIndex.load(),
7192  vkMemReq,
7193  finalCreateInfo,
7194  suballocType,
7195  pAllocation);
7196  if(res == VK_SUCCESS)
7197  {
7198  return res;
7199  }
7200 
7201  // 5. Try dedicated memory.
7202  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7203  {
7204  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7205  }
7206  else
7207  {
7208  res = AllocateDedicatedMemory(
7209  vkMemReq.size,
7210  suballocType,
7211  memTypeIndex,
7212  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7213  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7214  finalCreateInfo.pUserData,
7215  dedicatedBuffer,
7216  dedicatedImage,
7217  pAllocation);
7218  if(res == VK_SUCCESS)
7219  {
7220  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
7221  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
7222  return VK_SUCCESS;
7223  }
7224  else
7225  {
7226  // Everything failed: Return error code.
7227  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7228  return res;
7229  }
7230  }
7231  }
7232 }
7233 
7234 VkResult VmaAllocator_T::AllocateDedicatedMemory(
7235  VkDeviceSize size,
7236  VmaSuballocationType suballocType,
7237  uint32_t memTypeIndex,
7238  bool map,
7239  bool isUserDataString,
7240  void* pUserData,
7241  VkBuffer dedicatedBuffer,
7242  VkImage dedicatedImage,
7243  VmaAllocation* pAllocation)
7244 {
7245  VMA_ASSERT(pAllocation);
7246 
7247  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
7248  allocInfo.memoryTypeIndex = memTypeIndex;
7249  allocInfo.allocationSize = size;
7250 
7251  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
7252  if(m_UseKhrDedicatedAllocation)
7253  {
7254  if(dedicatedBuffer != VK_NULL_HANDLE)
7255  {
7256  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
7257  dedicatedAllocInfo.buffer = dedicatedBuffer;
7258  allocInfo.pNext = &dedicatedAllocInfo;
7259  }
7260  else if(dedicatedImage != VK_NULL_HANDLE)
7261  {
7262  dedicatedAllocInfo.image = dedicatedImage;
7263  allocInfo.pNext = &dedicatedAllocInfo;
7264  }
7265  }
7266 
7267  // Allocate VkDeviceMemory.
7268  VkDeviceMemory hMemory = VK_NULL_HANDLE;
7269  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
7270  if(res < 0)
7271  {
7272  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7273  return res;
7274  }
7275 
7276  void* pMappedData = VMA_NULL;
7277  if(map)
7278  {
7279  res = (*m_VulkanFunctions.vkMapMemory)(
7280  m_hDevice,
7281  hMemory,
7282  0,
7283  VK_WHOLE_SIZE,
7284  0,
7285  &pMappedData);
7286  if(res < 0)
7287  {
7288  VMA_DEBUG_LOG(" vkMapMemory FAILED");
7289  FreeVulkanMemory(memTypeIndex, size, hMemory);
7290  return res;
7291  }
7292  }
7293 
7294  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
7295  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
7296  (*pAllocation)->SetUserData(this, pUserData);
7297 
7298  // Register it in m_pDedicatedAllocations.
7299  {
7300  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7301  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7302  VMA_ASSERT(pDedicatedAllocations);
7303  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
7304  }
7305 
7306  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
7307 
7308  return VK_SUCCESS;
7309 }
7310 
7311 void VmaAllocator_T::GetBufferMemoryRequirements(
7312  VkBuffer hBuffer,
7313  VkMemoryRequirements& memReq,
7314  bool& requiresDedicatedAllocation,
7315  bool& prefersDedicatedAllocation) const
7316 {
7317  if(m_UseKhrDedicatedAllocation)
7318  {
7319  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
7320  memReqInfo.buffer = hBuffer;
7321 
7322  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7323 
7324  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7325  memReq2.pNext = &memDedicatedReq;
7326 
7327  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7328 
7329  memReq = memReq2.memoryRequirements;
7330  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7331  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7332  }
7333  else
7334  {
7335  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
7336  requiresDedicatedAllocation = false;
7337  prefersDedicatedAllocation = false;
7338  }
7339 }
7340 
7341 void VmaAllocator_T::GetImageMemoryRequirements(
7342  VkImage hImage,
7343  VkMemoryRequirements& memReq,
7344  bool& requiresDedicatedAllocation,
7345  bool& prefersDedicatedAllocation) const
7346 {
7347  if(m_UseKhrDedicatedAllocation)
7348  {
7349  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
7350  memReqInfo.image = hImage;
7351 
7352  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7353 
7354  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7355  memReq2.pNext = &memDedicatedReq;
7356 
7357  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7358 
7359  memReq = memReq2.memoryRequirements;
7360  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7361  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7362  }
7363  else
7364  {
7365  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
7366  requiresDedicatedAllocation = false;
7367  prefersDedicatedAllocation = false;
7368  }
7369 }
7370 
7371 VkResult VmaAllocator_T::AllocateMemory(
7372  const VkMemoryRequirements& vkMemReq,
7373  bool requiresDedicatedAllocation,
7374  bool prefersDedicatedAllocation,
7375  VkBuffer dedicatedBuffer,
7376  VkImage dedicatedImage,
7377  const VmaAllocationCreateInfo& createInfo,
7378  VmaSuballocationType suballocType,
7379  VmaAllocation* pAllocation)
7380 {
7381  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
7382  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7383  {
7384  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
7385  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7386  }
7387  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7388  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
7389  {
7390  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
7391  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7392  }
7393  if(requiresDedicatedAllocation)
7394  {
7395  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7396  {
7397  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
7398  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7399  }
7400  if(createInfo.pool != VK_NULL_HANDLE)
7401  {
7402  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
7403  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7404  }
7405  }
7406  if((createInfo.pool != VK_NULL_HANDLE) &&
7407  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
7408  {
7409  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
7410  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7411  }
7412 
7413  if(createInfo.pool != VK_NULL_HANDLE)
7414  {
7415  return createInfo.pool->m_BlockVector.Allocate(
7416  createInfo.pool,
7417  m_CurrentFrameIndex.load(),
7418  vkMemReq,
7419  createInfo,
7420  suballocType,
7421  pAllocation);
7422  }
7423  else
7424  {
7425  // Bit mask of memory Vulkan types acceptable for this allocation.
7426  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
7427  uint32_t memTypeIndex = UINT32_MAX;
7428  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7429  if(res == VK_SUCCESS)
7430  {
7431  res = AllocateMemoryOfType(
7432  vkMemReq,
7433  requiresDedicatedAllocation || prefersDedicatedAllocation,
7434  dedicatedBuffer,
7435  dedicatedImage,
7436  createInfo,
7437  memTypeIndex,
7438  suballocType,
7439  pAllocation);
7440  // Succeeded on first try.
7441  if(res == VK_SUCCESS)
7442  {
7443  return res;
7444  }
7445  // Allocation from this memory type failed. Try other compatible memory types.
7446  else
7447  {
7448  for(;;)
7449  {
7450  // Remove old memTypeIndex from list of possibilities.
7451  memoryTypeBits &= ~(1u << memTypeIndex);
7452  // Find alternative memTypeIndex.
7453  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7454  if(res == VK_SUCCESS)
7455  {
7456  res = AllocateMemoryOfType(
7457  vkMemReq,
7458  requiresDedicatedAllocation || prefersDedicatedAllocation,
7459  dedicatedBuffer,
7460  dedicatedImage,
7461  createInfo,
7462  memTypeIndex,
7463  suballocType,
7464  pAllocation);
7465  // Allocation from this alternative memory type succeeded.
7466  if(res == VK_SUCCESS)
7467  {
7468  return res;
7469  }
7470  // else: Allocation from this memory type failed. Try next one - next loop iteration.
7471  }
7472  // No other matching memory type index could be found.
7473  else
7474  {
7475  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
7476  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7477  }
7478  }
7479  }
7480  }
7481  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
7482  else
7483  return res;
7484  }
7485 }
7486 
7487 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
7488 {
7489  VMA_ASSERT(allocation);
7490 
7491  if(allocation->CanBecomeLost() == false ||
7492  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
7493  {
7494  switch(allocation->GetType())
7495  {
7496  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7497  {
7498  VmaBlockVector* pBlockVector = VMA_NULL;
7499  VmaPool hPool = allocation->GetPool();
7500  if(hPool != VK_NULL_HANDLE)
7501  {
7502  pBlockVector = &hPool->m_BlockVector;
7503  }
7504  else
7505  {
7506  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7507  pBlockVector = m_pBlockVectors[memTypeIndex];
7508  }
7509  pBlockVector->Free(allocation);
7510  }
7511  break;
7512  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7513  FreeDedicatedMemory(allocation);
7514  break;
7515  default:
7516  VMA_ASSERT(0);
7517  }
7518  }
7519 
7520  allocation->SetUserData(this, VMA_NULL);
7521  vma_delete(this, allocation);
7522 }
7523 
7524 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
7525 {
7526  // Initialize.
7527  InitStatInfo(pStats->total);
7528  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
7529  InitStatInfo(pStats->memoryType[i]);
7530  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
7531  InitStatInfo(pStats->memoryHeap[i]);
7532 
7533  // Process default pools.
7534  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7535  {
7536  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
7537  VMA_ASSERT(pBlockVector);
7538  pBlockVector->AddStats(pStats);
7539  }
7540 
7541  // Process custom pools.
7542  {
7543  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7544  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
7545  {
7546  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
7547  }
7548  }
7549 
7550  // Process dedicated allocations.
7551  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7552  {
7553  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7554  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7555  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7556  VMA_ASSERT(pDedicatedAllocVector);
7557  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
7558  {
7559  VmaStatInfo allocationStatInfo;
7560  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
7561  VmaAddStatInfo(pStats->total, allocationStatInfo);
7562  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
7563  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
7564  }
7565  }
7566 
7567  // Postprocess.
7568  VmaPostprocessCalcStatInfo(pStats->total);
7569  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
7570  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
7571  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
7572  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
7573 }
7574 
7575 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
7576 
7577 VkResult VmaAllocator_T::Defragment(
7578  VmaAllocation* pAllocations,
7579  size_t allocationCount,
7580  VkBool32* pAllocationsChanged,
7581  const VmaDefragmentationInfo* pDefragmentationInfo,
7582  VmaDefragmentationStats* pDefragmentationStats)
7583 {
7584  if(pAllocationsChanged != VMA_NULL)
7585  {
7586  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
7587  }
7588  if(pDefragmentationStats != VMA_NULL)
7589  {
7590  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
7591  }
7592 
7593  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
7594 
7595  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
7596 
7597  const size_t poolCount = m_Pools.size();
7598 
7599  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
7600  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
7601  {
7602  VmaAllocation hAlloc = pAllocations[allocIndex];
7603  VMA_ASSERT(hAlloc);
7604  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
7605  // DedicatedAlloc cannot be defragmented.
7606  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
7607  // Only HOST_VISIBLE memory types can be defragmented.
7608  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
7609  // Lost allocation cannot be defragmented.
7610  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
7611  {
7612  VmaBlockVector* pAllocBlockVector = VMA_NULL;
7613 
7614  const VmaPool hAllocPool = hAlloc->GetPool();
7615  // This allocation belongs to custom pool.
7616  if(hAllocPool != VK_NULL_HANDLE)
7617  {
7618  pAllocBlockVector = &hAllocPool->GetBlockVector();
7619  }
7620  // This allocation belongs to general pool.
7621  else
7622  {
7623  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
7624  }
7625 
7626  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
7627 
7628  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
7629  &pAllocationsChanged[allocIndex] : VMA_NULL;
7630  pDefragmentator->AddAllocation(hAlloc, pChanged);
7631  }
7632  }
7633 
7634  VkResult result = VK_SUCCESS;
7635 
7636  // ======== Main processing.
7637 
7638  VkDeviceSize maxBytesToMove = SIZE_MAX;
7639  uint32_t maxAllocationsToMove = UINT32_MAX;
7640  if(pDefragmentationInfo != VMA_NULL)
7641  {
7642  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
7643  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
7644  }
7645 
7646  // Process standard memory.
7647  for(uint32_t memTypeIndex = 0;
7648  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
7649  ++memTypeIndex)
7650  {
7651  // Only HOST_VISIBLE memory types can be defragmented.
7652  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7653  {
7654  result = m_pBlockVectors[memTypeIndex]->Defragment(
7655  pDefragmentationStats,
7656  maxBytesToMove,
7657  maxAllocationsToMove);
7658  }
7659  }
7660 
7661  // Process custom pools.
7662  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
7663  {
7664  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
7665  pDefragmentationStats,
7666  maxBytesToMove,
7667  maxAllocationsToMove);
7668  }
7669 
7670  // ======== Destroy defragmentators.
7671 
7672  // Process custom pools.
7673  for(size_t poolIndex = poolCount; poolIndex--; )
7674  {
7675  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
7676  }
7677 
7678  // Process standard memory.
7679  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
7680  {
7681  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7682  {
7683  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
7684  }
7685  }
7686 
7687  return result;
7688 }
7689 
7690 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
7691 {
7692  if(hAllocation->CanBecomeLost())
7693  {
7694  /*
7695  Warning: This is a carefully designed algorithm.
7696  Do not modify unless you really know what you're doing :)
7697  */
7698  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
7699  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
7700  for(;;)
7701  {
7702  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
7703  {
7704  pAllocationInfo->memoryType = UINT32_MAX;
7705  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
7706  pAllocationInfo->offset = 0;
7707  pAllocationInfo->size = hAllocation->GetSize();
7708  pAllocationInfo->pMappedData = VMA_NULL;
7709  pAllocationInfo->pUserData = hAllocation->GetUserData();
7710  return;
7711  }
7712  else if(localLastUseFrameIndex == localCurrFrameIndex)
7713  {
7714  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7715  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7716  pAllocationInfo->offset = hAllocation->GetOffset();
7717  pAllocationInfo->size = hAllocation->GetSize();
7718  pAllocationInfo->pMappedData = VMA_NULL;
7719  pAllocationInfo->pUserData = hAllocation->GetUserData();
7720  return;
7721  }
7722  else // Last use time earlier than current time.
7723  {
7724  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
7725  {
7726  localLastUseFrameIndex = localCurrFrameIndex;
7727  }
7728  }
7729  }
7730  }
7731  else
7732  {
7733  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7734  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7735  pAllocationInfo->offset = hAllocation->GetOffset();
7736  pAllocationInfo->size = hAllocation->GetSize();
7737  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
7738  pAllocationInfo->pUserData = hAllocation->GetUserData();
7739  }
7740 }
7741 
7742 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
7743 {
7744  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
7745 
7746  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
7747 
7748  if(newCreateInfo.maxBlockCount == 0)
7749  {
7750  newCreateInfo.maxBlockCount = SIZE_MAX;
7751  }
7752  if(newCreateInfo.blockSize == 0)
7753  {
7754  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
7755  }
7756 
7757  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
7758 
7759  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
7760  if(res != VK_SUCCESS)
7761  {
7762  vma_delete(this, *pPool);
7763  *pPool = VMA_NULL;
7764  return res;
7765  }
7766 
7767  // Add to m_Pools.
7768  {
7769  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7770  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
7771  }
7772 
7773  return VK_SUCCESS;
7774 }
7775 
7776 void VmaAllocator_T::DestroyPool(VmaPool pool)
7777 {
7778  // Remove from m_Pools.
7779  {
7780  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7781  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
7782  VMA_ASSERT(success && "Pool not found in Allocator.");
7783  }
7784 
7785  vma_delete(this, pool);
7786 }
7787 
7788 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
7789 {
7790  pool->m_BlockVector.GetPoolStats(pPoolStats);
7791 }
7792 
7793 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
7794 {
7795  m_CurrentFrameIndex.store(frameIndex);
7796 }
7797 
7798 void VmaAllocator_T::MakePoolAllocationsLost(
7799  VmaPool hPool,
7800  size_t* pLostAllocationCount)
7801 {
7802  hPool->m_BlockVector.MakePoolAllocationsLost(
7803  m_CurrentFrameIndex.load(),
7804  pLostAllocationCount);
7805 }
7806 
7807 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
7808 {
7809  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
7810  (*pAllocation)->InitLost();
7811 }
7812 
7813 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
7814 {
7815  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
7816 
7817  VkResult res;
7818  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7819  {
7820  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7821  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
7822  {
7823  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
7824  if(res == VK_SUCCESS)
7825  {
7826  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
7827  }
7828  }
7829  else
7830  {
7831  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
7832  }
7833  }
7834  else
7835  {
7836  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
7837  }
7838 
7839  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
7840  {
7841  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
7842  }
7843 
7844  return res;
7845 }
7846 
7847 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
7848 {
7849  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
7850  {
7851  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
7852  }
7853 
7854  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
7855 
7856  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
7857  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7858  {
7859  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7860  m_HeapSizeLimit[heapIndex] += size;
7861  }
7862 }
7863 
7864 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
7865 {
7866  if(hAllocation->CanBecomeLost())
7867  {
7868  return VK_ERROR_MEMORY_MAP_FAILED;
7869  }
7870 
7871  switch(hAllocation->GetType())
7872  {
7873  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7874  {
7875  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
7876  char *pBytes = VMA_NULL;
7877  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
7878  if(res == VK_SUCCESS)
7879  {
7880  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
7881  hAllocation->BlockAllocMap();
7882  }
7883  return res;
7884  }
7885  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7886  return hAllocation->DedicatedAllocMap(this, ppData);
7887  default:
7888  VMA_ASSERT(0);
7889  return VK_ERROR_MEMORY_MAP_FAILED;
7890  }
7891 }
7892 
7893 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
7894 {
7895  switch(hAllocation->GetType())
7896  {
7897  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7898  {
7899  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
7900  hAllocation->BlockAllocUnmap();
7901  pBlock->Unmap(this, 1);
7902  }
7903  break;
7904  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7905  hAllocation->DedicatedAllocUnmap(this);
7906  break;
7907  default:
7908  VMA_ASSERT(0);
7909  }
7910 }
7911 
7912 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
7913 {
7914  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
7915 
7916  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7917  {
7918  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7919  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7920  VMA_ASSERT(pDedicatedAllocations);
7921  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
7922  VMA_ASSERT(success);
7923  }
7924 
7925  VkDeviceMemory hMemory = allocation->GetMemory();
7926 
7927  if(allocation->GetMappedData() != VMA_NULL)
7928  {
7929  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
7930  }
7931 
7932  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
7933 
7934  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
7935 }
7936 
7937 #if VMA_STATS_STRING_ENABLED
7938 
7939 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
7940 {
7941  bool dedicatedAllocationsStarted = false;
7942  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7943  {
7944  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7945  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7946  VMA_ASSERT(pDedicatedAllocVector);
7947  if(pDedicatedAllocVector->empty() == false)
7948  {
7949  if(dedicatedAllocationsStarted == false)
7950  {
7951  dedicatedAllocationsStarted = true;
7952  json.WriteString("DedicatedAllocations");
7953  json.BeginObject();
7954  }
7955 
7956  json.BeginString("Type ");
7957  json.ContinueString(memTypeIndex);
7958  json.EndString();
7959 
7960  json.BeginArray();
7961 
7962  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
7963  {
7964  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
7965  json.BeginObject(true);
7966 
7967  json.WriteString("Type");
7968  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
7969 
7970  json.WriteString("Size");
7971  json.WriteNumber(hAlloc->GetSize());
7972 
7973  const void* pUserData = hAlloc->GetUserData();
7974  if(pUserData != VMA_NULL)
7975  {
7976  json.WriteString("UserData");
7977  if(hAlloc->IsUserDataString())
7978  {
7979  json.WriteString((const char*)pUserData);
7980  }
7981  else
7982  {
7983  json.BeginString();
7984  json.ContinueString_Pointer(pUserData);
7985  json.EndString();
7986  }
7987  }
7988 
7989  json.EndObject();
7990  }
7991 
7992  json.EndArray();
7993  }
7994  }
7995  if(dedicatedAllocationsStarted)
7996  {
7997  json.EndObject();
7998  }
7999 
8000  {
8001  bool allocationsStarted = false;
8002  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8003  {
8004  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
8005  {
8006  if(allocationsStarted == false)
8007  {
8008  allocationsStarted = true;
8009  json.WriteString("DefaultPools");
8010  json.BeginObject();
8011  }
8012 
8013  json.BeginString("Type ");
8014  json.ContinueString(memTypeIndex);
8015  json.EndString();
8016 
8017  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
8018  }
8019  }
8020  if(allocationsStarted)
8021  {
8022  json.EndObject();
8023  }
8024  }
8025 
8026  {
8027  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
8028  const size_t poolCount = m_Pools.size();
8029  if(poolCount > 0)
8030  {
8031  json.WriteString("Pools");
8032  json.BeginArray();
8033  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
8034  {
8035  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
8036  }
8037  json.EndArray();
8038  }
8039  }
8040 }
8041 
8042 #endif // #if VMA_STATS_STRING_ENABLED
8043 
8044 static VkResult AllocateMemoryForImage(
8045  VmaAllocator allocator,
8046  VkImage image,
8047  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8048  VmaSuballocationType suballocType,
8049  VmaAllocation* pAllocation)
8050 {
8051  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
8052 
8053  VkMemoryRequirements vkMemReq = {};
8054  bool requiresDedicatedAllocation = false;
8055  bool prefersDedicatedAllocation = false;
8056  allocator->GetImageMemoryRequirements(image, vkMemReq,
8057  requiresDedicatedAllocation, prefersDedicatedAllocation);
8058 
8059  return allocator->AllocateMemory(
8060  vkMemReq,
8061  requiresDedicatedAllocation,
8062  prefersDedicatedAllocation,
8063  VK_NULL_HANDLE, // dedicatedBuffer
8064  image, // dedicatedImage
8065  *pAllocationCreateInfo,
8066  suballocType,
8067  pAllocation);
8068 }
8069 
8071 // Public interface
8072 
8073 VkResult vmaCreateAllocator(
8074  const VmaAllocatorCreateInfo* pCreateInfo,
8075  VmaAllocator* pAllocator)
8076 {
8077  VMA_ASSERT(pCreateInfo && pAllocator);
8078  VMA_DEBUG_LOG("vmaCreateAllocator");
8079  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
8080  return VK_SUCCESS;
8081 }
8082 
8083 void vmaDestroyAllocator(
8084  VmaAllocator allocator)
8085 {
8086  if(allocator != VK_NULL_HANDLE)
8087  {
8088  VMA_DEBUG_LOG("vmaDestroyAllocator");
8089  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
8090  vma_delete(&allocationCallbacks, allocator);
8091  }
8092 }
8093 
8094 void vmaGetPhysicalDeviceProperties(
8095  VmaAllocator allocator,
8096  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
8097 {
8098  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
8099  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
8100 }
8101 
8102 void vmaGetMemoryProperties(
8103  VmaAllocator allocator,
8104  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
8105 {
8106  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
8107  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
8108 }
8109 
8110 void vmaGetMemoryTypeProperties(
8111  VmaAllocator allocator,
8112  uint32_t memoryTypeIndex,
8113  VkMemoryPropertyFlags* pFlags)
8114 {
8115  VMA_ASSERT(allocator && pFlags);
8116  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
8117  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
8118 }
8119 
8120 void vmaSetCurrentFrameIndex(
8121  VmaAllocator allocator,
8122  uint32_t frameIndex)
8123 {
8124  VMA_ASSERT(allocator);
8125  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
8126 
8127  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8128 
8129  allocator->SetCurrentFrameIndex(frameIndex);
8130 }
8131 
8132 void vmaCalculateStats(
8133  VmaAllocator allocator,
8134  VmaStats* pStats)
8135 {
8136  VMA_ASSERT(allocator && pStats);
8137  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8138  allocator->CalculateStats(pStats);
8139 }
8140 
8141 #if VMA_STATS_STRING_ENABLED
8142 
8143 void vmaBuildStatsString(
8144  VmaAllocator allocator,
8145  char** ppStatsString,
8146  VkBool32 detailedMap)
8147 {
8148  VMA_ASSERT(allocator && ppStatsString);
8149  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8150 
8151  VmaStringBuilder sb(allocator);
8152  {
8153  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
8154  json.BeginObject();
8155 
8156  VmaStats stats;
8157  allocator->CalculateStats(&stats);
8158 
8159  json.WriteString("Total");
8160  VmaPrintStatInfo(json, stats.total);
8161 
8162  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
8163  {
8164  json.BeginString("Heap ");
8165  json.ContinueString(heapIndex);
8166  json.EndString();
8167  json.BeginObject();
8168 
8169  json.WriteString("Size");
8170  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
8171 
8172  json.WriteString("Flags");
8173  json.BeginArray(true);
8174  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
8175  {
8176  json.WriteString("DEVICE_LOCAL");
8177  }
8178  json.EndArray();
8179 
8180  if(stats.memoryHeap[heapIndex].blockCount > 0)
8181  {
8182  json.WriteString("Stats");
8183  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
8184  }
8185 
8186  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
8187  {
8188  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
8189  {
8190  json.BeginString("Type ");
8191  json.ContinueString(typeIndex);
8192  json.EndString();
8193 
8194  json.BeginObject();
8195 
8196  json.WriteString("Flags");
8197  json.BeginArray(true);
8198  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
8199  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
8200  {
8201  json.WriteString("DEVICE_LOCAL");
8202  }
8203  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
8204  {
8205  json.WriteString("HOST_VISIBLE");
8206  }
8207  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
8208  {
8209  json.WriteString("HOST_COHERENT");
8210  }
8211  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
8212  {
8213  json.WriteString("HOST_CACHED");
8214  }
8215  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
8216  {
8217  json.WriteString("LAZILY_ALLOCATED");
8218  }
8219  json.EndArray();
8220 
8221  if(stats.memoryType[typeIndex].blockCount > 0)
8222  {
8223  json.WriteString("Stats");
8224  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
8225  }
8226 
8227  json.EndObject();
8228  }
8229  }
8230 
8231  json.EndObject();
8232  }
8233  if(detailedMap == VK_TRUE)
8234  {
8235  allocator->PrintDetailedMap(json);
8236  }
8237 
8238  json.EndObject();
8239  }
8240 
8241  const size_t len = sb.GetLength();
8242  char* const pChars = vma_new_array(allocator, char, len + 1);
8243  if(len > 0)
8244  {
8245  memcpy(pChars, sb.GetData(), len);
8246  }
8247  pChars[len] = '\0';
8248  *ppStatsString = pChars;
8249 }
8250 
8251 void vmaFreeStatsString(
8252  VmaAllocator allocator,
8253  char* pStatsString)
8254 {
8255  if(pStatsString != VMA_NULL)
8256  {
8257  VMA_ASSERT(allocator);
8258  size_t len = strlen(pStatsString);
8259  vma_delete_array(allocator, pStatsString, len + 1);
8260  }
8261 }
8262 
8263 #endif // #if VMA_STATS_STRING_ENABLED
8264 
8265 /*
8266 This function is not protected by any mutex because it just reads immutable data.
8267 */
8268 VkResult vmaFindMemoryTypeIndex(
8269  VmaAllocator allocator,
8270  uint32_t memoryTypeBits,
8271  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8272  uint32_t* pMemoryTypeIndex)
8273 {
8274  VMA_ASSERT(allocator != VK_NULL_HANDLE);
8275  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8276  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8277 
8278  if(pAllocationCreateInfo->memoryTypeBits != 0)
8279  {
8280  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
8281  }
8282 
8283  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
8284  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
8285 
8286  // Convert usage to requiredFlags and preferredFlags.
8287  switch(pAllocationCreateInfo->usage)
8288  {
8289  case VMA_MEMORY_USAGE_UNKNOWN:
8290  break;
8291  case VMA_MEMORY_USAGE_GPU_ONLY:
8292  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8293  break;
8294  case VMA_MEMORY_USAGE_CPU_ONLY:
8295  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
8296  break;
8297  case VMA_MEMORY_USAGE_CPU_TO_GPU:
8298  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8299  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8300  break;
8301  case VMA_MEMORY_USAGE_GPU_TO_CPU:
8302  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8303  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
8304  break;
8305  default:
8306  break;
8307  }
8308 
8309  *pMemoryTypeIndex = UINT32_MAX;
8310  uint32_t minCost = UINT32_MAX;
8311  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
8312  memTypeIndex < allocator->GetMemoryTypeCount();
8313  ++memTypeIndex, memTypeBit <<= 1)
8314  {
8315  // This memory type is acceptable according to memoryTypeBits bitmask.
8316  if((memTypeBit & memoryTypeBits) != 0)
8317  {
8318  const VkMemoryPropertyFlags currFlags =
8319  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
8320  // This memory type contains requiredFlags.
8321  if((requiredFlags & ~currFlags) == 0)
8322  {
8323  // Calculate cost as number of bits from preferredFlags not present in this memory type.
8324  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
8325  // Remember memory type with lowest cost.
8326  if(currCost < minCost)
8327  {
8328  *pMemoryTypeIndex = memTypeIndex;
8329  if(currCost == 0)
8330  {
8331  return VK_SUCCESS;
8332  }
8333  minCost = currCost;
8334  }
8335  }
8336  }
8337  }
8338  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
8339 }
8340 
8341 VkResult vmaCreatePool(
8342  VmaAllocator allocator,
8343  const VmaPoolCreateInfo* pCreateInfo,
8344  VmaPool* pPool)
8345 {
8346  VMA_ASSERT(allocator && pCreateInfo && pPool);
8347 
8348  VMA_DEBUG_LOG("vmaCreatePool");
8349 
8350  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8351 
8352  return allocator->CreatePool(pCreateInfo, pPool);
8353 }
8354 
8355 void vmaDestroyPool(
8356  VmaAllocator allocator,
8357  VmaPool pool)
8358 {
8359  VMA_ASSERT(allocator);
8360 
8361  if(pool == VK_NULL_HANDLE)
8362  {
8363  return;
8364  }
8365 
8366  VMA_DEBUG_LOG("vmaDestroyPool");
8367 
8368  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8369 
8370  allocator->DestroyPool(pool);
8371 }
8372 
8373 void vmaGetPoolStats(
8374  VmaAllocator allocator,
8375  VmaPool pool,
8376  VmaPoolStats* pPoolStats)
8377 {
8378  VMA_ASSERT(allocator && pool && pPoolStats);
8379 
8380  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8381 
8382  allocator->GetPoolStats(pool, pPoolStats);
8383 }
8384 
8385 void vmaMakePoolAllocationsLost(
8386  VmaAllocator allocator,
8387  VmaPool pool,
8388  size_t* pLostAllocationCount)
8389 {
8390  VMA_ASSERT(allocator && pool);
8391 
8392  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8393 
8394  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
8395 }
8396 
8397 VkResult vmaAllocateMemory(
8398  VmaAllocator allocator,
8399  const VkMemoryRequirements* pVkMemoryRequirements,
8400  const VmaAllocationCreateInfo* pCreateInfo,
8401  VmaAllocation* pAllocation,
8402  VmaAllocationInfo* pAllocationInfo)
8403 {
8404  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
8405 
8406  VMA_DEBUG_LOG("vmaAllocateMemory");
8407 
8408  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8409 
8410  VkResult result = allocator->AllocateMemory(
8411  *pVkMemoryRequirements,
8412  false, // requiresDedicatedAllocation
8413  false, // prefersDedicatedAllocation
8414  VK_NULL_HANDLE, // dedicatedBuffer
8415  VK_NULL_HANDLE, // dedicatedImage
8416  *pCreateInfo,
8417  VMA_SUBALLOCATION_TYPE_UNKNOWN,
8418  pAllocation);
8419 
8420  if(pAllocationInfo && result == VK_SUCCESS)
8421  {
8422  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8423  }
8424 
8425  return result;
8426 }
8427 
8428 VkResult vmaAllocateMemoryForBuffer(
8429  VmaAllocator allocator,
8430  VkBuffer buffer,
8431  const VmaAllocationCreateInfo* pCreateInfo,
8432  VmaAllocation* pAllocation,
8433  VmaAllocationInfo* pAllocationInfo)
8434 {
8435  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8436 
8437  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
8438 
8439  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8440 
8441  VkMemoryRequirements vkMemReq = {};
8442  bool requiresDedicatedAllocation = false;
8443  bool prefersDedicatedAllocation = false;
8444  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
8445  requiresDedicatedAllocation,
8446  prefersDedicatedAllocation);
8447 
8448  VkResult result = allocator->AllocateMemory(
8449  vkMemReq,
8450  requiresDedicatedAllocation,
8451  prefersDedicatedAllocation,
8452  buffer, // dedicatedBuffer
8453  VK_NULL_HANDLE, // dedicatedImage
8454  *pCreateInfo,
8455  VMA_SUBALLOCATION_TYPE_BUFFER,
8456  pAllocation);
8457 
8458  if(pAllocationInfo && result == VK_SUCCESS)
8459  {
8460  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8461  }
8462 
8463  return result;
8464 }
8465 
8466 VkResult vmaAllocateMemoryForImage(
8467  VmaAllocator allocator,
8468  VkImage image,
8469  const VmaAllocationCreateInfo* pCreateInfo,
8470  VmaAllocation* pAllocation,
8471  VmaAllocationInfo* pAllocationInfo)
8472 {
8473  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8474 
8475  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
8476 
8477  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8478 
8479  VkResult result = AllocateMemoryForImage(
8480  allocator,
8481  image,
8482  pCreateInfo,
8483  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
8484  pAllocation);
8485 
8486  if(pAllocationInfo && result == VK_SUCCESS)
8487  {
8488  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8489  }
8490 
8491  return result;
8492 }
8493 
8494 void vmaFreeMemory(
8495  VmaAllocator allocator,
8496  VmaAllocation allocation)
8497 {
8498  VMA_ASSERT(allocator && allocation);
8499 
8500  VMA_DEBUG_LOG("vmaFreeMemory");
8501 
8502  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8503 
8504  allocator->FreeMemory(allocation);
8505 }
8506 
8507 void vmaGetAllocationInfo(
8508  VmaAllocator allocator,
8509  VmaAllocation allocation,
8510  VmaAllocationInfo* pAllocationInfo)
8511 {
8512  VMA_ASSERT(allocator && allocation && pAllocationInfo);
8513 
8514  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8515 
8516  allocator->GetAllocationInfo(allocation, pAllocationInfo);
8517 }
8518 
8519 void vmaSetAllocationUserData(
8520  VmaAllocator allocator,
8521  VmaAllocation allocation,
8522  void* pUserData)
8523 {
8524  VMA_ASSERT(allocator && allocation);
8525 
8526  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8527 
8528  allocation->SetUserData(allocator, pUserData);
8529 }
8530 
8531 void vmaCreateLostAllocation(
8532  VmaAllocator allocator,
8533  VmaAllocation* pAllocation)
8534 {
8535  VMA_ASSERT(allocator && pAllocation);
8536 
8537  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
8538 
8539  allocator->CreateLostAllocation(pAllocation);
8540 }
8541 
8542 VkResult vmaMapMemory(
8543  VmaAllocator allocator,
8544  VmaAllocation allocation,
8545  void** ppData)
8546 {
8547  VMA_ASSERT(allocator && allocation && ppData);
8548 
8549  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8550 
8551  return allocator->Map(allocation, ppData);
8552 }
8553 
8554 void vmaUnmapMemory(
8555  VmaAllocator allocator,
8556  VmaAllocation allocation)
8557 {
8558  VMA_ASSERT(allocator && allocation);
8559 
8560  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8561 
8562  allocator->Unmap(allocation);
8563 }
8564 
8565 VkResult vmaDefragment(
8566  VmaAllocator allocator,
8567  VmaAllocation* pAllocations,
8568  size_t allocationCount,
8569  VkBool32* pAllocationsChanged,
8570  const VmaDefragmentationInfo *pDefragmentationInfo,
8571  VmaDefragmentationStats* pDefragmentationStats)
8572 {
8573  VMA_ASSERT(allocator && pAllocations);
8574 
8575  VMA_DEBUG_LOG("vmaDefragment");
8576 
8577  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8578 
8579  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
8580 }
8581 
8582 VkResult vmaCreateBuffer(
8583  VmaAllocator allocator,
8584  const VkBufferCreateInfo* pBufferCreateInfo,
8585  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8586  VkBuffer* pBuffer,
8587  VmaAllocation* pAllocation,
8588  VmaAllocationInfo* pAllocationInfo)
8589 {
8590  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
8591 
8592  VMA_DEBUG_LOG("vmaCreateBuffer");
8593 
8594  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8595 
8596  *pBuffer = VK_NULL_HANDLE;
8597  *pAllocation = VK_NULL_HANDLE;
8598 
8599  // 1. Create VkBuffer.
8600  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
8601  allocator->m_hDevice,
8602  pBufferCreateInfo,
8603  allocator->GetAllocationCallbacks(),
8604  pBuffer);
8605  if(res >= 0)
8606  {
8607  // 2. vkGetBufferMemoryRequirements.
8608  VkMemoryRequirements vkMemReq = {};
8609  bool requiresDedicatedAllocation = false;
8610  bool prefersDedicatedAllocation = false;
8611  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
8612  requiresDedicatedAllocation, prefersDedicatedAllocation);
8613 
8614  // Make sure alignment requirements for specific buffer usages reported
8615  // in Physical Device Properties are included in alignment reported by memory requirements.
8616  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
8617  {
8618  VMA_ASSERT(vkMemReq.alignment %
8619  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
8620  }
8621  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
8622  {
8623  VMA_ASSERT(vkMemReq.alignment %
8624  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
8625  }
8626  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
8627  {
8628  VMA_ASSERT(vkMemReq.alignment %
8629  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
8630  }
8631 
8632  // 3. Allocate memory using allocator.
8633  res = allocator->AllocateMemory(
8634  vkMemReq,
8635  requiresDedicatedAllocation,
8636  prefersDedicatedAllocation,
8637  *pBuffer, // dedicatedBuffer
8638  VK_NULL_HANDLE, // dedicatedImage
8639  *pAllocationCreateInfo,
8640  VMA_SUBALLOCATION_TYPE_BUFFER,
8641  pAllocation);
8642  if(res >= 0)
8643  {
8644  // 3. Bind buffer with memory.
8645  res = (*allocator->GetVulkanFunctions().vkBindBufferMemory)(
8646  allocator->m_hDevice,
8647  *pBuffer,
8648  (*pAllocation)->GetMemory(),
8649  (*pAllocation)->GetOffset());
8650  if(res >= 0)
8651  {
8652  // All steps succeeded.
8653  if(pAllocationInfo != VMA_NULL)
8654  {
8655  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8656  }
8657  return VK_SUCCESS;
8658  }
8659  allocator->FreeMemory(*pAllocation);
8660  *pAllocation = VK_NULL_HANDLE;
8661  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8662  *pBuffer = VK_NULL_HANDLE;
8663  return res;
8664  }
8665  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8666  *pBuffer = VK_NULL_HANDLE;
8667  return res;
8668  }
8669  return res;
8670 }
8671 
8672 void vmaDestroyBuffer(
8673  VmaAllocator allocator,
8674  VkBuffer buffer,
8675  VmaAllocation allocation)
8676 {
8677  if(buffer != VK_NULL_HANDLE)
8678  {
8679  VMA_ASSERT(allocator);
8680 
8681  VMA_DEBUG_LOG("vmaDestroyBuffer");
8682 
8683  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8684 
8685  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
8686 
8687  allocator->FreeMemory(allocation);
8688  }
8689 }
8690 
8691 VkResult vmaCreateImage(
8692  VmaAllocator allocator,
8693  const VkImageCreateInfo* pImageCreateInfo,
8694  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8695  VkImage* pImage,
8696  VmaAllocation* pAllocation,
8697  VmaAllocationInfo* pAllocationInfo)
8698 {
8699  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
8700 
8701  VMA_DEBUG_LOG("vmaCreateImage");
8702 
8703  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8704 
8705  *pImage = VK_NULL_HANDLE;
8706  *pAllocation = VK_NULL_HANDLE;
8707 
8708  // 1. Create VkImage.
8709  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
8710  allocator->m_hDevice,
8711  pImageCreateInfo,
8712  allocator->GetAllocationCallbacks(),
8713  pImage);
8714  if(res >= 0)
8715  {
8716  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
8717  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
8718  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
8719 
8720  // 2. Allocate memory using allocator.
8721  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
8722  if(res >= 0)
8723  {
8724  // 3. Bind image with memory.
8725  res = (*allocator->GetVulkanFunctions().vkBindImageMemory)(
8726  allocator->m_hDevice,
8727  *pImage,
8728  (*pAllocation)->GetMemory(),
8729  (*pAllocation)->GetOffset());
8730  if(res >= 0)
8731  {
8732  // All steps succeeded.
8733  if(pAllocationInfo != VMA_NULL)
8734  {
8735  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8736  }
8737  return VK_SUCCESS;
8738  }
8739  allocator->FreeMemory(*pAllocation);
8740  *pAllocation = VK_NULL_HANDLE;
8741  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
8742  *pImage = VK_NULL_HANDLE;
8743  return res;
8744  }
8745  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
8746  *pImage = VK_NULL_HANDLE;
8747  return res;
8748  }
8749  return res;
8750 }
8751 
8752 void vmaDestroyImage(
8753  VmaAllocator allocator,
8754  VkImage image,
8755  VmaAllocation allocation)
8756 {
8757  if(image != VK_NULL_HANDLE)
8758  {
8759  VMA_ASSERT(allocator);
8760 
8761  VMA_DEBUG_LOG("vmaDestroyImage");
8762 
8763  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8764 
8765  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
8766 
8767  allocator->FreeMemory(allocation);
8768  }
8769 }
8770 
8771 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:893
+
VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:1147
vmaUnmapMemory
void vmaUnmapMemory(VmaAllocator allocator, VmaAllocation allocation)
Unmaps memory represented by given allocation, mapped previously using vmaMapMemory().
-
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:872
+
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:918
vmaDefragment
VkResult vmaDefragment(VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
Compacts memory by moving allocations.
-
VmaVulkanFunctions::vkCreateBuffer
PFN_vkCreateBuffer vkCreateBuffer
Definition: vk_mem_alloc.h:857
+
VmaVulkanFunctions::vkCreateBuffer
PFN_vkCreateBuffer vkCreateBuffer
Definition: vk_mem_alloc.h:903
vmaFreeStatsString
void vmaFreeStatsString(VmaAllocator allocator, char *pStatsString)
VmaStats
struct VmaStats VmaStats
General statistics from current state of Allocator.
-
VMA_MEMORY_USAGE_GPU_ONLY
Definition: vk_mem_alloc.h:1058
-
VmaVulkanFunctions::vkMapMemory
PFN_vkMapMemory vkMapMemory
Definition: vk_mem_alloc.h:851
-
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:1369
-
VmaAllocatorCreateInfo::flags
VmaAllocatorCreateFlags flags
Flags for created allocator. Use VmaAllocatorCreateFlagBits enum.
Definition: vk_mem_alloc.h:869
-
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:1535
-
VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
Definition: vk_mem_alloc.h:1239
+
VMA_MEMORY_USAGE_GPU_ONLY
Definition: vk_mem_alloc.h:1104
+
VmaVulkanFunctions::vkMapMemory
PFN_vkMapMemory vkMapMemory
Definition: vk_mem_alloc.h:897
+
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:1415
+
VmaAllocatorCreateInfo::flags
VmaAllocatorCreateFlags flags
Flags for created allocator. Use VmaAllocatorCreateFlagBits enum.
Definition: vk_mem_alloc.h:915
+
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:1581
+
VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
Definition: vk_mem_alloc.h:1285
vmaMakePoolAllocationsLost
void vmaMakePoolAllocationsLost(VmaAllocator allocator, VmaPool pool, size_t *pLostAllocationCount)
Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInf...
-
VmaPoolStats::size
VkDeviceSize size
Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
Definition: vk_mem_alloc.h:1293
-
VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
Definition: vk_mem_alloc.h:1138
-
VmaAllocatorCreateFlags
VkFlags VmaAllocatorCreateFlags
Definition: vk_mem_alloc.h:840
-
VmaAllocationCreateInfo::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a memory type chosen for an allocation.
Definition: vk_mem_alloc.h:1176
-
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:1085
-
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom CPU memory allocation callbacks.
Definition: vk_mem_alloc.h:881
+
VmaPoolStats::size
VkDeviceSize size
Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
Definition: vk_mem_alloc.h:1339
+
VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
Definition: vk_mem_alloc.h:1184
+
VmaAllocatorCreateFlags
VkFlags VmaAllocatorCreateFlags
Definition: vk_mem_alloc.h:886
+
VmaAllocationCreateInfo::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a memory type chosen for an allocation.
Definition: vk_mem_alloc.h:1222
+
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:1131
+
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom CPU memory allocation callbacks.
Definition: vk_mem_alloc.h:927
vmaCalculateStats
void vmaCalculateStats(VmaAllocator allocator, VmaStats *pStats)
Retrieves statistics from current state of the Allocator.
-
VmaAllocatorCreateInfo::pVulkanFunctions
const VmaVulkanFunctions * pVulkanFunctions
Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
Definition: vk_mem_alloc.h:934
-
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:866
+
VmaAllocatorCreateInfo::pVulkanFunctions
const VmaVulkanFunctions * pVulkanFunctions
Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
Definition: vk_mem_alloc.h:980
+
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:912
vmaDestroyAllocator
void vmaDestroyAllocator(VmaAllocator allocator)
Destroys allocator object.
-
VmaAllocationCreateFlagBits
VmaAllocationCreateFlagBits
Flags to be passed as VmaAllocationCreateInfo::flags.
Definition: vk_mem_alloc.h:1089
+
VmaAllocationCreateFlagBits
VmaAllocationCreateFlagBits
Flags to be passed as VmaAllocationCreateInfo::flags.
Definition: vk_mem_alloc.h:1135
vmaGetAllocationInfo
void vmaGetAllocationInfo(VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
Returns current information about specified allocation.
-
VmaStatInfo::allocationSizeMax
VkDeviceSize allocationSizeMax
Definition: vk_mem_alloc.h:999
-
VmaVulkanFunctions::vkBindImageMemory
PFN_vkBindImageMemory vkBindImageMemory
Definition: vk_mem_alloc.h:854
-
VmaStatInfo::unusedBytes
VkDeviceSize unusedBytes
Total number of bytes occupied by unused ranges.
Definition: vk_mem_alloc.h:998
-
VmaVulkanFunctions::vkGetImageMemoryRequirements2KHR
PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR
Definition: vk_mem_alloc.h:862
-
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:1539
+
VmaStatInfo::allocationSizeMax
VkDeviceSize allocationSizeMax
Definition: vk_mem_alloc.h:1045
+
VmaVulkanFunctions::vkBindImageMemory
PFN_vkBindImageMemory vkBindImageMemory
Definition: vk_mem_alloc.h:900
+
VmaStatInfo::unusedBytes
VkDeviceSize unusedBytes
Total number of bytes occupied by unused ranges.
Definition: vk_mem_alloc.h:1044
+
VmaVulkanFunctions::vkGetImageMemoryRequirements2KHR
PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR
Definition: vk_mem_alloc.h:908
+
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:1585
vmaFreeMemory
void vmaFreeMemory(VmaAllocator allocator, VmaAllocation allocation)
Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
-
VmaAllocatorCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:898
-
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:1008
-
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:1547
-
VmaAllocationCreateInfo::flags
VmaAllocationCreateFlags flags
Use VmaAllocationCreateFlagBits enum.
Definition: vk_mem_alloc.h:1160
-
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1530
-
VmaVulkanFunctions::vkGetBufferMemoryRequirements
PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
Definition: vk_mem_alloc.h:855
-
PFN_vmaAllocateDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called after successful vkAllocateMemory.
Definition: vk_mem_alloc.h:782
-
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:875
-
VmaPoolCreateInfo
Describes parameter of created VmaPool.
Definition: vk_mem_alloc.h:1247
-
VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1241
-
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:1379
+
VmaAllocatorCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:944
+
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:1054
+
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:1593
+
VmaAllocationCreateInfo::flags
VmaAllocationCreateFlags flags
Use VmaAllocationCreateFlagBits enum.
Definition: vk_mem_alloc.h:1206
+
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1576
+
VmaVulkanFunctions::vkGetBufferMemoryRequirements
PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
Definition: vk_mem_alloc.h:901
+
PFN_vmaAllocateDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called after successful vkAllocateMemory.
Definition: vk_mem_alloc.h:828
+
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:921
+
VmaPoolCreateInfo
Describes parameter of created VmaPool.
Definition: vk_mem_alloc.h:1293
+
VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1287
+
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:1425
vmaGetMemoryTypeProperties
void vmaGetMemoryTypeProperties(VmaAllocator allocator, uint32_t memoryTypeIndex, VkMemoryPropertyFlags *pFlags)
Given Memory Type Index, returns Property Flags of this memory type.
-
VmaVulkanFunctions::vkUnmapMemory
PFN_vkUnmapMemory vkUnmapMemory
Definition: vk_mem_alloc.h:852
-
VmaAllocationCreateInfo::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:1197
-
VmaPoolCreateInfo::minBlockCount
size_t minBlockCount
Minimum number of blocks to be always allocated in this pool, even if they stay empty.
Definition: vk_mem_alloc.h:1263
-
VmaPoolStats::allocationCount
size_t allocationCount
Number of VmaAllocation objects created from this pool that were not destroyed or lost...
Definition: vk_mem_alloc.h:1299
+
VmaVulkanFunctions::vkUnmapMemory
PFN_vkUnmapMemory vkUnmapMemory
Definition: vk_mem_alloc.h:898
+
VmaAllocationCreateInfo::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:1243
+
VmaPoolCreateInfo::minBlockCount
size_t minBlockCount
Minimum number of blocks to be always allocated in this pool, even if they stay empty.
Definition: vk_mem_alloc.h:1309
+
VmaPoolStats::allocationCount
size_t allocationCount
Number of VmaAllocation objects created from this pool that were not destroyed or lost...
Definition: vk_mem_alloc.h:1345
VmaVulkanFunctions
struct VmaVulkanFunctions VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
-
VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:838
-
VmaPoolCreateInfo::memoryTypeIndex
uint32_t memoryTypeIndex
Vulkan memory type index to allocate this pool from.
Definition: vk_mem_alloc.h:1250
+
VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:884
+
VmaPoolCreateInfo::memoryTypeIndex
uint32_t memoryTypeIndex
Vulkan memory type index to allocate this pool from.
Definition: vk_mem_alloc.h:1296
vmaFindMemoryTypeIndex
VkResult vmaFindMemoryTypeIndex(VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
-
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:1036
+
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:1082
VmaAllocationInfo
struct VmaAllocationInfo VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
-
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:1525
+
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:1571
VmaPoolCreateInfo
struct VmaPoolCreateInfo VmaPoolCreateInfo
Describes parameter of created VmaPool.
vmaDestroyPool
void vmaDestroyPool(VmaAllocator allocator, VmaPool pool)
Destroys VmaPool object and frees Vulkan device memory.
-
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:1543
-
VMA_MEMORY_USAGE_CPU_TO_GPU
Definition: vk_mem_alloc.h:1075
-
VmaAllocationCreateInfo::memoryTypeBits
uint32_t memoryTypeBits
Bitmask containing one bit set for every memory type acceptable for this allocation.
Definition: vk_mem_alloc.h:1184
-
VmaVulkanFunctions::vkBindBufferMemory
PFN_vkBindBufferMemory vkBindBufferMemory
Definition: vk_mem_alloc.h:853
+
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:1589
+
VMA_MEMORY_USAGE_CPU_TO_GPU
Definition: vk_mem_alloc.h:1121
+
VmaAllocationCreateInfo::memoryTypeBits
uint32_t memoryTypeBits
Bitmask containing one bit set for every memory type acceptable for this allocation.
Definition: vk_mem_alloc.h:1230
+
VmaVulkanFunctions::vkBindBufferMemory
PFN_vkBindBufferMemory vkBindBufferMemory
Definition: vk_mem_alloc.h:899
vmaGetPoolStats
void vmaGetPoolStats(VmaAllocator allocator, VmaPool pool, VmaPoolStats *pPoolStats)
Retrieves statistics of existing VmaPool object.
VmaDefragmentationInfo
struct VmaDefragmentationInfo VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
-
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:1004
-
PFN_vmaFreeDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called before vkFreeMemory.
Definition: vk_mem_alloc.h:788
+
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:1050
+
PFN_vmaFreeDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called before vkFreeMemory.
Definition: vk_mem_alloc.h:834
vmaSetAllocationUserData
void vmaSetAllocationUserData(VmaAllocator allocator, VmaAllocation allocation, void *pUserData)
Sets pUserData in given allocation to new value.
vmaCreatePool
VkResult vmaCreatePool(VmaAllocator allocator, const VmaPoolCreateInfo *pCreateInfo, VmaPool *pPool)
Allocates Vulkan device memory and creates VmaPool object.
-
VmaAllocatorCreateFlagBits
VmaAllocatorCreateFlagBits
Flags for created VmaAllocator.
Definition: vk_mem_alloc.h:809
+
VmaAllocatorCreateFlagBits
VmaAllocatorCreateFlagBits
Flags for created VmaAllocator.
Definition: vk_mem_alloc.h:855
VmaStatInfo
struct VmaStatInfo VmaStatInfo
Calculated statistics of memory usage in entire allocator.
-
VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
Definition: vk_mem_alloc.h:814
-
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:1545
+
VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
Definition: vk_mem_alloc.h:860
+
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:1591
vmaCreateLostAllocation
void vmaCreateLostAllocation(VmaAllocator allocator, VmaAllocation *pAllocation)
Creates new allocation that is in lost state from the beginning.
-
VmaAllocationCreateInfo::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:1171
-
VmaPoolStats::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Size of the largest continuous free memory region.
Definition: vk_mem_alloc.h:1309
+
VmaAllocationCreateInfo::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:1217
+
VmaPoolStats::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Size of the largest continuous free memory region.
Definition: vk_mem_alloc.h:1355
vmaBuildStatsString
void vmaBuildStatsString(VmaAllocator allocator, char **ppStatsString, VkBool32 detailedMap)
Builds and returns statistics as string in JSON format.
-
VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
Definition: vk_mem_alloc.h:848
-
VmaStatInfo
Calculated statistics of memory usage in entire allocator.
Definition: vk_mem_alloc.h:987
-
VmaPoolCreateInfo::blockSize
VkDeviceSize blockSize
Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
Definition: vk_mem_alloc.h:1258
-
VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
Definition: vk_mem_alloc.h:801
+
VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
Definition: vk_mem_alloc.h:894
+
VmaStatInfo
Calculated statistics of memory usage in entire allocator.
Definition: vk_mem_alloc.h:1033
+
VmaPoolCreateInfo::blockSize
VkDeviceSize blockSize
Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
Definition: vk_mem_alloc.h:1304
+
VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
Definition: vk_mem_alloc.h:847
vmaCreateBuffer
VkResult vmaCreateBuffer(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
-
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
Definition: vk_mem_alloc.h:1145
-
VmaStatInfo::unusedRangeSizeMin
VkDeviceSize unusedRangeSizeMin
Definition: vk_mem_alloc.h:1000
-
VmaDeviceMemoryCallbacks::pfnFree
PFN_vmaFreeDeviceMemoryFunction pfnFree
Optional, can be null.
Definition: vk_mem_alloc.h:805
-
VmaPoolCreateInfo::flags
VmaPoolCreateFlags flags
Use combination of VmaPoolCreateFlagBits.
Definition: vk_mem_alloc.h:1253
-
VMA_MEMORY_USAGE_GPU_TO_CPU
Definition: vk_mem_alloc.h:1084
+
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
Definition: vk_mem_alloc.h:1191
+
VmaStatInfo::unusedRangeSizeMin
VkDeviceSize unusedRangeSizeMin
Definition: vk_mem_alloc.h:1046
+
VmaDeviceMemoryCallbacks::pfnFree
PFN_vmaFreeDeviceMemoryFunction pfnFree
Optional, can be null.
Definition: vk_mem_alloc.h:851
+
VmaPoolCreateInfo::flags
VmaPoolCreateFlags flags
Use combination of VmaPoolCreateFlagBits.
Definition: vk_mem_alloc.h:1299
+
VMA_MEMORY_USAGE_GPU_TO_CPU
Definition: vk_mem_alloc.h:1130
VmaPoolStats
struct VmaPoolStats VmaPoolStats
Describes parameter of existing VmaPool.
vmaCreateImage
VkResult vmaCreateImage(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
Function similar to vmaCreateBuffer().
-
VmaAllocationCreateInfo::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:1166
-
VmaAllocationCreateInfo
Definition: vk_mem_alloc.h:1157
-
VmaStatInfo::blockCount
uint32_t blockCount
Number of VkDeviceMemory Vulkan memory blocks allocated.
Definition: vk_mem_alloc.h:990
-
VmaVulkanFunctions::vkFreeMemory
PFN_vkFreeMemory vkFreeMemory
Definition: vk_mem_alloc.h:850
-
VmaPoolCreateInfo::maxBlockCount
size_t maxBlockCount
Maximum number of blocks that can be allocated in this pool.
Definition: vk_mem_alloc.h:1271
-
VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
Informative callbacks for vkAllocateMemory, vkFreeMemory.
Definition: vk_mem_alloc.h:884
-
VmaPoolStats::unusedRangeCount
size_t unusedRangeCount
Number of continuous memory ranges in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1302
-
VmaAllocationCreateFlags
VkFlags VmaAllocationCreateFlags
Definition: vk_mem_alloc.h:1155
-
VmaAllocationCreateInfo::pool
VmaPool pool
Pool that this allocation should be created in.
Definition: vk_mem_alloc.h:1190
+
VmaAllocationCreateInfo::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:1212
+
VmaAllocationCreateInfo
Definition: vk_mem_alloc.h:1203
+
VmaStatInfo::blockCount
uint32_t blockCount
Number of VkDeviceMemory Vulkan memory blocks allocated.
Definition: vk_mem_alloc.h:1036
+
VmaVulkanFunctions::vkFreeMemory
PFN_vkFreeMemory vkFreeMemory
Definition: vk_mem_alloc.h:896
+
VmaPoolCreateInfo::maxBlockCount
size_t maxBlockCount
Maximum number of blocks that can be allocated in this pool.
Definition: vk_mem_alloc.h:1317
+
VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
Informative callbacks for vkAllocateMemory, vkFreeMemory.
Definition: vk_mem_alloc.h:930
+
VmaPoolStats::unusedRangeCount
size_t unusedRangeCount
Number of continuous memory ranges in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1348
+
VmaAllocationCreateFlags
VkFlags VmaAllocationCreateFlags
Definition: vk_mem_alloc.h:1201
+
VmaAllocationCreateInfo::pool
VmaPool pool
Pool that this allocation should be created in.
Definition: vk_mem_alloc.h:1236
vmaGetMemoryProperties
void vmaGetMemoryProperties(VmaAllocator allocator, const VkPhysicalDeviceMemoryProperties **ppPhysicalDeviceMemoryProperties)
-
VmaAllocatorCreateInfo::pHeapSizeLimit
const VkDeviceSize * pHeapSizeLimit
Either NULL or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
Definition: vk_mem_alloc.h:922
-
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:1006
-
VMA_ALLOCATION_CREATE_MAPPED_BIT
Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
Definition: vk_mem_alloc.h:1125
-
VmaStatInfo::allocationSizeMin
VkDeviceSize allocationSizeMin
Definition: vk_mem_alloc.h:999
-
VmaVulkanFunctions::vkCreateImage
PFN_vkCreateImage vkCreateImage
Definition: vk_mem_alloc.h:859
-
VmaDeviceMemoryCallbacks::pfnAllocate
PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
Optional, can be null.
Definition: vk_mem_alloc.h:803
-
VmaVulkanFunctions::vkDestroyBuffer
PFN_vkDestroyBuffer vkDestroyBuffer
Definition: vk_mem_alloc.h:858
+
VmaAllocatorCreateInfo::pHeapSizeLimit
const VkDeviceSize * pHeapSizeLimit
Either NULL or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
Definition: vk_mem_alloc.h:968
+
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:1052
+
VMA_ALLOCATION_CREATE_MAPPED_BIT
Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
Definition: vk_mem_alloc.h:1171
+
VmaStatInfo::allocationSizeMin
VkDeviceSize allocationSizeMin
Definition: vk_mem_alloc.h:1045
+
VmaVulkanFunctions::vkCreateImage
PFN_vkCreateImage vkCreateImage
Definition: vk_mem_alloc.h:905
+
VmaDeviceMemoryCallbacks::pfnAllocate
PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
Optional, can be null.
Definition: vk_mem_alloc.h:849
+
VmaVulkanFunctions::vkDestroyBuffer
PFN_vkDestroyBuffer vkDestroyBuffer
Definition: vk_mem_alloc.h:904
vmaMapMemory
VkResult vmaMapMemory(VmaAllocator allocator, VmaAllocation allocation, void **ppData)
Maps memory represented by given allocation and returns pointer to it.
-
VmaPoolCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:1285
+
VmaPoolCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:1331
vmaAllocateMemoryForImage
VkResult vmaAllocateMemoryForImage(VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
Function similar to vmaAllocateMemoryForBuffer().
VmaAllocatorCreateInfo
struct VmaAllocatorCreateInfo VmaAllocatorCreateInfo
Description of a Allocator to be created.
-
VmaAllocationInfo::pUserData
void * pUserData
Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
Definition: vk_mem_alloc.h:1393
-
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from large heaps > 1 GiB...
Definition: vk_mem_alloc.h:878
-
VmaStatInfo::allocationSizeAvg
VkDeviceSize allocationSizeAvg
Definition: vk_mem_alloc.h:999
-
VmaStatInfo::usedBytes
VkDeviceSize usedBytes
Total number of bytes occupied by all allocations.
Definition: vk_mem_alloc.h:996
+
VmaAllocationInfo::pUserData
void * pUserData
Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
Definition: vk_mem_alloc.h:1439
+
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from large heaps > 1 GiB...
Definition: vk_mem_alloc.h:924
+
VmaStatInfo::allocationSizeAvg
VkDeviceSize allocationSizeAvg
Definition: vk_mem_alloc.h:1045
+
VmaStatInfo::usedBytes
VkDeviceSize usedBytes
Total number of bytes occupied by all allocations.
Definition: vk_mem_alloc.h:1042
VmaDeviceMemoryCallbacks
struct VmaDeviceMemoryCallbacks VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
-
VmaPoolStats
Describes parameter of existing VmaPool.
Definition: vk_mem_alloc.h:1290
-
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:1374
-
VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1153
-
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1541
-
VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
Definition: vk_mem_alloc.h:846
+
VmaPoolStats
Describes parameter of existing VmaPool.
Definition: vk_mem_alloc.h:1336
+
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:1420
+
VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1199
+
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1587
+
VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
Definition: vk_mem_alloc.h:892
vmaCreateAllocator
VkResult vmaCreateAllocator(const VmaAllocatorCreateInfo *pCreateInfo, VmaAllocator *pAllocator)
Creates Allocator object.
-
VmaVulkanFunctions::vkGetBufferMemoryRequirements2KHR
PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR
Definition: vk_mem_alloc.h:861
-
VmaStatInfo::unusedRangeCount
uint32_t unusedRangeCount
Number of free ranges of memory between allocations.
Definition: vk_mem_alloc.h:994
-
VMA_MEMORY_USAGE_UNKNOWN
Definition: vk_mem_alloc.h:1041
-
VmaPoolCreateFlags
VkFlags VmaPoolCreateFlags
Definition: vk_mem_alloc.h:1243
+
VmaVulkanFunctions::vkGetBufferMemoryRequirements2KHR
PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR
Definition: vk_mem_alloc.h:907
+
VmaStatInfo::unusedRangeCount
uint32_t unusedRangeCount
Number of free ranges of memory between allocations.
Definition: vk_mem_alloc.h:1040
+
VMA_MEMORY_USAGE_UNKNOWN
Definition: vk_mem_alloc.h:1087
+
VmaPoolCreateFlags
VkFlags VmaPoolCreateFlags
Definition: vk_mem_alloc.h:1289
vmaGetPhysicalDeviceProperties
void vmaGetPhysicalDeviceProperties(VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
-
VmaStatInfo::allocationCount
uint32_t allocationCount
Number of VmaAllocation allocation objects allocated.
Definition: vk_mem_alloc.h:992
-
VmaVulkanFunctions::vkGetImageMemoryRequirements
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
Definition: vk_mem_alloc.h:856
-
VmaVulkanFunctions::vkDestroyImage
PFN_vkDestroyImage vkDestroyImage
Definition: vk_mem_alloc.h:860
-
VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:1112
-
VMA_MEMORY_USAGE_CPU_ONLY
Definition: vk_mem_alloc.h:1068
-
VmaAllocationInfo::pMappedData
void * pMappedData
Pointer to the beginning of this allocation as mapped data.
Definition: vk_mem_alloc.h:1388
+
VmaStatInfo::allocationCount
uint32_t allocationCount
Number of VmaAllocation allocation objects allocated.
Definition: vk_mem_alloc.h:1038
+
VmaVulkanFunctions::vkGetImageMemoryRequirements
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
Definition: vk_mem_alloc.h:902
+
VmaVulkanFunctions::vkDestroyImage
PFN_vkDestroyImage vkDestroyImage
Definition: vk_mem_alloc.h:906
+
VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:1158
+
VMA_MEMORY_USAGE_CPU_ONLY
Definition: vk_mem_alloc.h:1114
+
VmaAllocationInfo::pMappedData
void * pMappedData
Pointer to the beginning of this allocation as mapped data.
Definition: vk_mem_alloc.h:1434
vmaDestroyImage
void vmaDestroyImage(VmaAllocator allocator, VkImage image, VmaAllocation allocation)
Destroys Vulkan image and frees allocated memory.
-
VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT
Enables usage of VK_KHR_dedicated_allocation extension.
Definition: vk_mem_alloc.h:836
+
VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT
Enables usage of VK_KHR_dedicated_allocation extension.
Definition: vk_mem_alloc.h:882
VmaDefragmentationStats
struct VmaDefragmentationStats VmaDefragmentationStats
Statistics returned by function vmaDefragment().
-
VmaVulkanFunctions::vkAllocateMemory
PFN_vkAllocateMemory vkAllocateMemory
Definition: vk_mem_alloc.h:849
-
VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
Definition: vk_mem_alloc.h:1355
+
VmaVulkanFunctions::vkAllocateMemory
PFN_vkAllocateMemory vkAllocateMemory
Definition: vk_mem_alloc.h:895
+
VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
Definition: vk_mem_alloc.h:1401
vmaAllocateMemory
VkResult vmaAllocateMemory(VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
General purpose memory allocation.
vmaSetCurrentFrameIndex
void vmaSetCurrentFrameIndex(VmaAllocator allocator, uint32_t frameIndex)
Sets index of the current frame.
VmaAllocationCreateInfo
struct VmaAllocationCreateInfo VmaAllocationCreateInfo
vmaAllocateMemoryForBuffer
VkResult vmaAllocateMemoryForBuffer(VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
-
VmaPoolCreateFlagBits
VmaPoolCreateFlagBits
Flags to be passed as VmaPoolCreateInfo::flags.
Definition: vk_mem_alloc.h:1221
-
VmaStatInfo::unusedRangeSizeAvg
VkDeviceSize unusedRangeSizeAvg
Definition: vk_mem_alloc.h:1000
-
VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT
Definition: vk_mem_alloc.h:1151
-
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:1007
+
VmaPoolCreateFlagBits
VmaPoolCreateFlagBits
Flags to be passed as VmaPoolCreateInfo::flags.
Definition: vk_mem_alloc.h:1267
+
VmaStatInfo::unusedRangeSizeAvg
VkDeviceSize unusedRangeSizeAvg
Definition: vk_mem_alloc.h:1046
+
VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT
Definition: vk_mem_alloc.h:1197
+
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:1053
vmaDestroyBuffer
void vmaDestroyBuffer(VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation)
Destroys Vulkan buffer and frees allocated memory.
-
VmaPoolStats::unusedSize
VkDeviceSize unusedSize
Total number of bytes in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1296
-
VmaStatInfo::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Definition: vk_mem_alloc.h:1000
-
VmaAllocationInfo::memoryType
uint32_t memoryType
Memory type index that this allocation was allocated from.
Definition: vk_mem_alloc.h:1360
+
VmaPoolStats::unusedSize
VkDeviceSize unusedSize
Total number of bytes in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1342
+
VmaStatInfo::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Definition: vk_mem_alloc.h:1046
+
VmaAllocationInfo::memoryType
uint32_t memoryType
Memory type index that this allocation was allocated from.
Definition: vk_mem_alloc.h:1406
diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 2691439..1205376 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -353,17 +353,63 @@ Also, Windows drivers from all 3 PC GPU vendors (AMD, Intel, NVIDIA) currently provide `VK_MEMORY_PROPERTY_HOST_COHERENT_BIT` flag on all memory types that are `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT`, so on this platform you may not need to bother. +\section memory_mapping_finding_if_memory_mappable Finding out if memory is mappable + +It may happen that your allocation ends up in memory that is `HOST_VISIBLE` (available for mapping) +despite it wasn't explicitly requested. +For example, application may work on integrated graphics with unified memory (like Intel) or +allocation from video memory might have failed, so the library chose system memory as fallback. + +You can detect this case and map such allocation to access its memory on CPU directly, +instead of launching a transfer operation. +You can even use `VMA_ALLOCATION_CREATE_MAPPED_BIT` flag while creating allocations +that are not necessarily `HOST_VISIBLE` (e.g. using `VMA_MEMORY_USAGE_GPU_ONLY`). +If the allocation ends up in memory type that is `HOST_VISIBLE`, it will be persistently mapped and you can use it directly. +If not, the flag is just ignored. +Example: + +\code +VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; +bufCreateInfo.size = sizeof(ConstantBuffer); +bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + +VmaAllocationCreateInfo allocCreateInfo = {}; +allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; +allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; + +VkBuffer buf; +VmaAllocation alloc; +VmaAllocationInfo allocInfo; +vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); + +if(allocInfo.pUserData != nullptr) +{ + // Allocation ended up in mappable memory. + // It's persistently mapped. You can access it directly. + memcpy(allocInfo.pMappedData, &constantBufferData, sizeof(constantBufferData)); +} +else +{ + // Allocation ended up in non-mappable memory. + // You need to create CPU-side copy in VMA_MEMORY_USAGE_CPU_ONLY and make a transfer. +} +\endcode + \page custom_memory_pools Custom memory pools -The library automatically creates and manages default memory pool for each -memory type available on the device. A pool contains a number of -`VkDeviceMemory` blocks. You can create custom pool and allocate memory out of -it. It can be useful if you want to: +A memory pool contains a number of `VkDeviceMemory` blocks. +The library automatically creates and manages default pool for each memory type available on the device. +Default memory pool automatically grows in size. +Size of allocated blocks is also variable and managed automatically. + +You can create custom pool and allocate memory out of it. +It can be useful if you want to: - Keep certain kind of allocations separate from others. -- Enforce particular size of Vulkan memory blocks. +- Enforce particular, fixed size of Vulkan memory blocks. - Limit maximum amount of Vulkan memory allocated for that pool. +- Reserve minimum or fixed amount of Vulkan memory always preallocated for that pool. To use custom memory pools: @@ -375,7 +421,7 @@ To use custom memory pools: Example: \code -// Create a pool that could have at most 2 blocks, 128 MiB each. +// Create a pool that can have at most 2 blocks, 128 MiB each. VmaPoolCreateInfo poolCreateInfo = {}; poolCreateInfo.memoryTypeIndex = ... poolCreateInfo.blockSize = 128ull * 1024 * 1024; @@ -1156,7 +1202,7 @@ typedef VkFlags VmaAllocationCreateFlags; typedef struct VmaAllocationCreateInfo { - /// Use VmaAllocationCreateFlagBits enum. + /// Use #VmaAllocationCreateFlagBits enum. VmaAllocationCreateFlags flags; /** \brief Intended usage of memory. @@ -1612,16 +1658,22 @@ for(size_t i = 0; i < allocations.size(); ++i) } \endcode -Warning! This function is not correct according to Vulkan specification. Use it -at your own risk. That's becuase Vulkan doesn't guarantee that memory +Note: Please don't expect memory to be fully compacted after this call. +Algorithms inside are based on some heuristics that try to maximize number of Vulkan +memory blocks to make totally empty to release them, as well as to maximimze continuous +empty space inside remaining blocks, while minimizing the number and size of data that +needs to be moved. Some fragmentation still remains after this call. This is normal. + +Warning: This function is not 100% correct according to Vulkan specification. Use it +at your own risk. That's because Vulkan doesn't guarantee that memory requirements (size and alignment) for a new buffer or image are consistent. They may be different even for subsequent calls with the same parameters. It really does happen on some platforms, especially with images. -This function may be time-consuming, so you shouldn't call it too often (like -every frame or after every resource creation/destruction), but rater you can -call it on special occasions (like when reloading a game level, when you just -destroyed a lot of objects). +Warning: This function may be time-consuming, so you shouldn't call it too often +(like every frame or after every resource creation/destruction). +You can call it on special occasions (like when reloading a game level or +when you just destroyed a lot of objects). */ VkResult vmaDefragment( VmaAllocator allocator,