diff --git a/docs/html/lost_allocations.html b/docs/html/lost_allocations.html index 6c3dee3..d9b48c6 100644 --- a/docs/html/lost_allocations.html +++ b/docs/html/lost_allocations.html @@ -77,7 +77,9 @@ $(function() {

Q: How do you inform the library when new frame starts?

You need to call function vmaSetCurrentFrameIndex().

Example code:

-
struct MyBuffer
{
VkBuffer m_Buf = nullptr;
VmaAllocation m_Alloc = nullptr;
// Called when the buffer is really needed in the current frame.
void EnsureBuffer();
};
void MyBuffer::EnsureBuffer()
{
// Buffer has been created.
if(m_Buf != VK_NULL_HANDLE)
{
// Check if its allocation is not lost + mark it as used in current frame.
if(vmaTouchAllocation(allocator, m_Alloc))
{
// It's all OK - safe to use m_Buf.
return;
}
}
// Buffer not yet exists or lost - destroy and recreate it.
vmaDestroyBuffer(allocator, m_Buf, m_Alloc);
VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
bufCreateInfo.size = 1024;
bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VmaAllocationCreateInfo allocCreateInfo = {};
allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
allocCreateInfo.flags = VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT |
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT;
vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &m_Buf, &m_Alloc, nullptr);
}

When using lost allocations, you may see some Vulkan validation layer warnings about overlapping regions of memory bound to different kinds of buffers and images. This is still valid as long as you implement proper handling of lost allocations (like in the example above) and don't use them.

+
struct MyBuffer
{
VkBuffer m_Buf = nullptr;
VmaAllocation m_Alloc = nullptr;
// Called when the buffer is really needed in the current frame.
void EnsureBuffer();
};
void MyBuffer::EnsureBuffer()
{
// Buffer has been created.
if(m_Buf != VK_NULL_HANDLE)
{
// Check if its allocation is not lost + mark it as used in current frame.
if(vmaTouchAllocation(allocator, m_Alloc))
{
// It's all OK - safe to use m_Buf.
return;
}
}
// Buffer not yet exists or lost - destroy and recreate it.
vmaDestroyBuffer(allocator, m_Buf, m_Alloc);
VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
bufCreateInfo.size = 1024;
bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VmaAllocationCreateInfo allocCreateInfo = {};
allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
allocCreateInfo.flags = VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT |
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT;
vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &m_Buf, &m_Alloc, nullptr);
}

When using lost allocations, you may see some Vulkan validation layer warnings about overlapping regions of memory bound to different kinds of buffers and images. This is still valid as long as you implement proper handling of lost allocations (like in the example above) and don't use them.

+

You can create an allocation that is already in lost state from the beginning using function vmaCreateLostAllocation(). It may be useful if you need a "dummy" allocation that is not null.

+

You can call function vmaMakePoolAllocationsLost() to set all eligible allocations in a specified custom pool to lost state. Allocations that have been "touched" in current frame or VmaPoolCreateInfo::frameInUseCount frames back cannot become lost.

diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index 5167773..7ce271b 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,157 +62,157 @@ $(function() {
vk_mem_alloc.h
-Go to the documentation of this file.
1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
935 #include <vulkan/vulkan.h>
936 
937 VK_DEFINE_HANDLE(VmaAllocator)
938 
939 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
941  VmaAllocator allocator,
942  uint32_t memoryType,
943  VkDeviceMemory memory,
944  VkDeviceSize size);
946 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
947  VmaAllocator allocator,
948  uint32_t memoryType,
949  VkDeviceMemory memory,
950  VkDeviceSize size);
951 
959 typedef struct VmaDeviceMemoryCallbacks {
961  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
963  PFN_vmaFreeDeviceMemoryFunction pfnFree;
964 } VmaDeviceMemoryCallbacks;
965 
967 typedef enum VmaAllocatorCreateFlagBits {
972  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
994  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
995 
996  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
997 } VmaAllocatorCreateFlagBits;
998 typedef VkFlags VmaAllocatorCreateFlags;
999 
1004 typedef struct VmaVulkanFunctions {
1005  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
1006  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
1007  PFN_vkAllocateMemory vkAllocateMemory;
1008  PFN_vkFreeMemory vkFreeMemory;
1009  PFN_vkMapMemory vkMapMemory;
1010  PFN_vkUnmapMemory vkUnmapMemory;
1011  PFN_vkBindBufferMemory vkBindBufferMemory;
1012  PFN_vkBindImageMemory vkBindImageMemory;
1013  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
1014  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
1015  PFN_vkCreateBuffer vkCreateBuffer;
1016  PFN_vkDestroyBuffer vkDestroyBuffer;
1017  PFN_vkCreateImage vkCreateImage;
1018  PFN_vkDestroyImage vkDestroyImage;
1019  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
1020  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
1021 } VmaVulkanFunctions;
1022 
1024 typedef struct VmaAllocatorCreateInfo
1025 {
1027  VmaAllocatorCreateFlags flags;
1029 
1030  VkPhysicalDevice physicalDevice;
1032 
1033  VkDevice device;
1035 
1036  VkDeviceSize preferredLargeHeapBlockSize;
1038 
1039  const VkAllocationCallbacks* pAllocationCallbacks;
1041 
1042  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
1056  uint32_t frameInUseCount;
1080  const VkDeviceSize* pHeapSizeLimit;
1092  const VmaVulkanFunctions* pVulkanFunctions;
1093 } VmaAllocatorCreateInfo;
1094 
1096 VkResult vmaCreateAllocator(
1097  const VmaAllocatorCreateInfo* pCreateInfo,
1098  VmaAllocator* pAllocator);
1099 
1101 void vmaDestroyAllocator(
1102  VmaAllocator allocator);
1103 
1108 void vmaGetPhysicalDeviceProperties(
1109  VmaAllocator allocator,
1110  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
1111 
1116 void vmaGetMemoryProperties(
1117  VmaAllocator allocator,
1118  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
1119 
1126 void vmaGetMemoryTypeProperties(
1127  VmaAllocator allocator,
1128  uint32_t memoryTypeIndex,
1129  VkMemoryPropertyFlags* pFlags);
1130 
1139 void vmaSetCurrentFrameIndex(
1140  VmaAllocator allocator,
1141  uint32_t frameIndex);
1142 
1145 typedef struct VmaStatInfo
1146 {
1148  uint32_t blockCount;
1150  uint32_t allocationCount;
1152  uint32_t unusedRangeCount;
1154  VkDeviceSize usedBytes;
1156  VkDeviceSize unusedBytes;
1157  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1158  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1159 } VmaStatInfo;
1160 
1162 typedef struct VmaStats
1163 {
1164  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1165  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1166  VmaStatInfo total;
1167 } VmaStats;
1168 
1170 void vmaCalculateStats(
1171  VmaAllocator allocator,
1172  VmaStats* pStats);
1173 
1174 #define VMA_STATS_STRING_ENABLED 1
1175 
1176 #if VMA_STATS_STRING_ENABLED
1177 
1179 
1181 void vmaBuildStatsString(
1182  VmaAllocator allocator,
1183  char** ppStatsString,
1184  VkBool32 detailedMap);
1185 
1186 void vmaFreeStatsString(
1187  VmaAllocator allocator,
1188  char* pStatsString);
1189 
1190 #endif // #if VMA_STATS_STRING_ENABLED
1191 
1192 VK_DEFINE_HANDLE(VmaPool)
1193 
1194 typedef enum VmaMemoryUsage
1195 {
1199  VMA_MEMORY_USAGE_UNKNOWN = 0,
1216  VMA_MEMORY_USAGE_GPU_ONLY = 1,
1226  VMA_MEMORY_USAGE_CPU_ONLY = 2,
1233  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1242  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1243  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1244 } VmaMemoryUsage;
1245 
1247 typedef enum VmaAllocationCreateFlagBits {
1259  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1260 
1270  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1283  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1296  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1303  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1309  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1310 
1311  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1312 } VmaAllocationCreateFlagBits;
1313 typedef VkFlags VmaAllocationCreateFlags;
1314 
1315 typedef struct VmaAllocationCreateInfo
1316 {
1318  VmaAllocationCreateFlags flags;
1324  VmaMemoryUsage usage;
1329  VkMemoryPropertyFlags requiredFlags;
1334  VkMemoryPropertyFlags preferredFlags;
1342  uint32_t memoryTypeBits;
1348  VmaPool pool;
1355  void* pUserData;
1356 } VmaAllocationCreateInfo;
1357 
1374 VkResult vmaFindMemoryTypeIndex(
1375  VmaAllocator allocator,
1376  uint32_t memoryTypeBits,
1377  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1378  uint32_t* pMemoryTypeIndex);
1379 
1392 VkResult vmaFindMemoryTypeIndexForBufferInfo(
1393  VmaAllocator allocator,
1394  const VkBufferCreateInfo* pBufferCreateInfo,
1395  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1396  uint32_t* pMemoryTypeIndex);
1397 
1410 VkResult vmaFindMemoryTypeIndexForImageInfo(
1411  VmaAllocator allocator,
1412  const VkImageCreateInfo* pImageCreateInfo,
1413  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1414  uint32_t* pMemoryTypeIndex);
1415 
1417 typedef enum VmaPoolCreateFlagBits {
1435  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1436 
1437  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1438 } VmaPoolCreateFlagBits;
1439 typedef VkFlags VmaPoolCreateFlags;
1440 
1443 typedef struct VmaPoolCreateInfo {
1446  uint32_t memoryTypeIndex;
1449  VmaPoolCreateFlags flags;
1454  VkDeviceSize blockSize;
1459  size_t minBlockCount;
1467  size_t maxBlockCount;
1481  uint32_t frameInUseCount;
1482 } VmaPoolCreateInfo;
1483 
1486 typedef struct VmaPoolStats {
1489  VkDeviceSize size;
1492  VkDeviceSize unusedSize;
1495  size_t allocationCount;
1498  size_t unusedRangeCount;
1505  VkDeviceSize unusedRangeSizeMax;
1506 } VmaPoolStats;
1507 
1514 VkResult vmaCreatePool(
1515  VmaAllocator allocator,
1516  const VmaPoolCreateInfo* pCreateInfo,
1517  VmaPool* pPool);
1518 
1521 void vmaDestroyPool(
1522  VmaAllocator allocator,
1523  VmaPool pool);
1524 
1531 void vmaGetPoolStats(
1532  VmaAllocator allocator,
1533  VmaPool pool,
1534  VmaPoolStats* pPoolStats);
1535 
1542 void vmaMakePoolAllocationsLost(
1543  VmaAllocator allocator,
1544  VmaPool pool,
1545  size_t* pLostAllocationCount);
1546 
1547 VK_DEFINE_HANDLE(VmaAllocation)
1548 
1549 
1551 typedef struct VmaAllocationInfo {
1556  uint32_t memoryType;
1565  VkDeviceMemory deviceMemory;
1570  VkDeviceSize offset;
1575  VkDeviceSize size;
1584  void* pMappedData;
1589  void* pUserData;
1590 } VmaAllocationInfo;
1591 
1602 VkResult vmaAllocateMemory(
1603  VmaAllocator allocator,
1604  const VkMemoryRequirements* pVkMemoryRequirements,
1605  const VmaAllocationCreateInfo* pCreateInfo,
1606  VmaAllocation* pAllocation,
1607  VmaAllocationInfo* pAllocationInfo);
1608 
1615 VkResult vmaAllocateMemoryForBuffer(
1616  VmaAllocator allocator,
1617  VkBuffer buffer,
1618  const VmaAllocationCreateInfo* pCreateInfo,
1619  VmaAllocation* pAllocation,
1620  VmaAllocationInfo* pAllocationInfo);
1621 
1623 VkResult vmaAllocateMemoryForImage(
1624  VmaAllocator allocator,
1625  VkImage image,
1626  const VmaAllocationCreateInfo* pCreateInfo,
1627  VmaAllocation* pAllocation,
1628  VmaAllocationInfo* pAllocationInfo);
1629 
1631 void vmaFreeMemory(
1632  VmaAllocator allocator,
1633  VmaAllocation allocation);
1634 
1651 void vmaGetAllocationInfo(
1652  VmaAllocator allocator,
1653  VmaAllocation allocation,
1654  VmaAllocationInfo* pAllocationInfo);
1655 
1670 VkBool32 vmaTouchAllocation(
1671  VmaAllocator allocator,
1672  VmaAllocation allocation);
1673 
1687 void vmaSetAllocationUserData(
1688  VmaAllocator allocator,
1689  VmaAllocation allocation,
1690  void* pUserData);
1691 
1702 void vmaCreateLostAllocation(
1703  VmaAllocator allocator,
1704  VmaAllocation* pAllocation);
1705 
1740 VkResult vmaMapMemory(
1741  VmaAllocator allocator,
1742  VmaAllocation allocation,
1743  void** ppData);
1744 
1749 void vmaUnmapMemory(
1750  VmaAllocator allocator,
1751  VmaAllocation allocation);
1752 
1754 typedef struct VmaDefragmentationInfo {
1759  VkDeviceSize maxBytesToMove;
1764  uint32_t maxAllocationsToMove;
1765 } VmaDefragmentationInfo;
1766 
1768 typedef struct VmaDefragmentationStats {
1770  VkDeviceSize bytesMoved;
1772  VkDeviceSize bytesFreed;
1774  uint32_t allocationsMoved;
1776  uint32_t deviceMemoryBlocksFreed;
1777 } VmaDefragmentationStats;
1778 
1861 VkResult vmaDefragment(
1862  VmaAllocator allocator,
1863  VmaAllocation* pAllocations,
1864  size_t allocationCount,
1865  VkBool32* pAllocationsChanged,
1866  const VmaDefragmentationInfo *pDefragmentationInfo,
1867  VmaDefragmentationStats* pDefragmentationStats);
1868 
1895 VkResult vmaCreateBuffer(
1896  VmaAllocator allocator,
1897  const VkBufferCreateInfo* pBufferCreateInfo,
1898  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1899  VkBuffer* pBuffer,
1900  VmaAllocation* pAllocation,
1901  VmaAllocationInfo* pAllocationInfo);
1902 
1914 void vmaDestroyBuffer(
1915  VmaAllocator allocator,
1916  VkBuffer buffer,
1917  VmaAllocation allocation);
1918 
1920 VkResult vmaCreateImage(
1921  VmaAllocator allocator,
1922  const VkImageCreateInfo* pImageCreateInfo,
1923  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1924  VkImage* pImage,
1925  VmaAllocation* pAllocation,
1926  VmaAllocationInfo* pAllocationInfo);
1927 
1939 void vmaDestroyImage(
1940  VmaAllocator allocator,
1941  VkImage image,
1942  VmaAllocation allocation);
1943 
1944 #ifdef __cplusplus
1945 }
1946 #endif
1947 
1948 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
1949 
1950 // For Visual Studio IntelliSense.
1951 #ifdef __INTELLISENSE__
1952 #define VMA_IMPLEMENTATION
1953 #endif
1954 
1955 #ifdef VMA_IMPLEMENTATION
1956 #undef VMA_IMPLEMENTATION
1957 
1958 #include <cstdint>
1959 #include <cstdlib>
1960 #include <cstring>
1961 
1962 /*******************************************************************************
1963 CONFIGURATION SECTION
1964 
1965 Define some of these macros before each #include of this header or change them
1966 here if you need other then default behavior depending on your environment.
1967 */
1968 
1969 /*
1970 Define this macro to 1 to make the library fetch pointers to Vulkan functions
1971 internally, like:
1972 
1973  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1974 
1975 Define to 0 if you are going to provide you own pointers to Vulkan functions via
1976 VmaAllocatorCreateInfo::pVulkanFunctions.
1977 */
1978 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
1979 #define VMA_STATIC_VULKAN_FUNCTIONS 1
1980 #endif
1981 
1982 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
1983 //#define VMA_USE_STL_CONTAINERS 1
1984 
1985 /* Set this macro to 1 to make the library including and using STL containers:
1986 std::pair, std::vector, std::list, std::unordered_map.
1987 
1988 Set it to 0 or undefined to make the library using its own implementation of
1989 the containers.
1990 */
1991 #if VMA_USE_STL_CONTAINERS
1992  #define VMA_USE_STL_VECTOR 1
1993  #define VMA_USE_STL_UNORDERED_MAP 1
1994  #define VMA_USE_STL_LIST 1
1995 #endif
1996 
1997 #if VMA_USE_STL_VECTOR
1998  #include <vector>
1999 #endif
2000 
2001 #if VMA_USE_STL_UNORDERED_MAP
2002  #include <unordered_map>
2003 #endif
2004 
2005 #if VMA_USE_STL_LIST
2006  #include <list>
2007 #endif
2008 
2009 /*
2010 Following headers are used in this CONFIGURATION section only, so feel free to
2011 remove them if not needed.
2012 */
2013 #include <cassert> // for assert
2014 #include <algorithm> // for min, max
2015 #include <mutex> // for std::mutex
2016 #include <atomic> // for std::atomic
2017 
2018 #if !defined(_WIN32) && !defined(__APPLE__)
2019  #include <malloc.h> // for aligned_alloc()
2020 #endif
2021 
2022 #ifndef VMA_NULL
2023  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
2024  #define VMA_NULL nullptr
2025 #endif
2026 
2027 #if defined(__APPLE__) || defined(__ANDROID__)
2028 #include <cstdlib>
2029 void *aligned_alloc(size_t alignment, size_t size)
2030 {
2031  // alignment must be >= sizeof(void*)
2032  if(alignment < sizeof(void*))
2033  {
2034  alignment = sizeof(void*);
2035  }
2036 
2037  void *pointer;
2038  if(posix_memalign(&pointer, alignment, size) == 0)
2039  return pointer;
2040  return VMA_NULL;
2041 }
2042 #endif
2043 
2044 // Normal assert to check for programmer's errors, especially in Debug configuration.
2045 #ifndef VMA_ASSERT
2046  #ifdef _DEBUG
2047  #define VMA_ASSERT(expr) assert(expr)
2048  #else
2049  #define VMA_ASSERT(expr)
2050  #endif
2051 #endif
2052 
2053 // Assert that will be called very often, like inside data structures e.g. operator[].
2054 // Making it non-empty can make program slow.
2055 #ifndef VMA_HEAVY_ASSERT
2056  #ifdef _DEBUG
2057  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
2058  #else
2059  #define VMA_HEAVY_ASSERT(expr)
2060  #endif
2061 #endif
2062 
2063 #ifndef VMA_ALIGN_OF
2064  #define VMA_ALIGN_OF(type) (__alignof(type))
2065 #endif
2066 
2067 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
2068  #if defined(_WIN32)
2069  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
2070  #else
2071  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
2072  #endif
2073 #endif
2074 
2075 #ifndef VMA_SYSTEM_FREE
2076  #if defined(_WIN32)
2077  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
2078  #else
2079  #define VMA_SYSTEM_FREE(ptr) free(ptr)
2080  #endif
2081 #endif
2082 
2083 #ifndef VMA_MIN
2084  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
2085 #endif
2086 
2087 #ifndef VMA_MAX
2088  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
2089 #endif
2090 
2091 #ifndef VMA_SWAP
2092  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
2093 #endif
2094 
2095 #ifndef VMA_SORT
2096  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
2097 #endif
2098 
2099 #ifndef VMA_DEBUG_LOG
2100  #define VMA_DEBUG_LOG(format, ...)
2101  /*
2102  #define VMA_DEBUG_LOG(format, ...) do { \
2103  printf(format, __VA_ARGS__); \
2104  printf("\n"); \
2105  } while(false)
2106  */
2107 #endif
2108 
2109 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
2110 #if VMA_STATS_STRING_ENABLED
2111  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
2112  {
2113  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
2114  }
2115  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
2116  {
2117  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
2118  }
2119  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
2120  {
2121  snprintf(outStr, strLen, "%p", ptr);
2122  }
2123 #endif
2124 
2125 #ifndef VMA_MUTEX
2126  class VmaMutex
2127  {
2128  public:
2129  VmaMutex() { }
2130  ~VmaMutex() { }
2131  void Lock() { m_Mutex.lock(); }
2132  void Unlock() { m_Mutex.unlock(); }
2133  private:
2134  std::mutex m_Mutex;
2135  };
2136  #define VMA_MUTEX VmaMutex
2137 #endif
2138 
2139 /*
2140 If providing your own implementation, you need to implement a subset of std::atomic:
2141 
2142 - Constructor(uint32_t desired)
2143 - uint32_t load() const
2144 - void store(uint32_t desired)
2145 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
2146 */
2147 #ifndef VMA_ATOMIC_UINT32
2148  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
2149 #endif
2150 
2151 #ifndef VMA_BEST_FIT
2152 
2164  #define VMA_BEST_FIT (1)
2165 #endif
2166 
2167 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
2168 
2172  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
2173 #endif
2174 
2175 #ifndef VMA_DEBUG_ALIGNMENT
2176 
2180  #define VMA_DEBUG_ALIGNMENT (1)
2181 #endif
2182 
2183 #ifndef VMA_DEBUG_MARGIN
2184 
2188  #define VMA_DEBUG_MARGIN (0)
2189 #endif
2190 
2191 #ifndef VMA_DEBUG_GLOBAL_MUTEX
2192 
2196  #define VMA_DEBUG_GLOBAL_MUTEX (0)
2197 #endif
2198 
2199 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
2200 
2204  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
2205 #endif
2206 
2207 #ifndef VMA_SMALL_HEAP_MAX_SIZE
2208  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
2210 #endif
2211 
2212 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
2213  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
2215 #endif
2216 
2217 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
2218 
2219 /*******************************************************************************
2220 END OF CONFIGURATION
2221 */
2222 
2223 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
2224  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
2225 
2226 // Returns number of bits set to 1 in (v).
2227 static inline uint32_t VmaCountBitsSet(uint32_t v)
2228 {
2229  uint32_t c = v - ((v >> 1) & 0x55555555);
2230  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
2231  c = ((c >> 4) + c) & 0x0F0F0F0F;
2232  c = ((c >> 8) + c) & 0x00FF00FF;
2233  c = ((c >> 16) + c) & 0x0000FFFF;
2234  return c;
2235 }
2236 
2237 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2238 // Use types like uint32_t, uint64_t as T.
2239 template <typename T>
2240 static inline T VmaAlignUp(T val, T align)
2241 {
2242  return (val + align - 1) / align * align;
2243 }
2244 
2245 // Division with mathematical rounding to nearest number.
2246 template <typename T>
2247 inline T VmaRoundDiv(T x, T y)
2248 {
2249  return (x + (y / (T)2)) / y;
2250 }
2251 
2252 #ifndef VMA_SORT
2253 
2254 template<typename Iterator, typename Compare>
2255 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2256 {
2257  Iterator centerValue = end; --centerValue;
2258  Iterator insertIndex = beg;
2259  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2260  {
2261  if(cmp(*memTypeIndex, *centerValue))
2262  {
2263  if(insertIndex != memTypeIndex)
2264  {
2265  VMA_SWAP(*memTypeIndex, *insertIndex);
2266  }
2267  ++insertIndex;
2268  }
2269  }
2270  if(insertIndex != centerValue)
2271  {
2272  VMA_SWAP(*insertIndex, *centerValue);
2273  }
2274  return insertIndex;
2275 }
2276 
2277 template<typename Iterator, typename Compare>
2278 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2279 {
2280  if(beg < end)
2281  {
2282  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2283  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2284  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2285  }
2286 }
2287 
2288 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2289 
2290 #endif // #ifndef VMA_SORT
2291 
2292 /*
2293 Returns true if two memory blocks occupy overlapping pages.
2294 ResourceA must be in less memory offset than ResourceB.
2295 
2296 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2297 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2298 */
2299 static inline bool VmaBlocksOnSamePage(
2300  VkDeviceSize resourceAOffset,
2301  VkDeviceSize resourceASize,
2302  VkDeviceSize resourceBOffset,
2303  VkDeviceSize pageSize)
2304 {
2305  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2306  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2307  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2308  VkDeviceSize resourceBStart = resourceBOffset;
2309  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2310  return resourceAEndPage == resourceBStartPage;
2311 }
2312 
2313 enum VmaSuballocationType
2314 {
2315  VMA_SUBALLOCATION_TYPE_FREE = 0,
2316  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2317  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2318  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2319  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2320  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2321  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2322 };
2323 
2324 /*
2325 Returns true if given suballocation types could conflict and must respect
2326 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2327 or linear image and another one is optimal image. If type is unknown, behave
2328 conservatively.
2329 */
2330 static inline bool VmaIsBufferImageGranularityConflict(
2331  VmaSuballocationType suballocType1,
2332  VmaSuballocationType suballocType2)
2333 {
2334  if(suballocType1 > suballocType2)
2335  {
2336  VMA_SWAP(suballocType1, suballocType2);
2337  }
2338 
2339  switch(suballocType1)
2340  {
2341  case VMA_SUBALLOCATION_TYPE_FREE:
2342  return false;
2343  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
2344  return true;
2345  case VMA_SUBALLOCATION_TYPE_BUFFER:
2346  return
2347  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2348  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2349  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
2350  return
2351  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2352  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
2353  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2354  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
2355  return
2356  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2357  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
2358  return false;
2359  default:
2360  VMA_ASSERT(0);
2361  return true;
2362  }
2363 }
2364 
2365 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
2366 struct VmaMutexLock
2367 {
2368 public:
2369  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
2370  m_pMutex(useMutex ? &mutex : VMA_NULL)
2371  {
2372  if(m_pMutex)
2373  {
2374  m_pMutex->Lock();
2375  }
2376  }
2377 
2378  ~VmaMutexLock()
2379  {
2380  if(m_pMutex)
2381  {
2382  m_pMutex->Unlock();
2383  }
2384  }
2385 
2386 private:
2387  VMA_MUTEX* m_pMutex;
2388 };
2389 
2390 #if VMA_DEBUG_GLOBAL_MUTEX
2391  static VMA_MUTEX gDebugGlobalMutex;
2392  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
2393 #else
2394  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
2395 #endif
2396 
2397 // Minimum size of a free suballocation to register it in the free suballocation collection.
2398 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
2399 
2400 /*
2401 Performs binary search and returns iterator to first element that is greater or
2402 equal to (key), according to comparison (cmp).
2403 
2404 Cmp should return true if first argument is less than second argument.
2405 
2406 Returned value is the found element, if present in the collection or place where
2407 new element with value (key) should be inserted.
2408 */
2409 template <typename IterT, typename KeyT, typename CmpT>
2410 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
2411 {
2412  size_t down = 0, up = (end - beg);
2413  while(down < up)
2414  {
2415  const size_t mid = (down + up) / 2;
2416  if(cmp(*(beg+mid), key))
2417  {
2418  down = mid + 1;
2419  }
2420  else
2421  {
2422  up = mid;
2423  }
2424  }
2425  return beg + down;
2426 }
2427 
2429 // Memory allocation
2430 
2431 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
2432 {
2433  if((pAllocationCallbacks != VMA_NULL) &&
2434  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
2435  {
2436  return (*pAllocationCallbacks->pfnAllocation)(
2437  pAllocationCallbacks->pUserData,
2438  size,
2439  alignment,
2440  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2441  }
2442  else
2443  {
2444  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
2445  }
2446 }
2447 
2448 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
2449 {
2450  if((pAllocationCallbacks != VMA_NULL) &&
2451  (pAllocationCallbacks->pfnFree != VMA_NULL))
2452  {
2453  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
2454  }
2455  else
2456  {
2457  VMA_SYSTEM_FREE(ptr);
2458  }
2459 }
2460 
2461 template<typename T>
2462 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
2463 {
2464  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
2465 }
2466 
2467 template<typename T>
2468 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
2469 {
2470  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
2471 }
2472 
2473 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
2474 
2475 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
2476 
2477 template<typename T>
2478 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
2479 {
2480  ptr->~T();
2481  VmaFree(pAllocationCallbacks, ptr);
2482 }
2483 
2484 template<typename T>
2485 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
2486 {
2487  if(ptr != VMA_NULL)
2488  {
2489  for(size_t i = count; i--; )
2490  {
2491  ptr[i].~T();
2492  }
2493  VmaFree(pAllocationCallbacks, ptr);
2494  }
2495 }
2496 
2497 // STL-compatible allocator.
2498 template<typename T>
2499 class VmaStlAllocator
2500 {
2501 public:
2502  const VkAllocationCallbacks* const m_pCallbacks;
2503  typedef T value_type;
2504 
2505  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
2506  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
2507 
2508  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
2509  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
2510 
2511  template<typename U>
2512  bool operator==(const VmaStlAllocator<U>& rhs) const
2513  {
2514  return m_pCallbacks == rhs.m_pCallbacks;
2515  }
2516  template<typename U>
2517  bool operator!=(const VmaStlAllocator<U>& rhs) const
2518  {
2519  return m_pCallbacks != rhs.m_pCallbacks;
2520  }
2521 
2522  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
2523 };
2524 
2525 #if VMA_USE_STL_VECTOR
2526 
2527 #define VmaVector std::vector
2528 
2529 template<typename T, typename allocatorT>
2530 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
2531 {
2532  vec.insert(vec.begin() + index, item);
2533 }
2534 
2535 template<typename T, typename allocatorT>
2536 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
2537 {
2538  vec.erase(vec.begin() + index);
2539 }
2540 
2541 #else // #if VMA_USE_STL_VECTOR
2542 
2543 /* Class with interface compatible with subset of std::vector.
2544 T must be POD because constructors and destructors are not called and memcpy is
2545 used for these objects. */
2546 template<typename T, typename AllocatorT>
2547 class VmaVector
2548 {
2549 public:
2550  typedef T value_type;
2551 
2552  VmaVector(const AllocatorT& allocator) :
2553  m_Allocator(allocator),
2554  m_pArray(VMA_NULL),
2555  m_Count(0),
2556  m_Capacity(0)
2557  {
2558  }
2559 
2560  VmaVector(size_t count, const AllocatorT& allocator) :
2561  m_Allocator(allocator),
2562  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
2563  m_Count(count),
2564  m_Capacity(count)
2565  {
2566  }
2567 
2568  VmaVector(const VmaVector<T, AllocatorT>& src) :
2569  m_Allocator(src.m_Allocator),
2570  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
2571  m_Count(src.m_Count),
2572  m_Capacity(src.m_Count)
2573  {
2574  if(m_Count != 0)
2575  {
2576  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
2577  }
2578  }
2579 
2580  ~VmaVector()
2581  {
2582  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2583  }
2584 
2585  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
2586  {
2587  if(&rhs != this)
2588  {
2589  resize(rhs.m_Count);
2590  if(m_Count != 0)
2591  {
2592  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
2593  }
2594  }
2595  return *this;
2596  }
2597 
2598  bool empty() const { return m_Count == 0; }
2599  size_t size() const { return m_Count; }
2600  T* data() { return m_pArray; }
2601  const T* data() const { return m_pArray; }
2602 
2603  T& operator[](size_t index)
2604  {
2605  VMA_HEAVY_ASSERT(index < m_Count);
2606  return m_pArray[index];
2607  }
2608  const T& operator[](size_t index) const
2609  {
2610  VMA_HEAVY_ASSERT(index < m_Count);
2611  return m_pArray[index];
2612  }
2613 
2614  T& front()
2615  {
2616  VMA_HEAVY_ASSERT(m_Count > 0);
2617  return m_pArray[0];
2618  }
2619  const T& front() const
2620  {
2621  VMA_HEAVY_ASSERT(m_Count > 0);
2622  return m_pArray[0];
2623  }
2624  T& back()
2625  {
2626  VMA_HEAVY_ASSERT(m_Count > 0);
2627  return m_pArray[m_Count - 1];
2628  }
2629  const T& back() const
2630  {
2631  VMA_HEAVY_ASSERT(m_Count > 0);
2632  return m_pArray[m_Count - 1];
2633  }
2634 
2635  void reserve(size_t newCapacity, bool freeMemory = false)
2636  {
2637  newCapacity = VMA_MAX(newCapacity, m_Count);
2638 
2639  if((newCapacity < m_Capacity) && !freeMemory)
2640  {
2641  newCapacity = m_Capacity;
2642  }
2643 
2644  if(newCapacity != m_Capacity)
2645  {
2646  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
2647  if(m_Count != 0)
2648  {
2649  memcpy(newArray, m_pArray, m_Count * sizeof(T));
2650  }
2651  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2652  m_Capacity = newCapacity;
2653  m_pArray = newArray;
2654  }
2655  }
2656 
2657  void resize(size_t newCount, bool freeMemory = false)
2658  {
2659  size_t newCapacity = m_Capacity;
2660  if(newCount > m_Capacity)
2661  {
2662  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
2663  }
2664  else if(freeMemory)
2665  {
2666  newCapacity = newCount;
2667  }
2668 
2669  if(newCapacity != m_Capacity)
2670  {
2671  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
2672  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
2673  if(elementsToCopy != 0)
2674  {
2675  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
2676  }
2677  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2678  m_Capacity = newCapacity;
2679  m_pArray = newArray;
2680  }
2681 
2682  m_Count = newCount;
2683  }
2684 
2685  void clear(bool freeMemory = false)
2686  {
2687  resize(0, freeMemory);
2688  }
2689 
2690  void insert(size_t index, const T& src)
2691  {
2692  VMA_HEAVY_ASSERT(index <= m_Count);
2693  const size_t oldCount = size();
2694  resize(oldCount + 1);
2695  if(index < oldCount)
2696  {
2697  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
2698  }
2699  m_pArray[index] = src;
2700  }
2701 
2702  void remove(size_t index)
2703  {
2704  VMA_HEAVY_ASSERT(index < m_Count);
2705  const size_t oldCount = size();
2706  if(index < oldCount - 1)
2707  {
2708  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
2709  }
2710  resize(oldCount - 1);
2711  }
2712 
2713  void push_back(const T& src)
2714  {
2715  const size_t newIndex = size();
2716  resize(newIndex + 1);
2717  m_pArray[newIndex] = src;
2718  }
2719 
2720  void pop_back()
2721  {
2722  VMA_HEAVY_ASSERT(m_Count > 0);
2723  resize(size() - 1);
2724  }
2725 
2726  void push_front(const T& src)
2727  {
2728  insert(0, src);
2729  }
2730 
2731  void pop_front()
2732  {
2733  VMA_HEAVY_ASSERT(m_Count > 0);
2734  remove(0);
2735  }
2736 
2737  typedef T* iterator;
2738 
2739  iterator begin() { return m_pArray; }
2740  iterator end() { return m_pArray + m_Count; }
2741 
2742 private:
2743  AllocatorT m_Allocator;
2744  T* m_pArray;
2745  size_t m_Count;
2746  size_t m_Capacity;
2747 };
2748 
2749 template<typename T, typename allocatorT>
2750 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2751 {
2752  vec.insert(index, item);
2753 }
2754 
2755 template<typename T, typename allocatorT>
2756 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2757 {
2758  vec.remove(index);
2759 }
2760 
2761 #endif // #if VMA_USE_STL_VECTOR
2762 
2763 template<typename CmpLess, typename VectorT>
2764 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2765 {
2766  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2767  vector.data(),
2768  vector.data() + vector.size(),
2769  value,
2770  CmpLess()) - vector.data();
2771  VmaVectorInsert(vector, indexToInsert, value);
2772  return indexToInsert;
2773 }
2774 
2775 template<typename CmpLess, typename VectorT>
2776 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2777 {
2778  CmpLess comparator;
2779  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2780  vector.begin(),
2781  vector.end(),
2782  value,
2783  comparator);
2784  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
2785  {
2786  size_t indexToRemove = it - vector.begin();
2787  VmaVectorRemove(vector, indexToRemove);
2788  return true;
2789  }
2790  return false;
2791 }
2792 
2793 template<typename CmpLess, typename VectorT>
2794 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
2795 {
2796  CmpLess comparator;
2797  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2798  vector.data(),
2799  vector.data() + vector.size(),
2800  value,
2801  comparator);
2802  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
2803  {
2804  return it - vector.begin();
2805  }
2806  else
2807  {
2808  return vector.size();
2809  }
2810 }
2811 
2813 // class VmaPoolAllocator
2814 
2815 /*
2816 Allocator for objects of type T using a list of arrays (pools) to speed up
2817 allocation. Number of elements that can be allocated is not bounded because
2818 allocator can create multiple blocks.
2819 */
2820 template<typename T>
2821 class VmaPoolAllocator
2822 {
2823 public:
2824  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
2825  ~VmaPoolAllocator();
2826  void Clear();
2827  T* Alloc();
2828  void Free(T* ptr);
2829 
2830 private:
2831  union Item
2832  {
2833  uint32_t NextFreeIndex;
2834  T Value;
2835  };
2836 
2837  struct ItemBlock
2838  {
2839  Item* pItems;
2840  uint32_t FirstFreeIndex;
2841  };
2842 
2843  const VkAllocationCallbacks* m_pAllocationCallbacks;
2844  size_t m_ItemsPerBlock;
2845  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
2846 
2847  ItemBlock& CreateNewBlock();
2848 };
2849 
2850 template<typename T>
2851 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
2852  m_pAllocationCallbacks(pAllocationCallbacks),
2853  m_ItemsPerBlock(itemsPerBlock),
2854  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
2855 {
2856  VMA_ASSERT(itemsPerBlock > 0);
2857 }
2858 
2859 template<typename T>
2860 VmaPoolAllocator<T>::~VmaPoolAllocator()
2861 {
2862  Clear();
2863 }
2864 
2865 template<typename T>
2866 void VmaPoolAllocator<T>::Clear()
2867 {
2868  for(size_t i = m_ItemBlocks.size(); i--; )
2869  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
2870  m_ItemBlocks.clear();
2871 }
2872 
2873 template<typename T>
2874 T* VmaPoolAllocator<T>::Alloc()
2875 {
2876  for(size_t i = m_ItemBlocks.size(); i--; )
2877  {
2878  ItemBlock& block = m_ItemBlocks[i];
2879  // This block has some free items: Use first one.
2880  if(block.FirstFreeIndex != UINT32_MAX)
2881  {
2882  Item* const pItem = &block.pItems[block.FirstFreeIndex];
2883  block.FirstFreeIndex = pItem->NextFreeIndex;
2884  return &pItem->Value;
2885  }
2886  }
2887 
2888  // No block has free item: Create new one and use it.
2889  ItemBlock& newBlock = CreateNewBlock();
2890  Item* const pItem = &newBlock.pItems[0];
2891  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
2892  return &pItem->Value;
2893 }
2894 
2895 template<typename T>
2896 void VmaPoolAllocator<T>::Free(T* ptr)
2897 {
2898  // Search all memory blocks to find ptr.
2899  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
2900  {
2901  ItemBlock& block = m_ItemBlocks[i];
2902 
2903  // Casting to union.
2904  Item* pItemPtr;
2905  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
2906 
2907  // Check if pItemPtr is in address range of this block.
2908  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
2909  {
2910  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
2911  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
2912  block.FirstFreeIndex = index;
2913  return;
2914  }
2915  }
2916  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
2917 }
2918 
2919 template<typename T>
2920 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
2921 {
2922  ItemBlock newBlock = {
2923  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
2924 
2925  m_ItemBlocks.push_back(newBlock);
2926 
2927  // Setup singly-linked list of all free items in this block.
2928  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
2929  newBlock.pItems[i].NextFreeIndex = i + 1;
2930  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
2931  return m_ItemBlocks.back();
2932 }
2933 
2935 // class VmaRawList, VmaList
2936 
2937 #if VMA_USE_STL_LIST
2938 
2939 #define VmaList std::list
2940 
2941 #else // #if VMA_USE_STL_LIST
2942 
2943 template<typename T>
2944 struct VmaListItem
2945 {
2946  VmaListItem* pPrev;
2947  VmaListItem* pNext;
2948  T Value;
2949 };
2950 
2951 // Doubly linked list.
2952 template<typename T>
2953 class VmaRawList
2954 {
2955 public:
2956  typedef VmaListItem<T> ItemType;
2957 
2958  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
2959  ~VmaRawList();
2960  void Clear();
2961 
2962  size_t GetCount() const { return m_Count; }
2963  bool IsEmpty() const { return m_Count == 0; }
2964 
2965  ItemType* Front() { return m_pFront; }
2966  const ItemType* Front() const { return m_pFront; }
2967  ItemType* Back() { return m_pBack; }
2968  const ItemType* Back() const { return m_pBack; }
2969 
2970  ItemType* PushBack();
2971  ItemType* PushFront();
2972  ItemType* PushBack(const T& value);
2973  ItemType* PushFront(const T& value);
2974  void PopBack();
2975  void PopFront();
2976 
2977  // Item can be null - it means PushBack.
2978  ItemType* InsertBefore(ItemType* pItem);
2979  // Item can be null - it means PushFront.
2980  ItemType* InsertAfter(ItemType* pItem);
2981 
2982  ItemType* InsertBefore(ItemType* pItem, const T& value);
2983  ItemType* InsertAfter(ItemType* pItem, const T& value);
2984 
2985  void Remove(ItemType* pItem);
2986 
2987 private:
2988  const VkAllocationCallbacks* const m_pAllocationCallbacks;
2989  VmaPoolAllocator<ItemType> m_ItemAllocator;
2990  ItemType* m_pFront;
2991  ItemType* m_pBack;
2992  size_t m_Count;
2993 
2994  // Declared not defined, to block copy constructor and assignment operator.
2995  VmaRawList(const VmaRawList<T>& src);
2996  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
2997 };
2998 
2999 template<typename T>
3000 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
3001  m_pAllocationCallbacks(pAllocationCallbacks),
3002  m_ItemAllocator(pAllocationCallbacks, 128),
3003  m_pFront(VMA_NULL),
3004  m_pBack(VMA_NULL),
3005  m_Count(0)
3006 {
3007 }
3008 
3009 template<typename T>
3010 VmaRawList<T>::~VmaRawList()
3011 {
3012  // Intentionally not calling Clear, because that would be unnecessary
3013  // computations to return all items to m_ItemAllocator as free.
3014 }
3015 
3016 template<typename T>
3017 void VmaRawList<T>::Clear()
3018 {
3019  if(IsEmpty() == false)
3020  {
3021  ItemType* pItem = m_pBack;
3022  while(pItem != VMA_NULL)
3023  {
3024  ItemType* const pPrevItem = pItem->pPrev;
3025  m_ItemAllocator.Free(pItem);
3026  pItem = pPrevItem;
3027  }
3028  m_pFront = VMA_NULL;
3029  m_pBack = VMA_NULL;
3030  m_Count = 0;
3031  }
3032 }
3033 
3034 template<typename T>
3035 VmaListItem<T>* VmaRawList<T>::PushBack()
3036 {
3037  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3038  pNewItem->pNext = VMA_NULL;
3039  if(IsEmpty())
3040  {
3041  pNewItem->pPrev = VMA_NULL;
3042  m_pFront = pNewItem;
3043  m_pBack = pNewItem;
3044  m_Count = 1;
3045  }
3046  else
3047  {
3048  pNewItem->pPrev = m_pBack;
3049  m_pBack->pNext = pNewItem;
3050  m_pBack = pNewItem;
3051  ++m_Count;
3052  }
3053  return pNewItem;
3054 }
3055 
3056 template<typename T>
3057 VmaListItem<T>* VmaRawList<T>::PushFront()
3058 {
3059  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3060  pNewItem->pPrev = VMA_NULL;
3061  if(IsEmpty())
3062  {
3063  pNewItem->pNext = VMA_NULL;
3064  m_pFront = pNewItem;
3065  m_pBack = pNewItem;
3066  m_Count = 1;
3067  }
3068  else
3069  {
3070  pNewItem->pNext = m_pFront;
3071  m_pFront->pPrev = pNewItem;
3072  m_pFront = pNewItem;
3073  ++m_Count;
3074  }
3075  return pNewItem;
3076 }
3077 
3078 template<typename T>
3079 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
3080 {
3081  ItemType* const pNewItem = PushBack();
3082  pNewItem->Value = value;
3083  return pNewItem;
3084 }
3085 
3086 template<typename T>
3087 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
3088 {
3089  ItemType* const pNewItem = PushFront();
3090  pNewItem->Value = value;
3091  return pNewItem;
3092 }
3093 
3094 template<typename T>
3095 void VmaRawList<T>::PopBack()
3096 {
3097  VMA_HEAVY_ASSERT(m_Count > 0);
3098  ItemType* const pBackItem = m_pBack;
3099  ItemType* const pPrevItem = pBackItem->pPrev;
3100  if(pPrevItem != VMA_NULL)
3101  {
3102  pPrevItem->pNext = VMA_NULL;
3103  }
3104  m_pBack = pPrevItem;
3105  m_ItemAllocator.Free(pBackItem);
3106  --m_Count;
3107 }
3108 
3109 template<typename T>
3110 void VmaRawList<T>::PopFront()
3111 {
3112  VMA_HEAVY_ASSERT(m_Count > 0);
3113  ItemType* const pFrontItem = m_pFront;
3114  ItemType* const pNextItem = pFrontItem->pNext;
3115  if(pNextItem != VMA_NULL)
3116  {
3117  pNextItem->pPrev = VMA_NULL;
3118  }
3119  m_pFront = pNextItem;
3120  m_ItemAllocator.Free(pFrontItem);
3121  --m_Count;
3122 }
3123 
3124 template<typename T>
3125 void VmaRawList<T>::Remove(ItemType* pItem)
3126 {
3127  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
3128  VMA_HEAVY_ASSERT(m_Count > 0);
3129 
3130  if(pItem->pPrev != VMA_NULL)
3131  {
3132  pItem->pPrev->pNext = pItem->pNext;
3133  }
3134  else
3135  {
3136  VMA_HEAVY_ASSERT(m_pFront == pItem);
3137  m_pFront = pItem->pNext;
3138  }
3139 
3140  if(pItem->pNext != VMA_NULL)
3141  {
3142  pItem->pNext->pPrev = pItem->pPrev;
3143  }
3144  else
3145  {
3146  VMA_HEAVY_ASSERT(m_pBack == pItem);
3147  m_pBack = pItem->pPrev;
3148  }
3149 
3150  m_ItemAllocator.Free(pItem);
3151  --m_Count;
3152 }
3153 
3154 template<typename T>
3155 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
3156 {
3157  if(pItem != VMA_NULL)
3158  {
3159  ItemType* const prevItem = pItem->pPrev;
3160  ItemType* const newItem = m_ItemAllocator.Alloc();
3161  newItem->pPrev = prevItem;
3162  newItem->pNext = pItem;
3163  pItem->pPrev = newItem;
3164  if(prevItem != VMA_NULL)
3165  {
3166  prevItem->pNext = newItem;
3167  }
3168  else
3169  {
3170  VMA_HEAVY_ASSERT(m_pFront == pItem);
3171  m_pFront = newItem;
3172  }
3173  ++m_Count;
3174  return newItem;
3175  }
3176  else
3177  return PushBack();
3178 }
3179 
3180 template<typename T>
3181 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
3182 {
3183  if(pItem != VMA_NULL)
3184  {
3185  ItemType* const nextItem = pItem->pNext;
3186  ItemType* const newItem = m_ItemAllocator.Alloc();
3187  newItem->pNext = nextItem;
3188  newItem->pPrev = pItem;
3189  pItem->pNext = newItem;
3190  if(nextItem != VMA_NULL)
3191  {
3192  nextItem->pPrev = newItem;
3193  }
3194  else
3195  {
3196  VMA_HEAVY_ASSERT(m_pBack == pItem);
3197  m_pBack = newItem;
3198  }
3199  ++m_Count;
3200  return newItem;
3201  }
3202  else
3203  return PushFront();
3204 }
3205 
3206 template<typename T>
3207 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
3208 {
3209  ItemType* const newItem = InsertBefore(pItem);
3210  newItem->Value = value;
3211  return newItem;
3212 }
3213 
3214 template<typename T>
3215 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
3216 {
3217  ItemType* const newItem = InsertAfter(pItem);
3218  newItem->Value = value;
3219  return newItem;
3220 }
3221 
3222 template<typename T, typename AllocatorT>
3223 class VmaList
3224 {
3225 public:
3226  class iterator
3227  {
3228  public:
3229  iterator() :
3230  m_pList(VMA_NULL),
3231  m_pItem(VMA_NULL)
3232  {
3233  }
3234 
3235  T& operator*() const
3236  {
3237  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3238  return m_pItem->Value;
3239  }
3240  T* operator->() const
3241  {
3242  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3243  return &m_pItem->Value;
3244  }
3245 
3246  iterator& operator++()
3247  {
3248  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3249  m_pItem = m_pItem->pNext;
3250  return *this;
3251  }
3252  iterator& operator--()
3253  {
3254  if(m_pItem != VMA_NULL)
3255  {
3256  m_pItem = m_pItem->pPrev;
3257  }
3258  else
3259  {
3260  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3261  m_pItem = m_pList->Back();
3262  }
3263  return *this;
3264  }
3265 
3266  iterator operator++(int)
3267  {
3268  iterator result = *this;
3269  ++*this;
3270  return result;
3271  }
3272  iterator operator--(int)
3273  {
3274  iterator result = *this;
3275  --*this;
3276  return result;
3277  }
3278 
3279  bool operator==(const iterator& rhs) const
3280  {
3281  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3282  return m_pItem == rhs.m_pItem;
3283  }
3284  bool operator!=(const iterator& rhs) const
3285  {
3286  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3287  return m_pItem != rhs.m_pItem;
3288  }
3289 
3290  private:
3291  VmaRawList<T>* m_pList;
3292  VmaListItem<T>* m_pItem;
3293 
3294  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3295  m_pList(pList),
3296  m_pItem(pItem)
3297  {
3298  }
3299 
3300  friend class VmaList<T, AllocatorT>;
3301  };
3302 
3303  class const_iterator
3304  {
3305  public:
3306  const_iterator() :
3307  m_pList(VMA_NULL),
3308  m_pItem(VMA_NULL)
3309  {
3310  }
3311 
3312  const_iterator(const iterator& src) :
3313  m_pList(src.m_pList),
3314  m_pItem(src.m_pItem)
3315  {
3316  }
3317 
3318  const T& operator*() const
3319  {
3320  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3321  return m_pItem->Value;
3322  }
3323  const T* operator->() const
3324  {
3325  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3326  return &m_pItem->Value;
3327  }
3328 
3329  const_iterator& operator++()
3330  {
3331  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3332  m_pItem = m_pItem->pNext;
3333  return *this;
3334  }
3335  const_iterator& operator--()
3336  {
3337  if(m_pItem != VMA_NULL)
3338  {
3339  m_pItem = m_pItem->pPrev;
3340  }
3341  else
3342  {
3343  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3344  m_pItem = m_pList->Back();
3345  }
3346  return *this;
3347  }
3348 
3349  const_iterator operator++(int)
3350  {
3351  const_iterator result = *this;
3352  ++*this;
3353  return result;
3354  }
3355  const_iterator operator--(int)
3356  {
3357  const_iterator result = *this;
3358  --*this;
3359  return result;
3360  }
3361 
3362  bool operator==(const const_iterator& rhs) const
3363  {
3364  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3365  return m_pItem == rhs.m_pItem;
3366  }
3367  bool operator!=(const const_iterator& rhs) const
3368  {
3369  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3370  return m_pItem != rhs.m_pItem;
3371  }
3372 
3373  private:
3374  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
3375  m_pList(pList),
3376  m_pItem(pItem)
3377  {
3378  }
3379 
3380  const VmaRawList<T>* m_pList;
3381  const VmaListItem<T>* m_pItem;
3382 
3383  friend class VmaList<T, AllocatorT>;
3384  };
3385 
3386  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
3387 
3388  bool empty() const { return m_RawList.IsEmpty(); }
3389  size_t size() const { return m_RawList.GetCount(); }
3390 
3391  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
3392  iterator end() { return iterator(&m_RawList, VMA_NULL); }
3393 
3394  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
3395  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
3396 
3397  void clear() { m_RawList.Clear(); }
3398  void push_back(const T& value) { m_RawList.PushBack(value); }
3399  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
3400  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
3401 
3402 private:
3403  VmaRawList<T> m_RawList;
3404 };
3405 
3406 #endif // #if VMA_USE_STL_LIST
3407 
3409 // class VmaMap
3410 
3411 // Unused in this version.
3412 #if 0
3413 
3414 #if VMA_USE_STL_UNORDERED_MAP
3415 
3416 #define VmaPair std::pair
3417 
3418 #define VMA_MAP_TYPE(KeyT, ValueT) \
3419  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
3420 
3421 #else // #if VMA_USE_STL_UNORDERED_MAP
3422 
3423 template<typename T1, typename T2>
3424 struct VmaPair
3425 {
3426  T1 first;
3427  T2 second;
3428 
3429  VmaPair() : first(), second() { }
3430  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
3431 };
3432 
3433 /* Class compatible with subset of interface of std::unordered_map.
3434 KeyT, ValueT must be POD because they will be stored in VmaVector.
3435 */
3436 template<typename KeyT, typename ValueT>
3437 class VmaMap
3438 {
3439 public:
3440  typedef VmaPair<KeyT, ValueT> PairType;
3441  typedef PairType* iterator;
3442 
3443  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
3444 
3445  iterator begin() { return m_Vector.begin(); }
3446  iterator end() { return m_Vector.end(); }
3447 
3448  void insert(const PairType& pair);
3449  iterator find(const KeyT& key);
3450  void erase(iterator it);
3451 
3452 private:
3453  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
3454 };
3455 
3456 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
3457 
3458 template<typename FirstT, typename SecondT>
3459 struct VmaPairFirstLess
3460 {
3461  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
3462  {
3463  return lhs.first < rhs.first;
3464  }
3465  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
3466  {
3467  return lhs.first < rhsFirst;
3468  }
3469 };
3470 
3471 template<typename KeyT, typename ValueT>
3472 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
3473 {
3474  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3475  m_Vector.data(),
3476  m_Vector.data() + m_Vector.size(),
3477  pair,
3478  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
3479  VmaVectorInsert(m_Vector, indexToInsert, pair);
3480 }
3481 
3482 template<typename KeyT, typename ValueT>
3483 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
3484 {
3485  PairType* it = VmaBinaryFindFirstNotLess(
3486  m_Vector.data(),
3487  m_Vector.data() + m_Vector.size(),
3488  key,
3489  VmaPairFirstLess<KeyT, ValueT>());
3490  if((it != m_Vector.end()) && (it->first == key))
3491  {
3492  return it;
3493  }
3494  else
3495  {
3496  return m_Vector.end();
3497  }
3498 }
3499 
3500 template<typename KeyT, typename ValueT>
3501 void VmaMap<KeyT, ValueT>::erase(iterator it)
3502 {
3503  VmaVectorRemove(m_Vector, it - m_Vector.begin());
3504 }
3505 
3506 #endif // #if VMA_USE_STL_UNORDERED_MAP
3507 
3508 #endif // #if 0
3509 
3511 
3512 class VmaDeviceMemoryBlock;
3513 
3514 struct VmaAllocation_T
3515 {
3516 private:
3517  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
3518 
3519  enum FLAGS
3520  {
3521  FLAG_USER_DATA_STRING = 0x01,
3522  };
3523 
3524 public:
3525  enum ALLOCATION_TYPE
3526  {
3527  ALLOCATION_TYPE_NONE,
3528  ALLOCATION_TYPE_BLOCK,
3529  ALLOCATION_TYPE_DEDICATED,
3530  };
3531 
3532  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
3533  m_Alignment(1),
3534  m_Size(0),
3535  m_pUserData(VMA_NULL),
3536  m_LastUseFrameIndex(currentFrameIndex),
3537  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
3538  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
3539  m_MapCount(0),
3540  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
3541  {
3542  }
3543 
3544  ~VmaAllocation_T()
3545  {
3546  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
3547 
3548  // Check if owned string was freed.
3549  VMA_ASSERT(m_pUserData == VMA_NULL);
3550  }
3551 
3552  void InitBlockAllocation(
3553  VmaPool hPool,
3554  VmaDeviceMemoryBlock* block,
3555  VkDeviceSize offset,
3556  VkDeviceSize alignment,
3557  VkDeviceSize size,
3558  VmaSuballocationType suballocationType,
3559  bool mapped,
3560  bool canBecomeLost)
3561  {
3562  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3563  VMA_ASSERT(block != VMA_NULL);
3564  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3565  m_Alignment = alignment;
3566  m_Size = size;
3567  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3568  m_SuballocationType = (uint8_t)suballocationType;
3569  m_BlockAllocation.m_hPool = hPool;
3570  m_BlockAllocation.m_Block = block;
3571  m_BlockAllocation.m_Offset = offset;
3572  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
3573  }
3574 
3575  void InitLost()
3576  {
3577  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3578  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
3579  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3580  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
3581  m_BlockAllocation.m_Block = VMA_NULL;
3582  m_BlockAllocation.m_Offset = 0;
3583  m_BlockAllocation.m_CanBecomeLost = true;
3584  }
3585 
3586  void ChangeBlockAllocation(
3587  VmaAllocator hAllocator,
3588  VmaDeviceMemoryBlock* block,
3589  VkDeviceSize offset);
3590 
3591  // pMappedData not null means allocation is created with MAPPED flag.
3592  void InitDedicatedAllocation(
3593  uint32_t memoryTypeIndex,
3594  VkDeviceMemory hMemory,
3595  VmaSuballocationType suballocationType,
3596  void* pMappedData,
3597  VkDeviceSize size)
3598  {
3599  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3600  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
3601  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
3602  m_Alignment = 0;
3603  m_Size = size;
3604  m_SuballocationType = (uint8_t)suballocationType;
3605  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3606  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
3607  m_DedicatedAllocation.m_hMemory = hMemory;
3608  m_DedicatedAllocation.m_pMappedData = pMappedData;
3609  }
3610 
3611  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
3612  VkDeviceSize GetAlignment() const { return m_Alignment; }
3613  VkDeviceSize GetSize() const { return m_Size; }
3614  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
3615  void* GetUserData() const { return m_pUserData; }
3616  void SetUserData(VmaAllocator hAllocator, void* pUserData);
3617  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
3618 
3619  VmaDeviceMemoryBlock* GetBlock() const
3620  {
3621  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
3622  return m_BlockAllocation.m_Block;
3623  }
3624  VkDeviceSize GetOffset() const;
3625  VkDeviceMemory GetMemory() const;
3626  uint32_t GetMemoryTypeIndex() const;
3627  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
3628  void* GetMappedData() const;
3629  bool CanBecomeLost() const;
3630  VmaPool GetPool() const;
3631 
3632  uint32_t GetLastUseFrameIndex() const
3633  {
3634  return m_LastUseFrameIndex.load();
3635  }
3636  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
3637  {
3638  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
3639  }
3640  /*
3641  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
3642  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
3643  - Else, returns false.
3644 
3645  If hAllocation is already lost, assert - you should not call it then.
3646  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
3647  */
3648  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3649 
3650  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
3651  {
3652  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
3653  outInfo.blockCount = 1;
3654  outInfo.allocationCount = 1;
3655  outInfo.unusedRangeCount = 0;
3656  outInfo.usedBytes = m_Size;
3657  outInfo.unusedBytes = 0;
3658  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
3659  outInfo.unusedRangeSizeMin = UINT64_MAX;
3660  outInfo.unusedRangeSizeMax = 0;
3661  }
3662 
3663  void BlockAllocMap();
3664  void BlockAllocUnmap();
3665  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
3666  void DedicatedAllocUnmap(VmaAllocator hAllocator);
3667 
3668 private:
3669  VkDeviceSize m_Alignment;
3670  VkDeviceSize m_Size;
3671  void* m_pUserData;
3672  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
3673  uint8_t m_Type; // ALLOCATION_TYPE
3674  uint8_t m_SuballocationType; // VmaSuballocationType
3675  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
3676  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
3677  uint8_t m_MapCount;
3678  uint8_t m_Flags; // enum FLAGS
3679 
3680  // Allocation out of VmaDeviceMemoryBlock.
3681  struct BlockAllocation
3682  {
3683  VmaPool m_hPool; // Null if belongs to general memory.
3684  VmaDeviceMemoryBlock* m_Block;
3685  VkDeviceSize m_Offset;
3686  bool m_CanBecomeLost;
3687  };
3688 
3689  // Allocation for an object that has its own private VkDeviceMemory.
3690  struct DedicatedAllocation
3691  {
3692  uint32_t m_MemoryTypeIndex;
3693  VkDeviceMemory m_hMemory;
3694  void* m_pMappedData; // Not null means memory is mapped.
3695  };
3696 
3697  union
3698  {
3699  // Allocation out of VmaDeviceMemoryBlock.
3700  BlockAllocation m_BlockAllocation;
3701  // Allocation for an object that has its own private VkDeviceMemory.
3702  DedicatedAllocation m_DedicatedAllocation;
3703  };
3704 
3705  void FreeUserDataString(VmaAllocator hAllocator);
3706 };
3707 
3708 /*
3709 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
3710 allocated memory block or free.
3711 */
3712 struct VmaSuballocation
3713 {
3714  VkDeviceSize offset;
3715  VkDeviceSize size;
3716  VmaAllocation hAllocation;
3717  VmaSuballocationType type;
3718 };
3719 
3720 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
3721 
3722 // Cost of one additional allocation lost, as equivalent in bytes.
3723 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
3724 
3725 /*
3726 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3727 
3728 If canMakeOtherLost was false:
3729 - item points to a FREE suballocation.
3730 - itemsToMakeLostCount is 0.
3731 
3732 If canMakeOtherLost was true:
3733 - item points to first of sequence of suballocations, which are either FREE,
3734  or point to VmaAllocations that can become lost.
3735 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3736  the requested allocation to succeed.
3737 */
3738 struct VmaAllocationRequest
3739 {
3740  VkDeviceSize offset;
3741  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3742  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3743  VmaSuballocationList::iterator item;
3744  size_t itemsToMakeLostCount;
3745 
3746  VkDeviceSize CalcCost() const
3747  {
3748  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3749  }
3750 };
3751 
3752 /*
3753 Data structure used for bookkeeping of allocations and unused ranges of memory
3754 in a single VkDeviceMemory block.
3755 */
3756 class VmaBlockMetadata
3757 {
3758 public:
3759  VmaBlockMetadata(VmaAllocator hAllocator);
3760  ~VmaBlockMetadata();
3761  void Init(VkDeviceSize size);
3762 
3763  // Validates all data structures inside this object. If not valid, returns false.
3764  bool Validate() const;
3765  VkDeviceSize GetSize() const { return m_Size; }
3766  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
3767  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3768  VkDeviceSize GetUnusedRangeSizeMax() const;
3769  // Returns true if this block is empty - contains only single free suballocation.
3770  bool IsEmpty() const;
3771 
3772  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3773  void AddPoolStats(VmaPoolStats& inoutStats) const;
3774 
3775 #if VMA_STATS_STRING_ENABLED
3776  void PrintDetailedMap(class VmaJsonWriter& json) const;
3777 #endif
3778 
3779  // Creates trivial request for case when block is empty.
3780  void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3781 
3782  // Tries to find a place for suballocation with given parameters inside this block.
3783  // If succeeded, fills pAllocationRequest and returns true.
3784  // If failed, returns false.
3785  bool CreateAllocationRequest(
3786  uint32_t currentFrameIndex,
3787  uint32_t frameInUseCount,
3788  VkDeviceSize bufferImageGranularity,
3789  VkDeviceSize allocSize,
3790  VkDeviceSize allocAlignment,
3791  VmaSuballocationType allocType,
3792  bool canMakeOtherLost,
3793  VmaAllocationRequest* pAllocationRequest);
3794 
3795  bool MakeRequestedAllocationsLost(
3796  uint32_t currentFrameIndex,
3797  uint32_t frameInUseCount,
3798  VmaAllocationRequest* pAllocationRequest);
3799 
3800  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3801 
3802  // Makes actual allocation based on request. Request must already be checked and valid.
3803  void Alloc(
3804  const VmaAllocationRequest& request,
3805  VmaSuballocationType type,
3806  VkDeviceSize allocSize,
3807  VmaAllocation hAllocation);
3808 
3809  // Frees suballocation assigned to given memory region.
3810  void Free(const VmaAllocation allocation);
3811  void FreeAtOffset(VkDeviceSize offset);
3812 
3813 private:
3814  VkDeviceSize m_Size;
3815  uint32_t m_FreeCount;
3816  VkDeviceSize m_SumFreeSize;
3817  VmaSuballocationList m_Suballocations;
3818  // Suballocations that are free and have size greater than certain threshold.
3819  // Sorted by size, ascending.
3820  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
3821 
3822  bool ValidateFreeSuballocationList() const;
3823 
3824  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
3825  // If yes, fills pOffset and returns true. If no, returns false.
3826  bool CheckAllocation(
3827  uint32_t currentFrameIndex,
3828  uint32_t frameInUseCount,
3829  VkDeviceSize bufferImageGranularity,
3830  VkDeviceSize allocSize,
3831  VkDeviceSize allocAlignment,
3832  VmaSuballocationType allocType,
3833  VmaSuballocationList::const_iterator suballocItem,
3834  bool canMakeOtherLost,
3835  VkDeviceSize* pOffset,
3836  size_t* itemsToMakeLostCount,
3837  VkDeviceSize* pSumFreeSize,
3838  VkDeviceSize* pSumItemSize) const;
3839  // Given free suballocation, it merges it with following one, which must also be free.
3840  void MergeFreeWithNext(VmaSuballocationList::iterator item);
3841  // Releases given suballocation, making it free.
3842  // Merges it with adjacent free suballocations if applicable.
3843  // Returns iterator to new free suballocation at this place.
3844  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
3845  // Given free suballocation, it inserts it into sorted list of
3846  // m_FreeSuballocationsBySize if it's suitable.
3847  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
3848  // Given free suballocation, it removes it from sorted list of
3849  // m_FreeSuballocationsBySize if it's suitable.
3850  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
3851 };
3852 
3853 // Helper class that represents mapped memory. Synchronized internally.
3854 class VmaDeviceMemoryMapping
3855 {
3856 public:
3857  VmaDeviceMemoryMapping();
3858  ~VmaDeviceMemoryMapping();
3859 
3860  void* GetMappedData() const { return m_pMappedData; }
3861 
3862  // ppData can be null.
3863  VkResult Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData);
3864  void Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count);
3865 
3866 private:
3867  VMA_MUTEX m_Mutex;
3868  uint32_t m_MapCount;
3869  void* m_pMappedData;
3870 };
3871 
3872 /*
3873 Represents a single block of device memory (`VkDeviceMemory`) with all the
3874 data about its regions (aka suballocations, `VmaAllocation`), assigned and free.
3875 
3876 Thread-safety: This class must be externally synchronized.
3877 */
3878 class VmaDeviceMemoryBlock
3879 {
3880 public:
3881  uint32_t m_MemoryTypeIndex;
3882  VkDeviceMemory m_hMemory;
3883  VmaDeviceMemoryMapping m_Mapping;
3884  VmaBlockMetadata m_Metadata;
3885 
3886  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
3887 
3888  ~VmaDeviceMemoryBlock()
3889  {
3890  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
3891  }
3892 
3893  // Always call after construction.
3894  void Init(
3895  uint32_t newMemoryTypeIndex,
3896  VkDeviceMemory newMemory,
3897  VkDeviceSize newSize);
3898  // Always call before destruction.
3899  void Destroy(VmaAllocator allocator);
3900 
3901  // Validates all data structures inside this object. If not valid, returns false.
3902  bool Validate() const;
3903 
3904  // ppData can be null.
3905  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
3906  void Unmap(VmaAllocator hAllocator, uint32_t count);
3907 };
3908 
3909 struct VmaPointerLess
3910 {
3911  bool operator()(const void* lhs, const void* rhs) const
3912  {
3913  return lhs < rhs;
3914  }
3915 };
3916 
3917 class VmaDefragmentator;
3918 
3919 /*
3920 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
3921 Vulkan memory type.
3922 
3923 Synchronized internally with a mutex.
3924 */
3925 struct VmaBlockVector
3926 {
3927  VmaBlockVector(
3928  VmaAllocator hAllocator,
3929  uint32_t memoryTypeIndex,
3930  VkDeviceSize preferredBlockSize,
3931  size_t minBlockCount,
3932  size_t maxBlockCount,
3933  VkDeviceSize bufferImageGranularity,
3934  uint32_t frameInUseCount,
3935  bool isCustomPool);
3936  ~VmaBlockVector();
3937 
3938  VkResult CreateMinBlocks();
3939 
3940  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
3941  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
3942  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
3943  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
3944 
3945  void GetPoolStats(VmaPoolStats* pStats);
3946 
3947  bool IsEmpty() const { return m_Blocks.empty(); }
3948 
3949  VkResult Allocate(
3950  VmaPool hCurrentPool,
3951  uint32_t currentFrameIndex,
3952  const VkMemoryRequirements& vkMemReq,
3953  const VmaAllocationCreateInfo& createInfo,
3954  VmaSuballocationType suballocType,
3955  VmaAllocation* pAllocation);
3956 
3957  void Free(
3958  VmaAllocation hAllocation);
3959 
3960  // Adds statistics of this BlockVector to pStats.
3961  void AddStats(VmaStats* pStats);
3962 
3963 #if VMA_STATS_STRING_ENABLED
3964  void PrintDetailedMap(class VmaJsonWriter& json);
3965 #endif
3966 
3967  void MakePoolAllocationsLost(
3968  uint32_t currentFrameIndex,
3969  size_t* pLostAllocationCount);
3970 
3971  VmaDefragmentator* EnsureDefragmentator(
3972  VmaAllocator hAllocator,
3973  uint32_t currentFrameIndex);
3974 
3975  VkResult Defragment(
3976  VmaDefragmentationStats* pDefragmentationStats,
3977  VkDeviceSize& maxBytesToMove,
3978  uint32_t& maxAllocationsToMove);
3979 
3980  void DestroyDefragmentator();
3981 
3982 private:
3983  friend class VmaDefragmentator;
3984 
3985  const VmaAllocator m_hAllocator;
3986  const uint32_t m_MemoryTypeIndex;
3987  const VkDeviceSize m_PreferredBlockSize;
3988  const size_t m_MinBlockCount;
3989  const size_t m_MaxBlockCount;
3990  const VkDeviceSize m_BufferImageGranularity;
3991  const uint32_t m_FrameInUseCount;
3992  const bool m_IsCustomPool;
3993  VMA_MUTEX m_Mutex;
3994  // Incrementally sorted by sumFreeSize, ascending.
3995  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
3996  /* There can be at most one allocation that is completely empty - a
3997  hysteresis to avoid pessimistic case of alternating creation and destruction
3998  of a VkDeviceMemory. */
3999  bool m_HasEmptyBlock;
4000  VmaDefragmentator* m_pDefragmentator;
4001 
4002  size_t CalcMaxBlockSize() const;
4003 
4004  // Finds and removes given block from vector.
4005  void Remove(VmaDeviceMemoryBlock* pBlock);
4006 
4007  // Performs single step in sorting m_Blocks. They may not be fully sorted
4008  // after this call.
4009  void IncrementallySortBlocks();
4010 
4011  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
4012 };
4013 
4014 struct VmaPool_T
4015 {
4016 public:
4017  VmaBlockVector m_BlockVector;
4018 
4019  // Takes ownership.
4020  VmaPool_T(
4021  VmaAllocator hAllocator,
4022  const VmaPoolCreateInfo& createInfo);
4023  ~VmaPool_T();
4024 
4025  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
4026 
4027 #if VMA_STATS_STRING_ENABLED
4028  //void PrintDetailedMap(class VmaStringBuilder& sb);
4029 #endif
4030 };
4031 
4032 class VmaDefragmentator
4033 {
4034  const VmaAllocator m_hAllocator;
4035  VmaBlockVector* const m_pBlockVector;
4036  uint32_t m_CurrentFrameIndex;
4037  VkDeviceSize m_BytesMoved;
4038  uint32_t m_AllocationsMoved;
4039 
4040  struct AllocationInfo
4041  {
4042  VmaAllocation m_hAllocation;
4043  VkBool32* m_pChanged;
4044 
4045  AllocationInfo() :
4046  m_hAllocation(VK_NULL_HANDLE),
4047  m_pChanged(VMA_NULL)
4048  {
4049  }
4050  };
4051 
4052  struct AllocationInfoSizeGreater
4053  {
4054  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
4055  {
4056  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
4057  }
4058  };
4059 
4060  // Used between AddAllocation and Defragment.
4061  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
4062 
4063  struct BlockInfo
4064  {
4065  VmaDeviceMemoryBlock* m_pBlock;
4066  bool m_HasNonMovableAllocations;
4067  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
4068 
4069  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
4070  m_pBlock(VMA_NULL),
4071  m_HasNonMovableAllocations(true),
4072  m_Allocations(pAllocationCallbacks),
4073  m_pMappedDataForDefragmentation(VMA_NULL)
4074  {
4075  }
4076 
4077  void CalcHasNonMovableAllocations()
4078  {
4079  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
4080  const size_t defragmentAllocCount = m_Allocations.size();
4081  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
4082  }
4083 
4084  void SortAllocationsBySizeDescecnding()
4085  {
4086  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
4087  }
4088 
4089  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
4090  void Unmap(VmaAllocator hAllocator);
4091 
4092  private:
4093  // Not null if mapped for defragmentation only, not originally mapped.
4094  void* m_pMappedDataForDefragmentation;
4095  };
4096 
4097  struct BlockPointerLess
4098  {
4099  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
4100  {
4101  return pLhsBlockInfo->m_pBlock < pRhsBlock;
4102  }
4103  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
4104  {
4105  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
4106  }
4107  };
4108 
4109  // 1. Blocks with some non-movable allocations go first.
4110  // 2. Blocks with smaller sumFreeSize go first.
4111  struct BlockInfoCompareMoveDestination
4112  {
4113  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
4114  {
4115  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
4116  {
4117  return true;
4118  }
4119  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
4120  {
4121  return false;
4122  }
4123  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
4124  {
4125  return true;
4126  }
4127  return false;
4128  }
4129  };
4130 
4131  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
4132  BlockInfoVector m_Blocks;
4133 
4134  VkResult DefragmentRound(
4135  VkDeviceSize maxBytesToMove,
4136  uint32_t maxAllocationsToMove);
4137 
4138  static bool MoveMakesSense(
4139  size_t dstBlockIndex, VkDeviceSize dstOffset,
4140  size_t srcBlockIndex, VkDeviceSize srcOffset);
4141 
4142 public:
4143  VmaDefragmentator(
4144  VmaAllocator hAllocator,
4145  VmaBlockVector* pBlockVector,
4146  uint32_t currentFrameIndex);
4147 
4148  ~VmaDefragmentator();
4149 
4150  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
4151  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
4152 
4153  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
4154 
4155  VkResult Defragment(
4156  VkDeviceSize maxBytesToMove,
4157  uint32_t maxAllocationsToMove);
4158 };
4159 
4160 // Main allocator object.
4161 struct VmaAllocator_T
4162 {
4163  bool m_UseMutex;
4164  bool m_UseKhrDedicatedAllocation;
4165  VkDevice m_hDevice;
4166  bool m_AllocationCallbacksSpecified;
4167  VkAllocationCallbacks m_AllocationCallbacks;
4168  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
4169 
4170  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
4171  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
4172  VMA_MUTEX m_HeapSizeLimitMutex;
4173 
4174  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
4175  VkPhysicalDeviceMemoryProperties m_MemProps;
4176 
4177  // Default pools.
4178  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
4179 
4180  // Each vector is sorted by memory (handle value).
4181  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
4182  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
4183  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
4184 
4185  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
4186  ~VmaAllocator_T();
4187 
4188  const VkAllocationCallbacks* GetAllocationCallbacks() const
4189  {
4190  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
4191  }
4192  const VmaVulkanFunctions& GetVulkanFunctions() const
4193  {
4194  return m_VulkanFunctions;
4195  }
4196 
4197  VkDeviceSize GetBufferImageGranularity() const
4198  {
4199  return VMA_MAX(
4200  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
4201  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
4202  }
4203 
4204  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
4205  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
4206 
4207  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
4208  {
4209  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
4210  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
4211  }
4212 
4213  void GetBufferMemoryRequirements(
4214  VkBuffer hBuffer,
4215  VkMemoryRequirements& memReq,
4216  bool& requiresDedicatedAllocation,
4217  bool& prefersDedicatedAllocation) const;
4218  void GetImageMemoryRequirements(
4219  VkImage hImage,
4220  VkMemoryRequirements& memReq,
4221  bool& requiresDedicatedAllocation,
4222  bool& prefersDedicatedAllocation) const;
4223 
4224  // Main allocation function.
4225  VkResult AllocateMemory(
4226  const VkMemoryRequirements& vkMemReq,
4227  bool requiresDedicatedAllocation,
4228  bool prefersDedicatedAllocation,
4229  VkBuffer dedicatedBuffer,
4230  VkImage dedicatedImage,
4231  const VmaAllocationCreateInfo& createInfo,
4232  VmaSuballocationType suballocType,
4233  VmaAllocation* pAllocation);
4234 
4235  // Main deallocation function.
4236  void FreeMemory(const VmaAllocation allocation);
4237 
4238  void CalculateStats(VmaStats* pStats);
4239 
4240 #if VMA_STATS_STRING_ENABLED
4241  void PrintDetailedMap(class VmaJsonWriter& json);
4242 #endif
4243 
4244  VkResult Defragment(
4245  VmaAllocation* pAllocations,
4246  size_t allocationCount,
4247  VkBool32* pAllocationsChanged,
4248  const VmaDefragmentationInfo* pDefragmentationInfo,
4249  VmaDefragmentationStats* pDefragmentationStats);
4250 
4251  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
4252  bool TouchAllocation(VmaAllocation hAllocation);
4253 
4254  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
4255  void DestroyPool(VmaPool pool);
4256  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
4257 
4258  void SetCurrentFrameIndex(uint32_t frameIndex);
4259 
4260  void MakePoolAllocationsLost(
4261  VmaPool hPool,
4262  size_t* pLostAllocationCount);
4263 
4264  void CreateLostAllocation(VmaAllocation* pAllocation);
4265 
4266  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
4267  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
4268 
4269  VkResult Map(VmaAllocation hAllocation, void** ppData);
4270  void Unmap(VmaAllocation hAllocation);
4271 
4272 private:
4273  VkDeviceSize m_PreferredLargeHeapBlockSize;
4274 
4275  VkPhysicalDevice m_PhysicalDevice;
4276  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
4277 
4278  VMA_MUTEX m_PoolsMutex;
4279  // Protected by m_PoolsMutex. Sorted by pointer value.
4280  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
4281 
4282  VmaVulkanFunctions m_VulkanFunctions;
4283 
4284  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
4285 
4286  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
4287 
4288  VkResult AllocateMemoryOfType(
4289  const VkMemoryRequirements& vkMemReq,
4290  bool dedicatedAllocation,
4291  VkBuffer dedicatedBuffer,
4292  VkImage dedicatedImage,
4293  const VmaAllocationCreateInfo& createInfo,
4294  uint32_t memTypeIndex,
4295  VmaSuballocationType suballocType,
4296  VmaAllocation* pAllocation);
4297 
4298  // Allocates and registers new VkDeviceMemory specifically for single allocation.
4299  VkResult AllocateDedicatedMemory(
4300  VkDeviceSize size,
4301  VmaSuballocationType suballocType,
4302  uint32_t memTypeIndex,
4303  bool map,
4304  bool isUserDataString,
4305  void* pUserData,
4306  VkBuffer dedicatedBuffer,
4307  VkImage dedicatedImage,
4308  VmaAllocation* pAllocation);
4309 
4310  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
4311  void FreeDedicatedMemory(VmaAllocation allocation);
4312 };
4313 
4315 // Memory allocation #2 after VmaAllocator_T definition
4316 
4317 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
4318 {
4319  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
4320 }
4321 
4322 static void VmaFree(VmaAllocator hAllocator, void* ptr)
4323 {
4324  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
4325 }
4326 
4327 template<typename T>
4328 static T* VmaAllocate(VmaAllocator hAllocator)
4329 {
4330  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
4331 }
4332 
4333 template<typename T>
4334 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
4335 {
4336  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
4337 }
4338 
4339 template<typename T>
4340 static void vma_delete(VmaAllocator hAllocator, T* ptr)
4341 {
4342  if(ptr != VMA_NULL)
4343  {
4344  ptr->~T();
4345  VmaFree(hAllocator, ptr);
4346  }
4347 }
4348 
4349 template<typename T>
4350 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
4351 {
4352  if(ptr != VMA_NULL)
4353  {
4354  for(size_t i = count; i--; )
4355  ptr[i].~T();
4356  VmaFree(hAllocator, ptr);
4357  }
4358 }
4359 
4361 // VmaStringBuilder
4362 
4363 #if VMA_STATS_STRING_ENABLED
4364 
4365 class VmaStringBuilder
4366 {
4367 public:
4368  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
4369  size_t GetLength() const { return m_Data.size(); }
4370  const char* GetData() const { return m_Data.data(); }
4371 
4372  void Add(char ch) { m_Data.push_back(ch); }
4373  void Add(const char* pStr);
4374  void AddNewLine() { Add('\n'); }
4375  void AddNumber(uint32_t num);
4376  void AddNumber(uint64_t num);
4377  void AddPointer(const void* ptr);
4378 
4379 private:
4380  VmaVector< char, VmaStlAllocator<char> > m_Data;
4381 };
4382 
4383 void VmaStringBuilder::Add(const char* pStr)
4384 {
4385  const size_t strLen = strlen(pStr);
4386  if(strLen > 0)
4387  {
4388  const size_t oldCount = m_Data.size();
4389  m_Data.resize(oldCount + strLen);
4390  memcpy(m_Data.data() + oldCount, pStr, strLen);
4391  }
4392 }
4393 
4394 void VmaStringBuilder::AddNumber(uint32_t num)
4395 {
4396  char buf[11];
4397  VmaUint32ToStr(buf, sizeof(buf), num);
4398  Add(buf);
4399 }
4400 
4401 void VmaStringBuilder::AddNumber(uint64_t num)
4402 {
4403  char buf[21];
4404  VmaUint64ToStr(buf, sizeof(buf), num);
4405  Add(buf);
4406 }
4407 
4408 void VmaStringBuilder::AddPointer(const void* ptr)
4409 {
4410  char buf[21];
4411  VmaPtrToStr(buf, sizeof(buf), ptr);
4412  Add(buf);
4413 }
4414 
4415 #endif // #if VMA_STATS_STRING_ENABLED
4416 
4418 // VmaJsonWriter
4419 
4420 #if VMA_STATS_STRING_ENABLED
4421 
4422 class VmaJsonWriter
4423 {
4424 public:
4425  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
4426  ~VmaJsonWriter();
4427 
4428  void BeginObject(bool singleLine = false);
4429  void EndObject();
4430 
4431  void BeginArray(bool singleLine = false);
4432  void EndArray();
4433 
4434  void WriteString(const char* pStr);
4435  void BeginString(const char* pStr = VMA_NULL);
4436  void ContinueString(const char* pStr);
4437  void ContinueString(uint32_t n);
4438  void ContinueString(uint64_t n);
4439  void ContinueString_Pointer(const void* ptr);
4440  void EndString(const char* pStr = VMA_NULL);
4441 
4442  void WriteNumber(uint32_t n);
4443  void WriteNumber(uint64_t n);
4444  void WriteBool(bool b);
4445  void WriteNull();
4446 
4447 private:
4448  static const char* const INDENT;
4449 
4450  enum COLLECTION_TYPE
4451  {
4452  COLLECTION_TYPE_OBJECT,
4453  COLLECTION_TYPE_ARRAY,
4454  };
4455  struct StackItem
4456  {
4457  COLLECTION_TYPE type;
4458  uint32_t valueCount;
4459  bool singleLineMode;
4460  };
4461 
4462  VmaStringBuilder& m_SB;
4463  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
4464  bool m_InsideString;
4465 
4466  void BeginValue(bool isString);
4467  void WriteIndent(bool oneLess = false);
4468 };
4469 
4470 const char* const VmaJsonWriter::INDENT = " ";
4471 
4472 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
4473  m_SB(sb),
4474  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
4475  m_InsideString(false)
4476 {
4477 }
4478 
4479 VmaJsonWriter::~VmaJsonWriter()
4480 {
4481  VMA_ASSERT(!m_InsideString);
4482  VMA_ASSERT(m_Stack.empty());
4483 }
4484 
4485 void VmaJsonWriter::BeginObject(bool singleLine)
4486 {
4487  VMA_ASSERT(!m_InsideString);
4488 
4489  BeginValue(false);
4490  m_SB.Add('{');
4491 
4492  StackItem item;
4493  item.type = COLLECTION_TYPE_OBJECT;
4494  item.valueCount = 0;
4495  item.singleLineMode = singleLine;
4496  m_Stack.push_back(item);
4497 }
4498 
4499 void VmaJsonWriter::EndObject()
4500 {
4501  VMA_ASSERT(!m_InsideString);
4502 
4503  WriteIndent(true);
4504  m_SB.Add('}');
4505 
4506  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
4507  m_Stack.pop_back();
4508 }
4509 
4510 void VmaJsonWriter::BeginArray(bool singleLine)
4511 {
4512  VMA_ASSERT(!m_InsideString);
4513 
4514  BeginValue(false);
4515  m_SB.Add('[');
4516 
4517  StackItem item;
4518  item.type = COLLECTION_TYPE_ARRAY;
4519  item.valueCount = 0;
4520  item.singleLineMode = singleLine;
4521  m_Stack.push_back(item);
4522 }
4523 
4524 void VmaJsonWriter::EndArray()
4525 {
4526  VMA_ASSERT(!m_InsideString);
4527 
4528  WriteIndent(true);
4529  m_SB.Add(']');
4530 
4531  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
4532  m_Stack.pop_back();
4533 }
4534 
4535 void VmaJsonWriter::WriteString(const char* pStr)
4536 {
4537  BeginString(pStr);
4538  EndString();
4539 }
4540 
4541 void VmaJsonWriter::BeginString(const char* pStr)
4542 {
4543  VMA_ASSERT(!m_InsideString);
4544 
4545  BeginValue(true);
4546  m_SB.Add('"');
4547  m_InsideString = true;
4548  if(pStr != VMA_NULL && pStr[0] != '\0')
4549  {
4550  ContinueString(pStr);
4551  }
4552 }
4553 
4554 void VmaJsonWriter::ContinueString(const char* pStr)
4555 {
4556  VMA_ASSERT(m_InsideString);
4557 
4558  const size_t strLen = strlen(pStr);
4559  for(size_t i = 0; i < strLen; ++i)
4560  {
4561  char ch = pStr[i];
4562  if(ch == '\'')
4563  {
4564  m_SB.Add("\\\\");
4565  }
4566  else if(ch == '"')
4567  {
4568  m_SB.Add("\\\"");
4569  }
4570  else if(ch >= 32)
4571  {
4572  m_SB.Add(ch);
4573  }
4574  else switch(ch)
4575  {
4576  case '\b':
4577  m_SB.Add("\\b");
4578  break;
4579  case '\f':
4580  m_SB.Add("\\f");
4581  break;
4582  case '\n':
4583  m_SB.Add("\\n");
4584  break;
4585  case '\r':
4586  m_SB.Add("\\r");
4587  break;
4588  case '\t':
4589  m_SB.Add("\\t");
4590  break;
4591  default:
4592  VMA_ASSERT(0 && "Character not currently supported.");
4593  break;
4594  }
4595  }
4596 }
4597 
4598 void VmaJsonWriter::ContinueString(uint32_t n)
4599 {
4600  VMA_ASSERT(m_InsideString);
4601  m_SB.AddNumber(n);
4602 }
4603 
4604 void VmaJsonWriter::ContinueString(uint64_t n)
4605 {
4606  VMA_ASSERT(m_InsideString);
4607  m_SB.AddNumber(n);
4608 }
4609 
4610 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
4611 {
4612  VMA_ASSERT(m_InsideString);
4613  m_SB.AddPointer(ptr);
4614 }
4615 
4616 void VmaJsonWriter::EndString(const char* pStr)
4617 {
4618  VMA_ASSERT(m_InsideString);
4619  if(pStr != VMA_NULL && pStr[0] != '\0')
4620  {
4621  ContinueString(pStr);
4622  }
4623  m_SB.Add('"');
4624  m_InsideString = false;
4625 }
4626 
4627 void VmaJsonWriter::WriteNumber(uint32_t n)
4628 {
4629  VMA_ASSERT(!m_InsideString);
4630  BeginValue(false);
4631  m_SB.AddNumber(n);
4632 }
4633 
4634 void VmaJsonWriter::WriteNumber(uint64_t n)
4635 {
4636  VMA_ASSERT(!m_InsideString);
4637  BeginValue(false);
4638  m_SB.AddNumber(n);
4639 }
4640 
4641 void VmaJsonWriter::WriteBool(bool b)
4642 {
4643  VMA_ASSERT(!m_InsideString);
4644  BeginValue(false);
4645  m_SB.Add(b ? "true" : "false");
4646 }
4647 
4648 void VmaJsonWriter::WriteNull()
4649 {
4650  VMA_ASSERT(!m_InsideString);
4651  BeginValue(false);
4652  m_SB.Add("null");
4653 }
4654 
4655 void VmaJsonWriter::BeginValue(bool isString)
4656 {
4657  if(!m_Stack.empty())
4658  {
4659  StackItem& currItem = m_Stack.back();
4660  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4661  currItem.valueCount % 2 == 0)
4662  {
4663  VMA_ASSERT(isString);
4664  }
4665 
4666  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4667  currItem.valueCount % 2 != 0)
4668  {
4669  m_SB.Add(": ");
4670  }
4671  else if(currItem.valueCount > 0)
4672  {
4673  m_SB.Add(", ");
4674  WriteIndent();
4675  }
4676  else
4677  {
4678  WriteIndent();
4679  }
4680  ++currItem.valueCount;
4681  }
4682 }
4683 
4684 void VmaJsonWriter::WriteIndent(bool oneLess)
4685 {
4686  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
4687  {
4688  m_SB.AddNewLine();
4689 
4690  size_t count = m_Stack.size();
4691  if(count > 0 && oneLess)
4692  {
4693  --count;
4694  }
4695  for(size_t i = 0; i < count; ++i)
4696  {
4697  m_SB.Add(INDENT);
4698  }
4699  }
4700 }
4701 
4702 #endif // #if VMA_STATS_STRING_ENABLED
4703 
4705 
4706 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
4707 {
4708  if(IsUserDataString())
4709  {
4710  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
4711 
4712  FreeUserDataString(hAllocator);
4713 
4714  if(pUserData != VMA_NULL)
4715  {
4716  const char* const newStrSrc = (char*)pUserData;
4717  const size_t newStrLen = strlen(newStrSrc);
4718  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
4719  memcpy(newStrDst, newStrSrc, newStrLen + 1);
4720  m_pUserData = newStrDst;
4721  }
4722  }
4723  else
4724  {
4725  m_pUserData = pUserData;
4726  }
4727 }
4728 
4729 void VmaAllocation_T::ChangeBlockAllocation(
4730  VmaAllocator hAllocator,
4731  VmaDeviceMemoryBlock* block,
4732  VkDeviceSize offset)
4733 {
4734  VMA_ASSERT(block != VMA_NULL);
4735  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4736 
4737  // Move mapping reference counter from old block to new block.
4738  if(block != m_BlockAllocation.m_Block)
4739  {
4740  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
4741  if(IsPersistentMap())
4742  ++mapRefCount;
4743  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
4744  block->Map(hAllocator, mapRefCount, VMA_NULL);
4745  }
4746 
4747  m_BlockAllocation.m_Block = block;
4748  m_BlockAllocation.m_Offset = offset;
4749 }
4750 
4751 VkDeviceSize VmaAllocation_T::GetOffset() const
4752 {
4753  switch(m_Type)
4754  {
4755  case ALLOCATION_TYPE_BLOCK:
4756  return m_BlockAllocation.m_Offset;
4757  case ALLOCATION_TYPE_DEDICATED:
4758  return 0;
4759  default:
4760  VMA_ASSERT(0);
4761  return 0;
4762  }
4763 }
4764 
4765 VkDeviceMemory VmaAllocation_T::GetMemory() const
4766 {
4767  switch(m_Type)
4768  {
4769  case ALLOCATION_TYPE_BLOCK:
4770  return m_BlockAllocation.m_Block->m_hMemory;
4771  case ALLOCATION_TYPE_DEDICATED:
4772  return m_DedicatedAllocation.m_hMemory;
4773  default:
4774  VMA_ASSERT(0);
4775  return VK_NULL_HANDLE;
4776  }
4777 }
4778 
4779 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
4780 {
4781  switch(m_Type)
4782  {
4783  case ALLOCATION_TYPE_BLOCK:
4784  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
4785  case ALLOCATION_TYPE_DEDICATED:
4786  return m_DedicatedAllocation.m_MemoryTypeIndex;
4787  default:
4788  VMA_ASSERT(0);
4789  return UINT32_MAX;
4790  }
4791 }
4792 
4793 void* VmaAllocation_T::GetMappedData() const
4794 {
4795  switch(m_Type)
4796  {
4797  case ALLOCATION_TYPE_BLOCK:
4798  if(m_MapCount != 0)
4799  {
4800  void* pBlockData = m_BlockAllocation.m_Block->m_Mapping.GetMappedData();
4801  VMA_ASSERT(pBlockData != VMA_NULL);
4802  return (char*)pBlockData + m_BlockAllocation.m_Offset;
4803  }
4804  else
4805  {
4806  return VMA_NULL;
4807  }
4808  break;
4809  case ALLOCATION_TYPE_DEDICATED:
4810  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
4811  return m_DedicatedAllocation.m_pMappedData;
4812  default:
4813  VMA_ASSERT(0);
4814  return VMA_NULL;
4815  }
4816 }
4817 
4818 bool VmaAllocation_T::CanBecomeLost() const
4819 {
4820  switch(m_Type)
4821  {
4822  case ALLOCATION_TYPE_BLOCK:
4823  return m_BlockAllocation.m_CanBecomeLost;
4824  case ALLOCATION_TYPE_DEDICATED:
4825  return false;
4826  default:
4827  VMA_ASSERT(0);
4828  return false;
4829  }
4830 }
4831 
4832 VmaPool VmaAllocation_T::GetPool() const
4833 {
4834  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4835  return m_BlockAllocation.m_hPool;
4836 }
4837 
4838 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4839 {
4840  VMA_ASSERT(CanBecomeLost());
4841 
4842  /*
4843  Warning: This is a carefully designed algorithm.
4844  Do not modify unless you really know what you're doing :)
4845  */
4846  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
4847  for(;;)
4848  {
4849  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
4850  {
4851  VMA_ASSERT(0);
4852  return false;
4853  }
4854  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
4855  {
4856  return false;
4857  }
4858  else // Last use time earlier than current time.
4859  {
4860  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
4861  {
4862  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
4863  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
4864  return true;
4865  }
4866  }
4867  }
4868 }
4869 
4870 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
4871 {
4872  VMA_ASSERT(IsUserDataString());
4873  if(m_pUserData != VMA_NULL)
4874  {
4875  char* const oldStr = (char*)m_pUserData;
4876  const size_t oldStrLen = strlen(oldStr);
4877  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
4878  m_pUserData = VMA_NULL;
4879  }
4880 }
4881 
4882 void VmaAllocation_T::BlockAllocMap()
4883 {
4884  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4885 
4886  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4887  {
4888  ++m_MapCount;
4889  }
4890  else
4891  {
4892  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
4893  }
4894 }
4895 
4896 void VmaAllocation_T::BlockAllocUnmap()
4897 {
4898  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4899 
4900  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4901  {
4902  --m_MapCount;
4903  }
4904  else
4905  {
4906  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
4907  }
4908 }
4909 
4910 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
4911 {
4912  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4913 
4914  if(m_MapCount != 0)
4915  {
4916  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4917  {
4918  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
4919  *ppData = m_DedicatedAllocation.m_pMappedData;
4920  ++m_MapCount;
4921  return VK_SUCCESS;
4922  }
4923  else
4924  {
4925  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
4926  return VK_ERROR_MEMORY_MAP_FAILED;
4927  }
4928  }
4929  else
4930  {
4931  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
4932  hAllocator->m_hDevice,
4933  m_DedicatedAllocation.m_hMemory,
4934  0, // offset
4935  VK_WHOLE_SIZE,
4936  0, // flags
4937  ppData);
4938  if(result == VK_SUCCESS)
4939  {
4940  m_DedicatedAllocation.m_pMappedData = *ppData;
4941  m_MapCount = 1;
4942  }
4943  return result;
4944  }
4945 }
4946 
4947 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
4948 {
4949  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4950 
4951  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4952  {
4953  --m_MapCount;
4954  if(m_MapCount == 0)
4955  {
4956  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
4957  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
4958  hAllocator->m_hDevice,
4959  m_DedicatedAllocation.m_hMemory);
4960  }
4961  }
4962  else
4963  {
4964  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
4965  }
4966 }
4967 
4968 #if VMA_STATS_STRING_ENABLED
4969 
4970 // Correspond to values of enum VmaSuballocationType.
4971 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
4972  "FREE",
4973  "UNKNOWN",
4974  "BUFFER",
4975  "IMAGE_UNKNOWN",
4976  "IMAGE_LINEAR",
4977  "IMAGE_OPTIMAL",
4978 };
4979 
4980 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
4981 {
4982  json.BeginObject();
4983 
4984  json.WriteString("Blocks");
4985  json.WriteNumber(stat.blockCount);
4986 
4987  json.WriteString("Allocations");
4988  json.WriteNumber(stat.allocationCount);
4989 
4990  json.WriteString("UnusedRanges");
4991  json.WriteNumber(stat.unusedRangeCount);
4992 
4993  json.WriteString("UsedBytes");
4994  json.WriteNumber(stat.usedBytes);
4995 
4996  json.WriteString("UnusedBytes");
4997  json.WriteNumber(stat.unusedBytes);
4998 
4999  if(stat.allocationCount > 1)
5000  {
5001  json.WriteString("AllocationSize");
5002  json.BeginObject(true);
5003  json.WriteString("Min");
5004  json.WriteNumber(stat.allocationSizeMin);
5005  json.WriteString("Avg");
5006  json.WriteNumber(stat.allocationSizeAvg);
5007  json.WriteString("Max");
5008  json.WriteNumber(stat.allocationSizeMax);
5009  json.EndObject();
5010  }
5011 
5012  if(stat.unusedRangeCount > 1)
5013  {
5014  json.WriteString("UnusedRangeSize");
5015  json.BeginObject(true);
5016  json.WriteString("Min");
5017  json.WriteNumber(stat.unusedRangeSizeMin);
5018  json.WriteString("Avg");
5019  json.WriteNumber(stat.unusedRangeSizeAvg);
5020  json.WriteString("Max");
5021  json.WriteNumber(stat.unusedRangeSizeMax);
5022  json.EndObject();
5023  }
5024 
5025  json.EndObject();
5026 }
5027 
5028 #endif // #if VMA_STATS_STRING_ENABLED
5029 
5030 struct VmaSuballocationItemSizeLess
5031 {
5032  bool operator()(
5033  const VmaSuballocationList::iterator lhs,
5034  const VmaSuballocationList::iterator rhs) const
5035  {
5036  return lhs->size < rhs->size;
5037  }
5038  bool operator()(
5039  const VmaSuballocationList::iterator lhs,
5040  VkDeviceSize rhsSize) const
5041  {
5042  return lhs->size < rhsSize;
5043  }
5044 };
5045 
5047 // class VmaBlockMetadata
5048 
5049 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
5050  m_Size(0),
5051  m_FreeCount(0),
5052  m_SumFreeSize(0),
5053  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
5054  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
5055 {
5056 }
5057 
5058 VmaBlockMetadata::~VmaBlockMetadata()
5059 {
5060 }
5061 
5062 void VmaBlockMetadata::Init(VkDeviceSize size)
5063 {
5064  m_Size = size;
5065  m_FreeCount = 1;
5066  m_SumFreeSize = size;
5067 
5068  VmaSuballocation suballoc = {};
5069  suballoc.offset = 0;
5070  suballoc.size = size;
5071  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5072  suballoc.hAllocation = VK_NULL_HANDLE;
5073 
5074  m_Suballocations.push_back(suballoc);
5075  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
5076  --suballocItem;
5077  m_FreeSuballocationsBySize.push_back(suballocItem);
5078 }
5079 
5080 bool VmaBlockMetadata::Validate() const
5081 {
5082  if(m_Suballocations.empty())
5083  {
5084  return false;
5085  }
5086 
5087  // Expected offset of new suballocation as calculates from previous ones.
5088  VkDeviceSize calculatedOffset = 0;
5089  // Expected number of free suballocations as calculated from traversing their list.
5090  uint32_t calculatedFreeCount = 0;
5091  // Expected sum size of free suballocations as calculated from traversing their list.
5092  VkDeviceSize calculatedSumFreeSize = 0;
5093  // Expected number of free suballocations that should be registered in
5094  // m_FreeSuballocationsBySize calculated from traversing their list.
5095  size_t freeSuballocationsToRegister = 0;
5096  // True if previous visisted suballocation was free.
5097  bool prevFree = false;
5098 
5099  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5100  suballocItem != m_Suballocations.cend();
5101  ++suballocItem)
5102  {
5103  const VmaSuballocation& subAlloc = *suballocItem;
5104 
5105  // Actual offset of this suballocation doesn't match expected one.
5106  if(subAlloc.offset != calculatedOffset)
5107  {
5108  return false;
5109  }
5110 
5111  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
5112  // Two adjacent free suballocations are invalid. They should be merged.
5113  if(prevFree && currFree)
5114  {
5115  return false;
5116  }
5117 
5118  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
5119  {
5120  return false;
5121  }
5122 
5123  if(currFree)
5124  {
5125  calculatedSumFreeSize += subAlloc.size;
5126  ++calculatedFreeCount;
5127  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5128  {
5129  ++freeSuballocationsToRegister;
5130  }
5131  }
5132  else
5133  {
5134  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
5135  {
5136  return false;
5137  }
5138  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
5139  {
5140  return false;
5141  }
5142  }
5143 
5144  calculatedOffset += subAlloc.size;
5145  prevFree = currFree;
5146  }
5147 
5148  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
5149  // match expected one.
5150  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
5151  {
5152  return false;
5153  }
5154 
5155  VkDeviceSize lastSize = 0;
5156  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
5157  {
5158  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
5159 
5160  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
5161  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5162  {
5163  return false;
5164  }
5165  // They must be sorted by size ascending.
5166  if(suballocItem->size < lastSize)
5167  {
5168  return false;
5169  }
5170 
5171  lastSize = suballocItem->size;
5172  }
5173 
5174  // Check if totals match calculacted values.
5175  if(!ValidateFreeSuballocationList() ||
5176  (calculatedOffset != m_Size) ||
5177  (calculatedSumFreeSize != m_SumFreeSize) ||
5178  (calculatedFreeCount != m_FreeCount))
5179  {
5180  return false;
5181  }
5182 
5183  return true;
5184 }
5185 
5186 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
5187 {
5188  if(!m_FreeSuballocationsBySize.empty())
5189  {
5190  return m_FreeSuballocationsBySize.back()->size;
5191  }
5192  else
5193  {
5194  return 0;
5195  }
5196 }
5197 
5198 bool VmaBlockMetadata::IsEmpty() const
5199 {
5200  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
5201 }
5202 
5203 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
5204 {
5205  outInfo.blockCount = 1;
5206 
5207  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5208  outInfo.allocationCount = rangeCount - m_FreeCount;
5209  outInfo.unusedRangeCount = m_FreeCount;
5210 
5211  outInfo.unusedBytes = m_SumFreeSize;
5212  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
5213 
5214  outInfo.allocationSizeMin = UINT64_MAX;
5215  outInfo.allocationSizeMax = 0;
5216  outInfo.unusedRangeSizeMin = UINT64_MAX;
5217  outInfo.unusedRangeSizeMax = 0;
5218 
5219  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5220  suballocItem != m_Suballocations.cend();
5221  ++suballocItem)
5222  {
5223  const VmaSuballocation& suballoc = *suballocItem;
5224  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
5225  {
5226  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
5227  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
5228  }
5229  else
5230  {
5231  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
5232  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
5233  }
5234  }
5235 }
5236 
5237 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
5238 {
5239  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5240 
5241  inoutStats.size += m_Size;
5242  inoutStats.unusedSize += m_SumFreeSize;
5243  inoutStats.allocationCount += rangeCount - m_FreeCount;
5244  inoutStats.unusedRangeCount += m_FreeCount;
5245  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
5246 }
5247 
5248 #if VMA_STATS_STRING_ENABLED
5249 
5250 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
5251 {
5252  json.BeginObject();
5253 
5254  json.WriteString("TotalBytes");
5255  json.WriteNumber(m_Size);
5256 
5257  json.WriteString("UnusedBytes");
5258  json.WriteNumber(m_SumFreeSize);
5259 
5260  json.WriteString("Allocations");
5261  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
5262 
5263  json.WriteString("UnusedRanges");
5264  json.WriteNumber(m_FreeCount);
5265 
5266  json.WriteString("Suballocations");
5267  json.BeginArray();
5268  size_t i = 0;
5269  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5270  suballocItem != m_Suballocations.cend();
5271  ++suballocItem, ++i)
5272  {
5273  json.BeginObject(true);
5274 
5275  json.WriteString("Type");
5276  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
5277 
5278  json.WriteString("Size");
5279  json.WriteNumber(suballocItem->size);
5280 
5281  json.WriteString("Offset");
5282  json.WriteNumber(suballocItem->offset);
5283 
5284  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5285  {
5286  const void* pUserData = suballocItem->hAllocation->GetUserData();
5287  if(pUserData != VMA_NULL)
5288  {
5289  json.WriteString("UserData");
5290  if(suballocItem->hAllocation->IsUserDataString())
5291  {
5292  json.WriteString((const char*)pUserData);
5293  }
5294  else
5295  {
5296  json.BeginString();
5297  json.ContinueString_Pointer(pUserData);
5298  json.EndString();
5299  }
5300  }
5301  }
5302 
5303  json.EndObject();
5304  }
5305  json.EndArray();
5306 
5307  json.EndObject();
5308 }
5309 
5310 #endif // #if VMA_STATS_STRING_ENABLED
5311 
5312 /*
5313 How many suitable free suballocations to analyze before choosing best one.
5314 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
5315  be chosen.
5316 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
5317  suballocations will be analized and best one will be chosen.
5318 - Any other value is also acceptable.
5319 */
5320 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
5321 
5322 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
5323 {
5324  VMA_ASSERT(IsEmpty());
5325  pAllocationRequest->offset = 0;
5326  pAllocationRequest->sumFreeSize = m_SumFreeSize;
5327  pAllocationRequest->sumItemSize = 0;
5328  pAllocationRequest->item = m_Suballocations.begin();
5329  pAllocationRequest->itemsToMakeLostCount = 0;
5330 }
5331 
5332 bool VmaBlockMetadata::CreateAllocationRequest(
5333  uint32_t currentFrameIndex,
5334  uint32_t frameInUseCount,
5335  VkDeviceSize bufferImageGranularity,
5336  VkDeviceSize allocSize,
5337  VkDeviceSize allocAlignment,
5338  VmaSuballocationType allocType,
5339  bool canMakeOtherLost,
5340  VmaAllocationRequest* pAllocationRequest)
5341 {
5342  VMA_ASSERT(allocSize > 0);
5343  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5344  VMA_ASSERT(pAllocationRequest != VMA_NULL);
5345  VMA_HEAVY_ASSERT(Validate());
5346 
5347  // There is not enough total free space in this block to fullfill the request: Early return.
5348  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
5349  {
5350  return false;
5351  }
5352 
5353  // New algorithm, efficiently searching freeSuballocationsBySize.
5354  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
5355  if(freeSuballocCount > 0)
5356  {
5357  if(VMA_BEST_FIT)
5358  {
5359  // Find first free suballocation with size not less than allocSize.
5360  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5361  m_FreeSuballocationsBySize.data(),
5362  m_FreeSuballocationsBySize.data() + freeSuballocCount,
5363  allocSize,
5364  VmaSuballocationItemSizeLess());
5365  size_t index = it - m_FreeSuballocationsBySize.data();
5366  for(; index < freeSuballocCount; ++index)
5367  {
5368  if(CheckAllocation(
5369  currentFrameIndex,
5370  frameInUseCount,
5371  bufferImageGranularity,
5372  allocSize,
5373  allocAlignment,
5374  allocType,
5375  m_FreeSuballocationsBySize[index],
5376  false, // canMakeOtherLost
5377  &pAllocationRequest->offset,
5378  &pAllocationRequest->itemsToMakeLostCount,
5379  &pAllocationRequest->sumFreeSize,
5380  &pAllocationRequest->sumItemSize))
5381  {
5382  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5383  return true;
5384  }
5385  }
5386  }
5387  else
5388  {
5389  // Search staring from biggest suballocations.
5390  for(size_t index = freeSuballocCount; index--; )
5391  {
5392  if(CheckAllocation(
5393  currentFrameIndex,
5394  frameInUseCount,
5395  bufferImageGranularity,
5396  allocSize,
5397  allocAlignment,
5398  allocType,
5399  m_FreeSuballocationsBySize[index],
5400  false, // canMakeOtherLost
5401  &pAllocationRequest->offset,
5402  &pAllocationRequest->itemsToMakeLostCount,
5403  &pAllocationRequest->sumFreeSize,
5404  &pAllocationRequest->sumItemSize))
5405  {
5406  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5407  return true;
5408  }
5409  }
5410  }
5411  }
5412 
5413  if(canMakeOtherLost)
5414  {
5415  // Brute-force algorithm. TODO: Come up with something better.
5416 
5417  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
5418  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
5419 
5420  VmaAllocationRequest tmpAllocRequest = {};
5421  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
5422  suballocIt != m_Suballocations.end();
5423  ++suballocIt)
5424  {
5425  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
5426  suballocIt->hAllocation->CanBecomeLost())
5427  {
5428  if(CheckAllocation(
5429  currentFrameIndex,
5430  frameInUseCount,
5431  bufferImageGranularity,
5432  allocSize,
5433  allocAlignment,
5434  allocType,
5435  suballocIt,
5436  canMakeOtherLost,
5437  &tmpAllocRequest.offset,
5438  &tmpAllocRequest.itemsToMakeLostCount,
5439  &tmpAllocRequest.sumFreeSize,
5440  &tmpAllocRequest.sumItemSize))
5441  {
5442  tmpAllocRequest.item = suballocIt;
5443 
5444  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
5445  {
5446  *pAllocationRequest = tmpAllocRequest;
5447  }
5448  }
5449  }
5450  }
5451 
5452  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
5453  {
5454  return true;
5455  }
5456  }
5457 
5458  return false;
5459 }
5460 
5461 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
5462  uint32_t currentFrameIndex,
5463  uint32_t frameInUseCount,
5464  VmaAllocationRequest* pAllocationRequest)
5465 {
5466  while(pAllocationRequest->itemsToMakeLostCount > 0)
5467  {
5468  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
5469  {
5470  ++pAllocationRequest->item;
5471  }
5472  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5473  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
5474  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
5475  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5476  {
5477  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
5478  --pAllocationRequest->itemsToMakeLostCount;
5479  }
5480  else
5481  {
5482  return false;
5483  }
5484  }
5485 
5486  VMA_HEAVY_ASSERT(Validate());
5487  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5488  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
5489 
5490  return true;
5491 }
5492 
5493 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5494 {
5495  uint32_t lostAllocationCount = 0;
5496  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
5497  it != m_Suballocations.end();
5498  ++it)
5499  {
5500  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
5501  it->hAllocation->CanBecomeLost() &&
5502  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5503  {
5504  it = FreeSuballocation(it);
5505  ++lostAllocationCount;
5506  }
5507  }
5508  return lostAllocationCount;
5509 }
5510 
5511 void VmaBlockMetadata::Alloc(
5512  const VmaAllocationRequest& request,
5513  VmaSuballocationType type,
5514  VkDeviceSize allocSize,
5515  VmaAllocation hAllocation)
5516 {
5517  VMA_ASSERT(request.item != m_Suballocations.end());
5518  VmaSuballocation& suballoc = *request.item;
5519  // Given suballocation is a free block.
5520  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5521  // Given offset is inside this suballocation.
5522  VMA_ASSERT(request.offset >= suballoc.offset);
5523  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
5524  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
5525  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
5526 
5527  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
5528  // it to become used.
5529  UnregisterFreeSuballocation(request.item);
5530 
5531  suballoc.offset = request.offset;
5532  suballoc.size = allocSize;
5533  suballoc.type = type;
5534  suballoc.hAllocation = hAllocation;
5535 
5536  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
5537  if(paddingEnd)
5538  {
5539  VmaSuballocation paddingSuballoc = {};
5540  paddingSuballoc.offset = request.offset + allocSize;
5541  paddingSuballoc.size = paddingEnd;
5542  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5543  VmaSuballocationList::iterator next = request.item;
5544  ++next;
5545  const VmaSuballocationList::iterator paddingEndItem =
5546  m_Suballocations.insert(next, paddingSuballoc);
5547  RegisterFreeSuballocation(paddingEndItem);
5548  }
5549 
5550  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
5551  if(paddingBegin)
5552  {
5553  VmaSuballocation paddingSuballoc = {};
5554  paddingSuballoc.offset = request.offset - paddingBegin;
5555  paddingSuballoc.size = paddingBegin;
5556  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5557  const VmaSuballocationList::iterator paddingBeginItem =
5558  m_Suballocations.insert(request.item, paddingSuballoc);
5559  RegisterFreeSuballocation(paddingBeginItem);
5560  }
5561 
5562  // Update totals.
5563  m_FreeCount = m_FreeCount - 1;
5564  if(paddingBegin > 0)
5565  {
5566  ++m_FreeCount;
5567  }
5568  if(paddingEnd > 0)
5569  {
5570  ++m_FreeCount;
5571  }
5572  m_SumFreeSize -= allocSize;
5573 }
5574 
5575 void VmaBlockMetadata::Free(const VmaAllocation allocation)
5576 {
5577  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5578  suballocItem != m_Suballocations.end();
5579  ++suballocItem)
5580  {
5581  VmaSuballocation& suballoc = *suballocItem;
5582  if(suballoc.hAllocation == allocation)
5583  {
5584  FreeSuballocation(suballocItem);
5585  VMA_HEAVY_ASSERT(Validate());
5586  return;
5587  }
5588  }
5589  VMA_ASSERT(0 && "Not found!");
5590 }
5591 
5592 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
5593 {
5594  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5595  suballocItem != m_Suballocations.end();
5596  ++suballocItem)
5597  {
5598  VmaSuballocation& suballoc = *suballocItem;
5599  if(suballoc.offset == offset)
5600  {
5601  FreeSuballocation(suballocItem);
5602  return;
5603  }
5604  }
5605  VMA_ASSERT(0 && "Not found!");
5606 }
5607 
5608 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
5609 {
5610  VkDeviceSize lastSize = 0;
5611  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
5612  {
5613  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
5614 
5615  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
5616  {
5617  VMA_ASSERT(0);
5618  return false;
5619  }
5620  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5621  {
5622  VMA_ASSERT(0);
5623  return false;
5624  }
5625  if(it->size < lastSize)
5626  {
5627  VMA_ASSERT(0);
5628  return false;
5629  }
5630 
5631  lastSize = it->size;
5632  }
5633  return true;
5634 }
5635 
5636 bool VmaBlockMetadata::CheckAllocation(
5637  uint32_t currentFrameIndex,
5638  uint32_t frameInUseCount,
5639  VkDeviceSize bufferImageGranularity,
5640  VkDeviceSize allocSize,
5641  VkDeviceSize allocAlignment,
5642  VmaSuballocationType allocType,
5643  VmaSuballocationList::const_iterator suballocItem,
5644  bool canMakeOtherLost,
5645  VkDeviceSize* pOffset,
5646  size_t* itemsToMakeLostCount,
5647  VkDeviceSize* pSumFreeSize,
5648  VkDeviceSize* pSumItemSize) const
5649 {
5650  VMA_ASSERT(allocSize > 0);
5651  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5652  VMA_ASSERT(suballocItem != m_Suballocations.cend());
5653  VMA_ASSERT(pOffset != VMA_NULL);
5654 
5655  *itemsToMakeLostCount = 0;
5656  *pSumFreeSize = 0;
5657  *pSumItemSize = 0;
5658 
5659  if(canMakeOtherLost)
5660  {
5661  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5662  {
5663  *pSumFreeSize = suballocItem->size;
5664  }
5665  else
5666  {
5667  if(suballocItem->hAllocation->CanBecomeLost() &&
5668  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5669  {
5670  ++*itemsToMakeLostCount;
5671  *pSumItemSize = suballocItem->size;
5672  }
5673  else
5674  {
5675  return false;
5676  }
5677  }
5678 
5679  // Remaining size is too small for this request: Early return.
5680  if(m_Size - suballocItem->offset < allocSize)
5681  {
5682  return false;
5683  }
5684 
5685  // Start from offset equal to beginning of this suballocation.
5686  *pOffset = suballocItem->offset;
5687 
5688  // Apply VMA_DEBUG_MARGIN at the beginning.
5689  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5690  {
5691  *pOffset += VMA_DEBUG_MARGIN;
5692  }
5693 
5694  // Apply alignment.
5695  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5696  *pOffset = VmaAlignUp(*pOffset, alignment);
5697 
5698  // Check previous suballocations for BufferImageGranularity conflicts.
5699  // Make bigger alignment if necessary.
5700  if(bufferImageGranularity > 1)
5701  {
5702  bool bufferImageGranularityConflict = false;
5703  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5704  while(prevSuballocItem != m_Suballocations.cbegin())
5705  {
5706  --prevSuballocItem;
5707  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5708  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5709  {
5710  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5711  {
5712  bufferImageGranularityConflict = true;
5713  break;
5714  }
5715  }
5716  else
5717  // Already on previous page.
5718  break;
5719  }
5720  if(bufferImageGranularityConflict)
5721  {
5722  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5723  }
5724  }
5725 
5726  // Now that we have final *pOffset, check if we are past suballocItem.
5727  // If yes, return false - this function should be called for another suballocItem as starting point.
5728  if(*pOffset >= suballocItem->offset + suballocItem->size)
5729  {
5730  return false;
5731  }
5732 
5733  // Calculate padding at the beginning based on current offset.
5734  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
5735 
5736  // Calculate required margin at the end if this is not last suballocation.
5737  VmaSuballocationList::const_iterator next = suballocItem;
5738  ++next;
5739  const VkDeviceSize requiredEndMargin =
5740  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5741 
5742  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
5743  // Another early return check.
5744  if(suballocItem->offset + totalSize > m_Size)
5745  {
5746  return false;
5747  }
5748 
5749  // Advance lastSuballocItem until desired size is reached.
5750  // Update itemsToMakeLostCount.
5751  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
5752  if(totalSize > suballocItem->size)
5753  {
5754  VkDeviceSize remainingSize = totalSize - suballocItem->size;
5755  while(remainingSize > 0)
5756  {
5757  ++lastSuballocItem;
5758  if(lastSuballocItem == m_Suballocations.cend())
5759  {
5760  return false;
5761  }
5762  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5763  {
5764  *pSumFreeSize += lastSuballocItem->size;
5765  }
5766  else
5767  {
5768  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
5769  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
5770  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5771  {
5772  ++*itemsToMakeLostCount;
5773  *pSumItemSize += lastSuballocItem->size;
5774  }
5775  else
5776  {
5777  return false;
5778  }
5779  }
5780  remainingSize = (lastSuballocItem->size < remainingSize) ?
5781  remainingSize - lastSuballocItem->size : 0;
5782  }
5783  }
5784 
5785  // Check next suballocations for BufferImageGranularity conflicts.
5786  // If conflict exists, we must mark more allocations lost or fail.
5787  if(bufferImageGranularity > 1)
5788  {
5789  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
5790  ++nextSuballocItem;
5791  while(nextSuballocItem != m_Suballocations.cend())
5792  {
5793  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5794  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5795  {
5796  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5797  {
5798  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
5799  if(nextSuballoc.hAllocation->CanBecomeLost() &&
5800  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5801  {
5802  ++*itemsToMakeLostCount;
5803  }
5804  else
5805  {
5806  return false;
5807  }
5808  }
5809  }
5810  else
5811  {
5812  // Already on next page.
5813  break;
5814  }
5815  ++nextSuballocItem;
5816  }
5817  }
5818  }
5819  else
5820  {
5821  const VmaSuballocation& suballoc = *suballocItem;
5822  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5823 
5824  *pSumFreeSize = suballoc.size;
5825 
5826  // Size of this suballocation is too small for this request: Early return.
5827  if(suballoc.size < allocSize)
5828  {
5829  return false;
5830  }
5831 
5832  // Start from offset equal to beginning of this suballocation.
5833  *pOffset = suballoc.offset;
5834 
5835  // Apply VMA_DEBUG_MARGIN at the beginning.
5836  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5837  {
5838  *pOffset += VMA_DEBUG_MARGIN;
5839  }
5840 
5841  // Apply alignment.
5842  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5843  *pOffset = VmaAlignUp(*pOffset, alignment);
5844 
5845  // Check previous suballocations for BufferImageGranularity conflicts.
5846  // Make bigger alignment if necessary.
5847  if(bufferImageGranularity > 1)
5848  {
5849  bool bufferImageGranularityConflict = false;
5850  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5851  while(prevSuballocItem != m_Suballocations.cbegin())
5852  {
5853  --prevSuballocItem;
5854  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5855  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5856  {
5857  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5858  {
5859  bufferImageGranularityConflict = true;
5860  break;
5861  }
5862  }
5863  else
5864  // Already on previous page.
5865  break;
5866  }
5867  if(bufferImageGranularityConflict)
5868  {
5869  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5870  }
5871  }
5872 
5873  // Calculate padding at the beginning based on current offset.
5874  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
5875 
5876  // Calculate required margin at the end if this is not last suballocation.
5877  VmaSuballocationList::const_iterator next = suballocItem;
5878  ++next;
5879  const VkDeviceSize requiredEndMargin =
5880  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5881 
5882  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
5883  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
5884  {
5885  return false;
5886  }
5887 
5888  // Check next suballocations for BufferImageGranularity conflicts.
5889  // If conflict exists, allocation cannot be made here.
5890  if(bufferImageGranularity > 1)
5891  {
5892  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
5893  ++nextSuballocItem;
5894  while(nextSuballocItem != m_Suballocations.cend())
5895  {
5896  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5897  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5898  {
5899  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5900  {
5901  return false;
5902  }
5903  }
5904  else
5905  {
5906  // Already on next page.
5907  break;
5908  }
5909  ++nextSuballocItem;
5910  }
5911  }
5912  }
5913 
5914  // All tests passed: Success. pOffset is already filled.
5915  return true;
5916 }
5917 
5918 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
5919 {
5920  VMA_ASSERT(item != m_Suballocations.end());
5921  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5922 
5923  VmaSuballocationList::iterator nextItem = item;
5924  ++nextItem;
5925  VMA_ASSERT(nextItem != m_Suballocations.end());
5926  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
5927 
5928  item->size += nextItem->size;
5929  --m_FreeCount;
5930  m_Suballocations.erase(nextItem);
5931 }
5932 
5933 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
5934 {
5935  // Change this suballocation to be marked as free.
5936  VmaSuballocation& suballoc = *suballocItem;
5937  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5938  suballoc.hAllocation = VK_NULL_HANDLE;
5939 
5940  // Update totals.
5941  ++m_FreeCount;
5942  m_SumFreeSize += suballoc.size;
5943 
5944  // Merge with previous and/or next suballocation if it's also free.
5945  bool mergeWithNext = false;
5946  bool mergeWithPrev = false;
5947 
5948  VmaSuballocationList::iterator nextItem = suballocItem;
5949  ++nextItem;
5950  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
5951  {
5952  mergeWithNext = true;
5953  }
5954 
5955  VmaSuballocationList::iterator prevItem = suballocItem;
5956  if(suballocItem != m_Suballocations.begin())
5957  {
5958  --prevItem;
5959  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5960  {
5961  mergeWithPrev = true;
5962  }
5963  }
5964 
5965  if(mergeWithNext)
5966  {
5967  UnregisterFreeSuballocation(nextItem);
5968  MergeFreeWithNext(suballocItem);
5969  }
5970 
5971  if(mergeWithPrev)
5972  {
5973  UnregisterFreeSuballocation(prevItem);
5974  MergeFreeWithNext(prevItem);
5975  RegisterFreeSuballocation(prevItem);
5976  return prevItem;
5977  }
5978  else
5979  {
5980  RegisterFreeSuballocation(suballocItem);
5981  return suballocItem;
5982  }
5983 }
5984 
5985 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
5986 {
5987  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5988  VMA_ASSERT(item->size > 0);
5989 
5990  // You may want to enable this validation at the beginning or at the end of
5991  // this function, depending on what do you want to check.
5992  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5993 
5994  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5995  {
5996  if(m_FreeSuballocationsBySize.empty())
5997  {
5998  m_FreeSuballocationsBySize.push_back(item);
5999  }
6000  else
6001  {
6002  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
6003  }
6004  }
6005 
6006  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6007 }
6008 
6009 
6010 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
6011 {
6012  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6013  VMA_ASSERT(item->size > 0);
6014 
6015  // You may want to enable this validation at the beginning or at the end of
6016  // this function, depending on what do you want to check.
6017  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6018 
6019  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6020  {
6021  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
6022  m_FreeSuballocationsBySize.data(),
6023  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
6024  item,
6025  VmaSuballocationItemSizeLess());
6026  for(size_t index = it - m_FreeSuballocationsBySize.data();
6027  index < m_FreeSuballocationsBySize.size();
6028  ++index)
6029  {
6030  if(m_FreeSuballocationsBySize[index] == item)
6031  {
6032  VmaVectorRemove(m_FreeSuballocationsBySize, index);
6033  return;
6034  }
6035  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
6036  }
6037  VMA_ASSERT(0 && "Not found.");
6038  }
6039 
6040  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6041 }
6042 
6044 // class VmaDeviceMemoryMapping
6045 
6046 VmaDeviceMemoryMapping::VmaDeviceMemoryMapping() :
6047  m_MapCount(0),
6048  m_pMappedData(VMA_NULL)
6049 {
6050 }
6051 
6052 VmaDeviceMemoryMapping::~VmaDeviceMemoryMapping()
6053 {
6054  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
6055 }
6056 
6057 VkResult VmaDeviceMemoryMapping::Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData)
6058 {
6059  if(count == 0)
6060  {
6061  return VK_SUCCESS;
6062  }
6063 
6064  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6065  if(m_MapCount != 0)
6066  {
6067  m_MapCount += count;
6068  VMA_ASSERT(m_pMappedData != VMA_NULL);
6069  if(ppData != VMA_NULL)
6070  {
6071  *ppData = m_pMappedData;
6072  }
6073  return VK_SUCCESS;
6074  }
6075  else
6076  {
6077  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
6078  hAllocator->m_hDevice,
6079  hMemory,
6080  0, // offset
6081  VK_WHOLE_SIZE,
6082  0, // flags
6083  &m_pMappedData);
6084  if(result == VK_SUCCESS)
6085  {
6086  if(ppData != VMA_NULL)
6087  {
6088  *ppData = m_pMappedData;
6089  }
6090  m_MapCount = count;
6091  }
6092  return result;
6093  }
6094 }
6095 
6096 void VmaDeviceMemoryMapping::Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count)
6097 {
6098  if(count == 0)
6099  {
6100  return;
6101  }
6102 
6103  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6104  if(m_MapCount >= count)
6105  {
6106  m_MapCount -= count;
6107  if(m_MapCount == 0)
6108  {
6109  m_pMappedData = VMA_NULL;
6110  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, hMemory);
6111  }
6112  }
6113  else
6114  {
6115  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
6116  }
6117 }
6118 
6120 // class VmaDeviceMemoryBlock
6121 
6122 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
6123  m_MemoryTypeIndex(UINT32_MAX),
6124  m_hMemory(VK_NULL_HANDLE),
6125  m_Metadata(hAllocator)
6126 {
6127 }
6128 
6129 void VmaDeviceMemoryBlock::Init(
6130  uint32_t newMemoryTypeIndex,
6131  VkDeviceMemory newMemory,
6132  VkDeviceSize newSize)
6133 {
6134  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
6135 
6136  m_MemoryTypeIndex = newMemoryTypeIndex;
6137  m_hMemory = newMemory;
6138 
6139  m_Metadata.Init(newSize);
6140 }
6141 
6142 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
6143 {
6144  // This is the most important assert in the entire library.
6145  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
6146  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
6147 
6148  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
6149  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
6150  m_hMemory = VK_NULL_HANDLE;
6151 }
6152 
6153 bool VmaDeviceMemoryBlock::Validate() const
6154 {
6155  if((m_hMemory == VK_NULL_HANDLE) ||
6156  (m_Metadata.GetSize() == 0))
6157  {
6158  return false;
6159  }
6160 
6161  return m_Metadata.Validate();
6162 }
6163 
6164 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
6165 {
6166  return m_Mapping.Map(hAllocator, m_hMemory, count, ppData);
6167 }
6168 
6169 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
6170 {
6171  m_Mapping.Unmap(hAllocator, m_hMemory, count);
6172 }
6173 
6174 static void InitStatInfo(VmaStatInfo& outInfo)
6175 {
6176  memset(&outInfo, 0, sizeof(outInfo));
6177  outInfo.allocationSizeMin = UINT64_MAX;
6178  outInfo.unusedRangeSizeMin = UINT64_MAX;
6179 }
6180 
6181 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
6182 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
6183 {
6184  inoutInfo.blockCount += srcInfo.blockCount;
6185  inoutInfo.allocationCount += srcInfo.allocationCount;
6186  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
6187  inoutInfo.usedBytes += srcInfo.usedBytes;
6188  inoutInfo.unusedBytes += srcInfo.unusedBytes;
6189  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
6190  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
6191  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
6192  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
6193 }
6194 
6195 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
6196 {
6197  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
6198  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
6199  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
6200  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
6201 }
6202 
6203 VmaPool_T::VmaPool_T(
6204  VmaAllocator hAllocator,
6205  const VmaPoolCreateInfo& createInfo) :
6206  m_BlockVector(
6207  hAllocator,
6208  createInfo.memoryTypeIndex,
6209  createInfo.blockSize,
6210  createInfo.minBlockCount,
6211  createInfo.maxBlockCount,
6212  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
6213  createInfo.frameInUseCount,
6214  true) // isCustomPool
6215 {
6216 }
6217 
6218 VmaPool_T::~VmaPool_T()
6219 {
6220 }
6221 
6222 #if VMA_STATS_STRING_ENABLED
6223 
6224 #endif // #if VMA_STATS_STRING_ENABLED
6225 
6226 VmaBlockVector::VmaBlockVector(
6227  VmaAllocator hAllocator,
6228  uint32_t memoryTypeIndex,
6229  VkDeviceSize preferredBlockSize,
6230  size_t minBlockCount,
6231  size_t maxBlockCount,
6232  VkDeviceSize bufferImageGranularity,
6233  uint32_t frameInUseCount,
6234  bool isCustomPool) :
6235  m_hAllocator(hAllocator),
6236  m_MemoryTypeIndex(memoryTypeIndex),
6237  m_PreferredBlockSize(preferredBlockSize),
6238  m_MinBlockCount(minBlockCount),
6239  m_MaxBlockCount(maxBlockCount),
6240  m_BufferImageGranularity(bufferImageGranularity),
6241  m_FrameInUseCount(frameInUseCount),
6242  m_IsCustomPool(isCustomPool),
6243  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
6244  m_HasEmptyBlock(false),
6245  m_pDefragmentator(VMA_NULL)
6246 {
6247 }
6248 
6249 VmaBlockVector::~VmaBlockVector()
6250 {
6251  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
6252 
6253  for(size_t i = m_Blocks.size(); i--; )
6254  {
6255  m_Blocks[i]->Destroy(m_hAllocator);
6256  vma_delete(m_hAllocator, m_Blocks[i]);
6257  }
6258 }
6259 
6260 VkResult VmaBlockVector::CreateMinBlocks()
6261 {
6262  for(size_t i = 0; i < m_MinBlockCount; ++i)
6263  {
6264  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
6265  if(res != VK_SUCCESS)
6266  {
6267  return res;
6268  }
6269  }
6270  return VK_SUCCESS;
6271 }
6272 
6273 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
6274 {
6275  pStats->size = 0;
6276  pStats->unusedSize = 0;
6277  pStats->allocationCount = 0;
6278  pStats->unusedRangeCount = 0;
6279  pStats->unusedRangeSizeMax = 0;
6280 
6281  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6282 
6283  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6284  {
6285  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6286  VMA_ASSERT(pBlock);
6287  VMA_HEAVY_ASSERT(pBlock->Validate());
6288  pBlock->m_Metadata.AddPoolStats(*pStats);
6289  }
6290 }
6291 
6292 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
6293 
6294 VkResult VmaBlockVector::Allocate(
6295  VmaPool hCurrentPool,
6296  uint32_t currentFrameIndex,
6297  const VkMemoryRequirements& vkMemReq,
6298  const VmaAllocationCreateInfo& createInfo,
6299  VmaSuballocationType suballocType,
6300  VmaAllocation* pAllocation)
6301 {
6302  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
6303  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
6304 
6305  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6306 
6307  // 1. Search existing allocations. Try to allocate without making other allocations lost.
6308  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6309  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6310  {
6311  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6312  VMA_ASSERT(pCurrBlock);
6313  VmaAllocationRequest currRequest = {};
6314  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6315  currentFrameIndex,
6316  m_FrameInUseCount,
6317  m_BufferImageGranularity,
6318  vkMemReq.size,
6319  vkMemReq.alignment,
6320  suballocType,
6321  false, // canMakeOtherLost
6322  &currRequest))
6323  {
6324  // Allocate from pCurrBlock.
6325  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
6326 
6327  if(mapped)
6328  {
6329  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
6330  if(res != VK_SUCCESS)
6331  {
6332  return res;
6333  }
6334  }
6335 
6336  // We no longer have an empty Allocation.
6337  if(pCurrBlock->m_Metadata.IsEmpty())
6338  {
6339  m_HasEmptyBlock = false;
6340  }
6341 
6342  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6343  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
6344  (*pAllocation)->InitBlockAllocation(
6345  hCurrentPool,
6346  pCurrBlock,
6347  currRequest.offset,
6348  vkMemReq.alignment,
6349  vkMemReq.size,
6350  suballocType,
6351  mapped,
6352  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6353  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
6354  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6355  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6356  return VK_SUCCESS;
6357  }
6358  }
6359 
6360  const bool canCreateNewBlock =
6361  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
6362  (m_Blocks.size() < m_MaxBlockCount);
6363 
6364  // 2. Try to create new block.
6365  if(canCreateNewBlock)
6366  {
6367  // Calculate optimal size for new block.
6368  VkDeviceSize newBlockSize = m_PreferredBlockSize;
6369  uint32_t newBlockSizeShift = 0;
6370  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
6371 
6372  // Allocating blocks of other sizes is allowed only in default pools.
6373  // In custom pools block size is fixed.
6374  if(m_IsCustomPool == false)
6375  {
6376  // Allocate 1/8, 1/4, 1/2 as first blocks.
6377  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
6378  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
6379  {
6380  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6381  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= vkMemReq.size * 2)
6382  {
6383  newBlockSize = smallerNewBlockSize;
6384  ++newBlockSizeShift;
6385  }
6386  else
6387  {
6388  break;
6389  }
6390  }
6391  }
6392 
6393  size_t newBlockIndex = 0;
6394  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
6395  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
6396  if(m_IsCustomPool == false)
6397  {
6398  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
6399  {
6400  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6401  if(smallerNewBlockSize >= vkMemReq.size)
6402  {
6403  newBlockSize = smallerNewBlockSize;
6404  ++newBlockSizeShift;
6405  res = CreateBlock(newBlockSize, &newBlockIndex);
6406  }
6407  else
6408  {
6409  break;
6410  }
6411  }
6412  }
6413 
6414  if(res == VK_SUCCESS)
6415  {
6416  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
6417  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= vkMemReq.size);
6418 
6419  if(mapped)
6420  {
6421  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
6422  if(res != VK_SUCCESS)
6423  {
6424  return res;
6425  }
6426  }
6427 
6428  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
6429  VmaAllocationRequest allocRequest;
6430  pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
6431  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6432  pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
6433  (*pAllocation)->InitBlockAllocation(
6434  hCurrentPool,
6435  pBlock,
6436  allocRequest.offset,
6437  vkMemReq.alignment,
6438  vkMemReq.size,
6439  suballocType,
6440  mapped,
6441  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6442  VMA_HEAVY_ASSERT(pBlock->Validate());
6443  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
6444  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6445  return VK_SUCCESS;
6446  }
6447  }
6448 
6449  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
6450 
6451  // 3. Try to allocate from existing blocks with making other allocations lost.
6452  if(canMakeOtherLost)
6453  {
6454  uint32_t tryIndex = 0;
6455  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
6456  {
6457  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
6458  VmaAllocationRequest bestRequest = {};
6459  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
6460 
6461  // 1. Search existing allocations.
6462  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6463  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6464  {
6465  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6466  VMA_ASSERT(pCurrBlock);
6467  VmaAllocationRequest currRequest = {};
6468  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6469  currentFrameIndex,
6470  m_FrameInUseCount,
6471  m_BufferImageGranularity,
6472  vkMemReq.size,
6473  vkMemReq.alignment,
6474  suballocType,
6475  canMakeOtherLost,
6476  &currRequest))
6477  {
6478  const VkDeviceSize currRequestCost = currRequest.CalcCost();
6479  if(pBestRequestBlock == VMA_NULL ||
6480  currRequestCost < bestRequestCost)
6481  {
6482  pBestRequestBlock = pCurrBlock;
6483  bestRequest = currRequest;
6484  bestRequestCost = currRequestCost;
6485 
6486  if(bestRequestCost == 0)
6487  {
6488  break;
6489  }
6490  }
6491  }
6492  }
6493 
6494  if(pBestRequestBlock != VMA_NULL)
6495  {
6496  if(mapped)
6497  {
6498  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
6499  if(res != VK_SUCCESS)
6500  {
6501  return res;
6502  }
6503  }
6504 
6505  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
6506  currentFrameIndex,
6507  m_FrameInUseCount,
6508  &bestRequest))
6509  {
6510  // We no longer have an empty Allocation.
6511  if(pBestRequestBlock->m_Metadata.IsEmpty())
6512  {
6513  m_HasEmptyBlock = false;
6514  }
6515  // Allocate from this pBlock.
6516  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6517  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
6518  (*pAllocation)->InitBlockAllocation(
6519  hCurrentPool,
6520  pBestRequestBlock,
6521  bestRequest.offset,
6522  vkMemReq.alignment,
6523  vkMemReq.size,
6524  suballocType,
6525  mapped,
6526  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6527  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
6528  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6529  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6530  return VK_SUCCESS;
6531  }
6532  // else: Some allocations must have been touched while we are here. Next try.
6533  }
6534  else
6535  {
6536  // Could not find place in any of the blocks - break outer loop.
6537  break;
6538  }
6539  }
6540  /* Maximum number of tries exceeded - a very unlike event when many other
6541  threads are simultaneously touching allocations making it impossible to make
6542  lost at the same time as we try to allocate. */
6543  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
6544  {
6545  return VK_ERROR_TOO_MANY_OBJECTS;
6546  }
6547  }
6548 
6549  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6550 }
6551 
6552 void VmaBlockVector::Free(
6553  VmaAllocation hAllocation)
6554 {
6555  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
6556 
6557  // Scope for lock.
6558  {
6559  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6560 
6561  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
6562 
6563  if(hAllocation->IsPersistentMap())
6564  {
6565  pBlock->m_Mapping.Unmap(m_hAllocator, pBlock->m_hMemory, 1);
6566  }
6567 
6568  pBlock->m_Metadata.Free(hAllocation);
6569  VMA_HEAVY_ASSERT(pBlock->Validate());
6570 
6571  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
6572 
6573  // pBlock became empty after this deallocation.
6574  if(pBlock->m_Metadata.IsEmpty())
6575  {
6576  // Already has empty Allocation. We don't want to have two, so delete this one.
6577  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
6578  {
6579  pBlockToDelete = pBlock;
6580  Remove(pBlock);
6581  }
6582  // We now have first empty Allocation.
6583  else
6584  {
6585  m_HasEmptyBlock = true;
6586  }
6587  }
6588  // pBlock didn't become empty, but we have another empty block - find and free that one.
6589  // (This is optional, heuristics.)
6590  else if(m_HasEmptyBlock)
6591  {
6592  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
6593  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
6594  {
6595  pBlockToDelete = pLastBlock;
6596  m_Blocks.pop_back();
6597  m_HasEmptyBlock = false;
6598  }
6599  }
6600 
6601  IncrementallySortBlocks();
6602  }
6603 
6604  // Destruction of a free Allocation. Deferred until this point, outside of mutex
6605  // lock, for performance reason.
6606  if(pBlockToDelete != VMA_NULL)
6607  {
6608  VMA_DEBUG_LOG(" Deleted empty allocation");
6609  pBlockToDelete->Destroy(m_hAllocator);
6610  vma_delete(m_hAllocator, pBlockToDelete);
6611  }
6612 }
6613 
6614 size_t VmaBlockVector::CalcMaxBlockSize() const
6615 {
6616  size_t result = 0;
6617  for(size_t i = m_Blocks.size(); i--; )
6618  {
6619  result = VMA_MAX((uint64_t)result, (uint64_t)m_Blocks[i]->m_Metadata.GetSize());
6620  if(result >= m_PreferredBlockSize)
6621  {
6622  break;
6623  }
6624  }
6625  return result;
6626 }
6627 
6628 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
6629 {
6630  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6631  {
6632  if(m_Blocks[blockIndex] == pBlock)
6633  {
6634  VmaVectorRemove(m_Blocks, blockIndex);
6635  return;
6636  }
6637  }
6638  VMA_ASSERT(0);
6639 }
6640 
6641 void VmaBlockVector::IncrementallySortBlocks()
6642 {
6643  // Bubble sort only until first swap.
6644  for(size_t i = 1; i < m_Blocks.size(); ++i)
6645  {
6646  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
6647  {
6648  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
6649  return;
6650  }
6651  }
6652 }
6653 
6654 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
6655 {
6656  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6657  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
6658  allocInfo.allocationSize = blockSize;
6659  VkDeviceMemory mem = VK_NULL_HANDLE;
6660  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
6661  if(res < 0)
6662  {
6663  return res;
6664  }
6665 
6666  // New VkDeviceMemory successfully created.
6667 
6668  // Create new Allocation for it.
6669  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
6670  pBlock->Init(
6671  m_MemoryTypeIndex,
6672  mem,
6673  allocInfo.allocationSize);
6674 
6675  m_Blocks.push_back(pBlock);
6676  if(pNewBlockIndex != VMA_NULL)
6677  {
6678  *pNewBlockIndex = m_Blocks.size() - 1;
6679  }
6680 
6681  return VK_SUCCESS;
6682 }
6683 
6684 #if VMA_STATS_STRING_ENABLED
6685 
6686 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
6687 {
6688  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6689 
6690  json.BeginObject();
6691 
6692  if(m_IsCustomPool)
6693  {
6694  json.WriteString("MemoryTypeIndex");
6695  json.WriteNumber(m_MemoryTypeIndex);
6696 
6697  json.WriteString("BlockSize");
6698  json.WriteNumber(m_PreferredBlockSize);
6699 
6700  json.WriteString("BlockCount");
6701  json.BeginObject(true);
6702  if(m_MinBlockCount > 0)
6703  {
6704  json.WriteString("Min");
6705  json.WriteNumber((uint64_t)m_MinBlockCount);
6706  }
6707  if(m_MaxBlockCount < SIZE_MAX)
6708  {
6709  json.WriteString("Max");
6710  json.WriteNumber((uint64_t)m_MaxBlockCount);
6711  }
6712  json.WriteString("Cur");
6713  json.WriteNumber((uint64_t)m_Blocks.size());
6714  json.EndObject();
6715 
6716  if(m_FrameInUseCount > 0)
6717  {
6718  json.WriteString("FrameInUseCount");
6719  json.WriteNumber(m_FrameInUseCount);
6720  }
6721  }
6722  else
6723  {
6724  json.WriteString("PreferredBlockSize");
6725  json.WriteNumber(m_PreferredBlockSize);
6726  }
6727 
6728  json.WriteString("Blocks");
6729  json.BeginArray();
6730  for(size_t i = 0; i < m_Blocks.size(); ++i)
6731  {
6732  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
6733  }
6734  json.EndArray();
6735 
6736  json.EndObject();
6737 }
6738 
6739 #endif // #if VMA_STATS_STRING_ENABLED
6740 
6741 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
6742  VmaAllocator hAllocator,
6743  uint32_t currentFrameIndex)
6744 {
6745  if(m_pDefragmentator == VMA_NULL)
6746  {
6747  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
6748  hAllocator,
6749  this,
6750  currentFrameIndex);
6751  }
6752 
6753  return m_pDefragmentator;
6754 }
6755 
6756 VkResult VmaBlockVector::Defragment(
6757  VmaDefragmentationStats* pDefragmentationStats,
6758  VkDeviceSize& maxBytesToMove,
6759  uint32_t& maxAllocationsToMove)
6760 {
6761  if(m_pDefragmentator == VMA_NULL)
6762  {
6763  return VK_SUCCESS;
6764  }
6765 
6766  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6767 
6768  // Defragment.
6769  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
6770 
6771  // Accumulate statistics.
6772  if(pDefragmentationStats != VMA_NULL)
6773  {
6774  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
6775  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
6776  pDefragmentationStats->bytesMoved += bytesMoved;
6777  pDefragmentationStats->allocationsMoved += allocationsMoved;
6778  VMA_ASSERT(bytesMoved <= maxBytesToMove);
6779  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
6780  maxBytesToMove -= bytesMoved;
6781  maxAllocationsToMove -= allocationsMoved;
6782  }
6783 
6784  // Free empty blocks.
6785  m_HasEmptyBlock = false;
6786  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
6787  {
6788  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
6789  if(pBlock->m_Metadata.IsEmpty())
6790  {
6791  if(m_Blocks.size() > m_MinBlockCount)
6792  {
6793  if(pDefragmentationStats != VMA_NULL)
6794  {
6795  ++pDefragmentationStats->deviceMemoryBlocksFreed;
6796  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
6797  }
6798 
6799  VmaVectorRemove(m_Blocks, blockIndex);
6800  pBlock->Destroy(m_hAllocator);
6801  vma_delete(m_hAllocator, pBlock);
6802  }
6803  else
6804  {
6805  m_HasEmptyBlock = true;
6806  }
6807  }
6808  }
6809 
6810  return result;
6811 }
6812 
6813 void VmaBlockVector::DestroyDefragmentator()
6814 {
6815  if(m_pDefragmentator != VMA_NULL)
6816  {
6817  vma_delete(m_hAllocator, m_pDefragmentator);
6818  m_pDefragmentator = VMA_NULL;
6819  }
6820 }
6821 
6822 void VmaBlockVector::MakePoolAllocationsLost(
6823  uint32_t currentFrameIndex,
6824  size_t* pLostAllocationCount)
6825 {
6826  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6827  size_t lostAllocationCount = 0;
6828  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6829  {
6830  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6831  VMA_ASSERT(pBlock);
6832  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
6833  }
6834  if(pLostAllocationCount != VMA_NULL)
6835  {
6836  *pLostAllocationCount = lostAllocationCount;
6837  }
6838 }
6839 
6840 void VmaBlockVector::AddStats(VmaStats* pStats)
6841 {
6842  const uint32_t memTypeIndex = m_MemoryTypeIndex;
6843  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
6844 
6845  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6846 
6847  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6848  {
6849  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6850  VMA_ASSERT(pBlock);
6851  VMA_HEAVY_ASSERT(pBlock->Validate());
6852  VmaStatInfo allocationStatInfo;
6853  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
6854  VmaAddStatInfo(pStats->total, allocationStatInfo);
6855  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
6856  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
6857  }
6858 }
6859 
6861 // VmaDefragmentator members definition
6862 
6863 VmaDefragmentator::VmaDefragmentator(
6864  VmaAllocator hAllocator,
6865  VmaBlockVector* pBlockVector,
6866  uint32_t currentFrameIndex) :
6867  m_hAllocator(hAllocator),
6868  m_pBlockVector(pBlockVector),
6869  m_CurrentFrameIndex(currentFrameIndex),
6870  m_BytesMoved(0),
6871  m_AllocationsMoved(0),
6872  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
6873  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
6874 {
6875 }
6876 
6877 VmaDefragmentator::~VmaDefragmentator()
6878 {
6879  for(size_t i = m_Blocks.size(); i--; )
6880  {
6881  vma_delete(m_hAllocator, m_Blocks[i]);
6882  }
6883 }
6884 
6885 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
6886 {
6887  AllocationInfo allocInfo;
6888  allocInfo.m_hAllocation = hAlloc;
6889  allocInfo.m_pChanged = pChanged;
6890  m_Allocations.push_back(allocInfo);
6891 }
6892 
6893 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
6894 {
6895  // It has already been mapped for defragmentation.
6896  if(m_pMappedDataForDefragmentation)
6897  {
6898  *ppMappedData = m_pMappedDataForDefragmentation;
6899  return VK_SUCCESS;
6900  }
6901 
6902  // It is originally mapped.
6903  if(m_pBlock->m_Mapping.GetMappedData())
6904  {
6905  *ppMappedData = m_pBlock->m_Mapping.GetMappedData();
6906  return VK_SUCCESS;
6907  }
6908 
6909  // Map on first usage.
6910  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
6911  *ppMappedData = m_pMappedDataForDefragmentation;
6912  return res;
6913 }
6914 
6915 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
6916 {
6917  if(m_pMappedDataForDefragmentation != VMA_NULL)
6918  {
6919  m_pBlock->Unmap(hAllocator, 1);
6920  }
6921 }
6922 
6923 VkResult VmaDefragmentator::DefragmentRound(
6924  VkDeviceSize maxBytesToMove,
6925  uint32_t maxAllocationsToMove)
6926 {
6927  if(m_Blocks.empty())
6928  {
6929  return VK_SUCCESS;
6930  }
6931 
6932  size_t srcBlockIndex = m_Blocks.size() - 1;
6933  size_t srcAllocIndex = SIZE_MAX;
6934  for(;;)
6935  {
6936  // 1. Find next allocation to move.
6937  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
6938  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
6939  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
6940  {
6941  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
6942  {
6943  // Finished: no more allocations to process.
6944  if(srcBlockIndex == 0)
6945  {
6946  return VK_SUCCESS;
6947  }
6948  else
6949  {
6950  --srcBlockIndex;
6951  srcAllocIndex = SIZE_MAX;
6952  }
6953  }
6954  else
6955  {
6956  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
6957  }
6958  }
6959 
6960  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
6961  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
6962 
6963  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
6964  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
6965  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
6966  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
6967 
6968  // 2. Try to find new place for this allocation in preceding or current block.
6969  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
6970  {
6971  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
6972  VmaAllocationRequest dstAllocRequest;
6973  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
6974  m_CurrentFrameIndex,
6975  m_pBlockVector->GetFrameInUseCount(),
6976  m_pBlockVector->GetBufferImageGranularity(),
6977  size,
6978  alignment,
6979  suballocType,
6980  false, // canMakeOtherLost
6981  &dstAllocRequest) &&
6982  MoveMakesSense(
6983  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
6984  {
6985  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
6986 
6987  // Reached limit on number of allocations or bytes to move.
6988  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
6989  (m_BytesMoved + size > maxBytesToMove))
6990  {
6991  return VK_INCOMPLETE;
6992  }
6993 
6994  void* pDstMappedData = VMA_NULL;
6995  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
6996  if(res != VK_SUCCESS)
6997  {
6998  return res;
6999  }
7000 
7001  void* pSrcMappedData = VMA_NULL;
7002  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
7003  if(res != VK_SUCCESS)
7004  {
7005  return res;
7006  }
7007 
7008  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
7009  memcpy(
7010  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
7011  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
7012  static_cast<size_t>(size));
7013 
7014  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
7015  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
7016 
7017  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
7018 
7019  if(allocInfo.m_pChanged != VMA_NULL)
7020  {
7021  *allocInfo.m_pChanged = VK_TRUE;
7022  }
7023 
7024  ++m_AllocationsMoved;
7025  m_BytesMoved += size;
7026 
7027  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
7028 
7029  break;
7030  }
7031  }
7032 
7033  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
7034 
7035  if(srcAllocIndex > 0)
7036  {
7037  --srcAllocIndex;
7038  }
7039  else
7040  {
7041  if(srcBlockIndex > 0)
7042  {
7043  --srcBlockIndex;
7044  srcAllocIndex = SIZE_MAX;
7045  }
7046  else
7047  {
7048  return VK_SUCCESS;
7049  }
7050  }
7051  }
7052 }
7053 
7054 VkResult VmaDefragmentator::Defragment(
7055  VkDeviceSize maxBytesToMove,
7056  uint32_t maxAllocationsToMove)
7057 {
7058  if(m_Allocations.empty())
7059  {
7060  return VK_SUCCESS;
7061  }
7062 
7063  // Create block info for each block.
7064  const size_t blockCount = m_pBlockVector->m_Blocks.size();
7065  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
7066  {
7067  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
7068  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
7069  m_Blocks.push_back(pBlockInfo);
7070  }
7071 
7072  // Sort them by m_pBlock pointer value.
7073  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
7074 
7075  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
7076  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
7077  {
7078  AllocationInfo& allocInfo = m_Allocations[blockIndex];
7079  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
7080  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
7081  {
7082  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
7083  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
7084  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
7085  {
7086  (*it)->m_Allocations.push_back(allocInfo);
7087  }
7088  else
7089  {
7090  VMA_ASSERT(0);
7091  }
7092  }
7093  }
7094  m_Allocations.clear();
7095 
7096  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
7097  {
7098  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
7099  pBlockInfo->CalcHasNonMovableAllocations();
7100  pBlockInfo->SortAllocationsBySizeDescecnding();
7101  }
7102 
7103  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
7104  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
7105 
7106  // Execute defragmentation rounds (the main part).
7107  VkResult result = VK_SUCCESS;
7108  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
7109  {
7110  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
7111  }
7112 
7113  // Unmap blocks that were mapped for defragmentation.
7114  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
7115  {
7116  m_Blocks[blockIndex]->Unmap(m_hAllocator);
7117  }
7118 
7119  return result;
7120 }
7121 
7122 bool VmaDefragmentator::MoveMakesSense(
7123  size_t dstBlockIndex, VkDeviceSize dstOffset,
7124  size_t srcBlockIndex, VkDeviceSize srcOffset)
7125 {
7126  if(dstBlockIndex < srcBlockIndex)
7127  {
7128  return true;
7129  }
7130  if(dstBlockIndex > srcBlockIndex)
7131  {
7132  return false;
7133  }
7134  if(dstOffset < srcOffset)
7135  {
7136  return true;
7137  }
7138  return false;
7139 }
7140 
7142 // VmaAllocator_T
7143 
7144 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
7145  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
7146  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
7147  m_hDevice(pCreateInfo->device),
7148  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
7149  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
7150  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
7151  m_PreferredLargeHeapBlockSize(0),
7152  m_PhysicalDevice(pCreateInfo->physicalDevice),
7153  m_CurrentFrameIndex(0),
7154  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
7155 {
7156  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
7157 
7158  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
7159  memset(&m_MemProps, 0, sizeof(m_MemProps));
7160  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
7161 
7162  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
7163  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
7164 
7165  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
7166  {
7167  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
7168  }
7169 
7170  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
7171  {
7172  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
7173  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
7174  }
7175 
7176  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
7177 
7178  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
7179  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
7180 
7181  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
7182  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
7183 
7184  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
7185  {
7186  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
7187  {
7188  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
7189  if(limit != VK_WHOLE_SIZE)
7190  {
7191  m_HeapSizeLimit[heapIndex] = limit;
7192  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
7193  {
7194  m_MemProps.memoryHeaps[heapIndex].size = limit;
7195  }
7196  }
7197  }
7198  }
7199 
7200  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7201  {
7202  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
7203 
7204  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
7205  this,
7206  memTypeIndex,
7207  preferredBlockSize,
7208  0,
7209  SIZE_MAX,
7210  GetBufferImageGranularity(),
7211  pCreateInfo->frameInUseCount,
7212  false); // isCustomPool
7213  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
7214  // becase minBlockCount is 0.
7215  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
7216  }
7217 }
7218 
7219 VmaAllocator_T::~VmaAllocator_T()
7220 {
7221  VMA_ASSERT(m_Pools.empty());
7222 
7223  for(size_t i = GetMemoryTypeCount(); i--; )
7224  {
7225  vma_delete(this, m_pDedicatedAllocations[i]);
7226  vma_delete(this, m_pBlockVectors[i]);
7227  }
7228 }
7229 
7230 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
7231 {
7232 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7233  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
7234  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
7235  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
7236  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
7237  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
7238  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
7239  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
7240  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
7241  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
7242  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
7243  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
7244  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
7245  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
7246  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
7247  if(m_UseKhrDedicatedAllocation)
7248  {
7249  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
7250  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
7251  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
7252  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
7253  }
7254 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7255 
7256 #define VMA_COPY_IF_NOT_NULL(funcName) \
7257  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
7258 
7259  if(pVulkanFunctions != VMA_NULL)
7260  {
7261  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
7262  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
7263  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
7264  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
7265  VMA_COPY_IF_NOT_NULL(vkMapMemory);
7266  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
7267  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
7268  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
7269  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
7270  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
7271  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
7272  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
7273  VMA_COPY_IF_NOT_NULL(vkCreateImage);
7274  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
7275  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
7276  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
7277  }
7278 
7279 #undef VMA_COPY_IF_NOT_NULL
7280 
7281  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
7282  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
7283  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
7284  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
7285  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
7286  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
7287  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
7288  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
7289  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
7290  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
7291  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
7292  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
7293  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
7294  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
7295  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
7296  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
7297  if(m_UseKhrDedicatedAllocation)
7298  {
7299  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
7300  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
7301  }
7302 }
7303 
7304 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
7305 {
7306  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7307  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
7308  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
7309  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
7310 }
7311 
7312 VkResult VmaAllocator_T::AllocateMemoryOfType(
7313  const VkMemoryRequirements& vkMemReq,
7314  bool dedicatedAllocation,
7315  VkBuffer dedicatedBuffer,
7316  VkImage dedicatedImage,
7317  const VmaAllocationCreateInfo& createInfo,
7318  uint32_t memTypeIndex,
7319  VmaSuballocationType suballocType,
7320  VmaAllocation* pAllocation)
7321 {
7322  VMA_ASSERT(pAllocation != VMA_NULL);
7323  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
7324 
7325  VmaAllocationCreateInfo finalCreateInfo = createInfo;
7326 
7327  // If memory type is not HOST_VISIBLE, disable MAPPED.
7328  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7329  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
7330  {
7331  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
7332  }
7333 
7334  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
7335  VMA_ASSERT(blockVector);
7336 
7337  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
7338  bool preferDedicatedMemory =
7339  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
7340  dedicatedAllocation ||
7341  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
7342  vkMemReq.size > preferredBlockSize / 2;
7343 
7344  if(preferDedicatedMemory &&
7345  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
7346  finalCreateInfo.pool == VK_NULL_HANDLE)
7347  {
7348  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
7349  }
7350 
7351  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
7352  {
7353  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7354  {
7355  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7356  }
7357  else
7358  {
7359  return AllocateDedicatedMemory(
7360  vkMemReq.size,
7361  suballocType,
7362  memTypeIndex,
7363  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7364  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7365  finalCreateInfo.pUserData,
7366  dedicatedBuffer,
7367  dedicatedImage,
7368  pAllocation);
7369  }
7370  }
7371  else
7372  {
7373  VkResult res = blockVector->Allocate(
7374  VK_NULL_HANDLE, // hCurrentPool
7375  m_CurrentFrameIndex.load(),
7376  vkMemReq,
7377  finalCreateInfo,
7378  suballocType,
7379  pAllocation);
7380  if(res == VK_SUCCESS)
7381  {
7382  return res;
7383  }
7384 
7385  // 5. Try dedicated memory.
7386  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7387  {
7388  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7389  }
7390  else
7391  {
7392  res = AllocateDedicatedMemory(
7393  vkMemReq.size,
7394  suballocType,
7395  memTypeIndex,
7396  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7397  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7398  finalCreateInfo.pUserData,
7399  dedicatedBuffer,
7400  dedicatedImage,
7401  pAllocation);
7402  if(res == VK_SUCCESS)
7403  {
7404  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
7405  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
7406  return VK_SUCCESS;
7407  }
7408  else
7409  {
7410  // Everything failed: Return error code.
7411  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7412  return res;
7413  }
7414  }
7415  }
7416 }
7417 
7418 VkResult VmaAllocator_T::AllocateDedicatedMemory(
7419  VkDeviceSize size,
7420  VmaSuballocationType suballocType,
7421  uint32_t memTypeIndex,
7422  bool map,
7423  bool isUserDataString,
7424  void* pUserData,
7425  VkBuffer dedicatedBuffer,
7426  VkImage dedicatedImage,
7427  VmaAllocation* pAllocation)
7428 {
7429  VMA_ASSERT(pAllocation);
7430 
7431  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
7432  allocInfo.memoryTypeIndex = memTypeIndex;
7433  allocInfo.allocationSize = size;
7434 
7435  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
7436  if(m_UseKhrDedicatedAllocation)
7437  {
7438  if(dedicatedBuffer != VK_NULL_HANDLE)
7439  {
7440  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
7441  dedicatedAllocInfo.buffer = dedicatedBuffer;
7442  allocInfo.pNext = &dedicatedAllocInfo;
7443  }
7444  else if(dedicatedImage != VK_NULL_HANDLE)
7445  {
7446  dedicatedAllocInfo.image = dedicatedImage;
7447  allocInfo.pNext = &dedicatedAllocInfo;
7448  }
7449  }
7450 
7451  // Allocate VkDeviceMemory.
7452  VkDeviceMemory hMemory = VK_NULL_HANDLE;
7453  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
7454  if(res < 0)
7455  {
7456  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7457  return res;
7458  }
7459 
7460  void* pMappedData = VMA_NULL;
7461  if(map)
7462  {
7463  res = (*m_VulkanFunctions.vkMapMemory)(
7464  m_hDevice,
7465  hMemory,
7466  0,
7467  VK_WHOLE_SIZE,
7468  0,
7469  &pMappedData);
7470  if(res < 0)
7471  {
7472  VMA_DEBUG_LOG(" vkMapMemory FAILED");
7473  FreeVulkanMemory(memTypeIndex, size, hMemory);
7474  return res;
7475  }
7476  }
7477 
7478  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
7479  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
7480  (*pAllocation)->SetUserData(this, pUserData);
7481 
7482  // Register it in m_pDedicatedAllocations.
7483  {
7484  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7485  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7486  VMA_ASSERT(pDedicatedAllocations);
7487  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
7488  }
7489 
7490  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
7491 
7492  return VK_SUCCESS;
7493 }
7494 
7495 void VmaAllocator_T::GetBufferMemoryRequirements(
7496  VkBuffer hBuffer,
7497  VkMemoryRequirements& memReq,
7498  bool& requiresDedicatedAllocation,
7499  bool& prefersDedicatedAllocation) const
7500 {
7501  if(m_UseKhrDedicatedAllocation)
7502  {
7503  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
7504  memReqInfo.buffer = hBuffer;
7505 
7506  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7507 
7508  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7509  memReq2.pNext = &memDedicatedReq;
7510 
7511  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7512 
7513  memReq = memReq2.memoryRequirements;
7514  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7515  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7516  }
7517  else
7518  {
7519  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
7520  requiresDedicatedAllocation = false;
7521  prefersDedicatedAllocation = false;
7522  }
7523 }
7524 
7525 void VmaAllocator_T::GetImageMemoryRequirements(
7526  VkImage hImage,
7527  VkMemoryRequirements& memReq,
7528  bool& requiresDedicatedAllocation,
7529  bool& prefersDedicatedAllocation) const
7530 {
7531  if(m_UseKhrDedicatedAllocation)
7532  {
7533  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
7534  memReqInfo.image = hImage;
7535 
7536  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7537 
7538  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7539  memReq2.pNext = &memDedicatedReq;
7540 
7541  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7542 
7543  memReq = memReq2.memoryRequirements;
7544  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7545  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7546  }
7547  else
7548  {
7549  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
7550  requiresDedicatedAllocation = false;
7551  prefersDedicatedAllocation = false;
7552  }
7553 }
7554 
7555 VkResult VmaAllocator_T::AllocateMemory(
7556  const VkMemoryRequirements& vkMemReq,
7557  bool requiresDedicatedAllocation,
7558  bool prefersDedicatedAllocation,
7559  VkBuffer dedicatedBuffer,
7560  VkImage dedicatedImage,
7561  const VmaAllocationCreateInfo& createInfo,
7562  VmaSuballocationType suballocType,
7563  VmaAllocation* pAllocation)
7564 {
7565  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
7566  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7567  {
7568  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
7569  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7570  }
7571  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7572  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
7573  {
7574  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
7575  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7576  }
7577  if(requiresDedicatedAllocation)
7578  {
7579  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7580  {
7581  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
7582  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7583  }
7584  if(createInfo.pool != VK_NULL_HANDLE)
7585  {
7586  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
7587  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7588  }
7589  }
7590  if((createInfo.pool != VK_NULL_HANDLE) &&
7591  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
7592  {
7593  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
7594  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7595  }
7596 
7597  if(createInfo.pool != VK_NULL_HANDLE)
7598  {
7599  return createInfo.pool->m_BlockVector.Allocate(
7600  createInfo.pool,
7601  m_CurrentFrameIndex.load(),
7602  vkMemReq,
7603  createInfo,
7604  suballocType,
7605  pAllocation);
7606  }
7607  else
7608  {
7609  // Bit mask of memory Vulkan types acceptable for this allocation.
7610  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
7611  uint32_t memTypeIndex = UINT32_MAX;
7612  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7613  if(res == VK_SUCCESS)
7614  {
7615  res = AllocateMemoryOfType(
7616  vkMemReq,
7617  requiresDedicatedAllocation || prefersDedicatedAllocation,
7618  dedicatedBuffer,
7619  dedicatedImage,
7620  createInfo,
7621  memTypeIndex,
7622  suballocType,
7623  pAllocation);
7624  // Succeeded on first try.
7625  if(res == VK_SUCCESS)
7626  {
7627  return res;
7628  }
7629  // Allocation from this memory type failed. Try other compatible memory types.
7630  else
7631  {
7632  for(;;)
7633  {
7634  // Remove old memTypeIndex from list of possibilities.
7635  memoryTypeBits &= ~(1u << memTypeIndex);
7636  // Find alternative memTypeIndex.
7637  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7638  if(res == VK_SUCCESS)
7639  {
7640  res = AllocateMemoryOfType(
7641  vkMemReq,
7642  requiresDedicatedAllocation || prefersDedicatedAllocation,
7643  dedicatedBuffer,
7644  dedicatedImage,
7645  createInfo,
7646  memTypeIndex,
7647  suballocType,
7648  pAllocation);
7649  // Allocation from this alternative memory type succeeded.
7650  if(res == VK_SUCCESS)
7651  {
7652  return res;
7653  }
7654  // else: Allocation from this memory type failed. Try next one - next loop iteration.
7655  }
7656  // No other matching memory type index could be found.
7657  else
7658  {
7659  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
7660  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7661  }
7662  }
7663  }
7664  }
7665  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
7666  else
7667  return res;
7668  }
7669 }
7670 
7671 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
7672 {
7673  VMA_ASSERT(allocation);
7674 
7675  if(allocation->CanBecomeLost() == false ||
7676  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
7677  {
7678  switch(allocation->GetType())
7679  {
7680  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7681  {
7682  VmaBlockVector* pBlockVector = VMA_NULL;
7683  VmaPool hPool = allocation->GetPool();
7684  if(hPool != VK_NULL_HANDLE)
7685  {
7686  pBlockVector = &hPool->m_BlockVector;
7687  }
7688  else
7689  {
7690  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7691  pBlockVector = m_pBlockVectors[memTypeIndex];
7692  }
7693  pBlockVector->Free(allocation);
7694  }
7695  break;
7696  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7697  FreeDedicatedMemory(allocation);
7698  break;
7699  default:
7700  VMA_ASSERT(0);
7701  }
7702  }
7703 
7704  allocation->SetUserData(this, VMA_NULL);
7705  vma_delete(this, allocation);
7706 }
7707 
7708 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
7709 {
7710  // Initialize.
7711  InitStatInfo(pStats->total);
7712  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
7713  InitStatInfo(pStats->memoryType[i]);
7714  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
7715  InitStatInfo(pStats->memoryHeap[i]);
7716 
7717  // Process default pools.
7718  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7719  {
7720  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
7721  VMA_ASSERT(pBlockVector);
7722  pBlockVector->AddStats(pStats);
7723  }
7724 
7725  // Process custom pools.
7726  {
7727  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7728  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
7729  {
7730  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
7731  }
7732  }
7733 
7734  // Process dedicated allocations.
7735  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7736  {
7737  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7738  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7739  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7740  VMA_ASSERT(pDedicatedAllocVector);
7741  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
7742  {
7743  VmaStatInfo allocationStatInfo;
7744  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
7745  VmaAddStatInfo(pStats->total, allocationStatInfo);
7746  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
7747  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
7748  }
7749  }
7750 
7751  // Postprocess.
7752  VmaPostprocessCalcStatInfo(pStats->total);
7753  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
7754  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
7755  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
7756  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
7757 }
7758 
7759 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
7760 
7761 VkResult VmaAllocator_T::Defragment(
7762  VmaAllocation* pAllocations,
7763  size_t allocationCount,
7764  VkBool32* pAllocationsChanged,
7765  const VmaDefragmentationInfo* pDefragmentationInfo,
7766  VmaDefragmentationStats* pDefragmentationStats)
7767 {
7768  if(pAllocationsChanged != VMA_NULL)
7769  {
7770  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
7771  }
7772  if(pDefragmentationStats != VMA_NULL)
7773  {
7774  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
7775  }
7776 
7777  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
7778 
7779  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
7780 
7781  const size_t poolCount = m_Pools.size();
7782 
7783  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
7784  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
7785  {
7786  VmaAllocation hAlloc = pAllocations[allocIndex];
7787  VMA_ASSERT(hAlloc);
7788  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
7789  // DedicatedAlloc cannot be defragmented.
7790  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
7791  // Only HOST_VISIBLE memory types can be defragmented.
7792  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
7793  // Lost allocation cannot be defragmented.
7794  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
7795  {
7796  VmaBlockVector* pAllocBlockVector = VMA_NULL;
7797 
7798  const VmaPool hAllocPool = hAlloc->GetPool();
7799  // This allocation belongs to custom pool.
7800  if(hAllocPool != VK_NULL_HANDLE)
7801  {
7802  pAllocBlockVector = &hAllocPool->GetBlockVector();
7803  }
7804  // This allocation belongs to general pool.
7805  else
7806  {
7807  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
7808  }
7809 
7810  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
7811 
7812  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
7813  &pAllocationsChanged[allocIndex] : VMA_NULL;
7814  pDefragmentator->AddAllocation(hAlloc, pChanged);
7815  }
7816  }
7817 
7818  VkResult result = VK_SUCCESS;
7819 
7820  // ======== Main processing.
7821 
7822  VkDeviceSize maxBytesToMove = SIZE_MAX;
7823  uint32_t maxAllocationsToMove = UINT32_MAX;
7824  if(pDefragmentationInfo != VMA_NULL)
7825  {
7826  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
7827  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
7828  }
7829 
7830  // Process standard memory.
7831  for(uint32_t memTypeIndex = 0;
7832  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
7833  ++memTypeIndex)
7834  {
7835  // Only HOST_VISIBLE memory types can be defragmented.
7836  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7837  {
7838  result = m_pBlockVectors[memTypeIndex]->Defragment(
7839  pDefragmentationStats,
7840  maxBytesToMove,
7841  maxAllocationsToMove);
7842  }
7843  }
7844 
7845  // Process custom pools.
7846  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
7847  {
7848  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
7849  pDefragmentationStats,
7850  maxBytesToMove,
7851  maxAllocationsToMove);
7852  }
7853 
7854  // ======== Destroy defragmentators.
7855 
7856  // Process custom pools.
7857  for(size_t poolIndex = poolCount; poolIndex--; )
7858  {
7859  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
7860  }
7861 
7862  // Process standard memory.
7863  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
7864  {
7865  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7866  {
7867  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
7868  }
7869  }
7870 
7871  return result;
7872 }
7873 
7874 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
7875 {
7876  if(hAllocation->CanBecomeLost())
7877  {
7878  /*
7879  Warning: This is a carefully designed algorithm.
7880  Do not modify unless you really know what you're doing :)
7881  */
7882  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
7883  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
7884  for(;;)
7885  {
7886  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
7887  {
7888  pAllocationInfo->memoryType = UINT32_MAX;
7889  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
7890  pAllocationInfo->offset = 0;
7891  pAllocationInfo->size = hAllocation->GetSize();
7892  pAllocationInfo->pMappedData = VMA_NULL;
7893  pAllocationInfo->pUserData = hAllocation->GetUserData();
7894  return;
7895  }
7896  else if(localLastUseFrameIndex == localCurrFrameIndex)
7897  {
7898  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7899  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7900  pAllocationInfo->offset = hAllocation->GetOffset();
7901  pAllocationInfo->size = hAllocation->GetSize();
7902  pAllocationInfo->pMappedData = VMA_NULL;
7903  pAllocationInfo->pUserData = hAllocation->GetUserData();
7904  return;
7905  }
7906  else // Last use time earlier than current time.
7907  {
7908  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
7909  {
7910  localLastUseFrameIndex = localCurrFrameIndex;
7911  }
7912  }
7913  }
7914  }
7915  else
7916  {
7917  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7918  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7919  pAllocationInfo->offset = hAllocation->GetOffset();
7920  pAllocationInfo->size = hAllocation->GetSize();
7921  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
7922  pAllocationInfo->pUserData = hAllocation->GetUserData();
7923  }
7924 }
7925 
7926 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
7927 {
7928  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
7929  if(hAllocation->CanBecomeLost())
7930  {
7931  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
7932  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
7933  for(;;)
7934  {
7935  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
7936  {
7937  return false;
7938  }
7939  else if(localLastUseFrameIndex == localCurrFrameIndex)
7940  {
7941  return true;
7942  }
7943  else // Last use time earlier than current time.
7944  {
7945  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
7946  {
7947  localLastUseFrameIndex = localCurrFrameIndex;
7948  }
7949  }
7950  }
7951  }
7952  else
7953  {
7954  return true;
7955  }
7956 }
7957 
7958 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
7959 {
7960  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
7961 
7962  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
7963 
7964  if(newCreateInfo.maxBlockCount == 0)
7965  {
7966  newCreateInfo.maxBlockCount = SIZE_MAX;
7967  }
7968  if(newCreateInfo.blockSize == 0)
7969  {
7970  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
7971  }
7972 
7973  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
7974 
7975  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
7976  if(res != VK_SUCCESS)
7977  {
7978  vma_delete(this, *pPool);
7979  *pPool = VMA_NULL;
7980  return res;
7981  }
7982 
7983  // Add to m_Pools.
7984  {
7985  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7986  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
7987  }
7988 
7989  return VK_SUCCESS;
7990 }
7991 
7992 void VmaAllocator_T::DestroyPool(VmaPool pool)
7993 {
7994  // Remove from m_Pools.
7995  {
7996  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7997  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
7998  VMA_ASSERT(success && "Pool not found in Allocator.");
7999  }
8000 
8001  vma_delete(this, pool);
8002 }
8003 
8004 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
8005 {
8006  pool->m_BlockVector.GetPoolStats(pPoolStats);
8007 }
8008 
8009 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
8010 {
8011  m_CurrentFrameIndex.store(frameIndex);
8012 }
8013 
8014 void VmaAllocator_T::MakePoolAllocationsLost(
8015  VmaPool hPool,
8016  size_t* pLostAllocationCount)
8017 {
8018  hPool->m_BlockVector.MakePoolAllocationsLost(
8019  m_CurrentFrameIndex.load(),
8020  pLostAllocationCount);
8021 }
8022 
8023 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
8024 {
8025  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
8026  (*pAllocation)->InitLost();
8027 }
8028 
8029 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
8030 {
8031  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
8032 
8033  VkResult res;
8034  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
8035  {
8036  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
8037  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
8038  {
8039  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
8040  if(res == VK_SUCCESS)
8041  {
8042  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
8043  }
8044  }
8045  else
8046  {
8047  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
8048  }
8049  }
8050  else
8051  {
8052  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
8053  }
8054 
8055  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
8056  {
8057  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
8058  }
8059 
8060  return res;
8061 }
8062 
8063 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
8064 {
8065  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
8066  {
8067  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
8068  }
8069 
8070  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
8071 
8072  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
8073  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
8074  {
8075  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
8076  m_HeapSizeLimit[heapIndex] += size;
8077  }
8078 }
8079 
8080 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
8081 {
8082  if(hAllocation->CanBecomeLost())
8083  {
8084  return VK_ERROR_MEMORY_MAP_FAILED;
8085  }
8086 
8087  switch(hAllocation->GetType())
8088  {
8089  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
8090  {
8091  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
8092  char *pBytes = VMA_NULL;
8093  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
8094  if(res == VK_SUCCESS)
8095  {
8096  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
8097  hAllocation->BlockAllocMap();
8098  }
8099  return res;
8100  }
8101  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
8102  return hAllocation->DedicatedAllocMap(this, ppData);
8103  default:
8104  VMA_ASSERT(0);
8105  return VK_ERROR_MEMORY_MAP_FAILED;
8106  }
8107 }
8108 
8109 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
8110 {
8111  switch(hAllocation->GetType())
8112  {
8113  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
8114  {
8115  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
8116  hAllocation->BlockAllocUnmap();
8117  pBlock->Unmap(this, 1);
8118  }
8119  break;
8120  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
8121  hAllocation->DedicatedAllocUnmap(this);
8122  break;
8123  default:
8124  VMA_ASSERT(0);
8125  }
8126 }
8127 
8128 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
8129 {
8130  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
8131 
8132  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
8133  {
8134  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
8135  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
8136  VMA_ASSERT(pDedicatedAllocations);
8137  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
8138  VMA_ASSERT(success);
8139  }
8140 
8141  VkDeviceMemory hMemory = allocation->GetMemory();
8142 
8143  if(allocation->GetMappedData() != VMA_NULL)
8144  {
8145  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
8146  }
8147 
8148  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
8149 
8150  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
8151 }
8152 
8153 #if VMA_STATS_STRING_ENABLED
8154 
8155 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
8156 {
8157  bool dedicatedAllocationsStarted = false;
8158  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8159  {
8160  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
8161  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
8162  VMA_ASSERT(pDedicatedAllocVector);
8163  if(pDedicatedAllocVector->empty() == false)
8164  {
8165  if(dedicatedAllocationsStarted == false)
8166  {
8167  dedicatedAllocationsStarted = true;
8168  json.WriteString("DedicatedAllocations");
8169  json.BeginObject();
8170  }
8171 
8172  json.BeginString("Type ");
8173  json.ContinueString(memTypeIndex);
8174  json.EndString();
8175 
8176  json.BeginArray();
8177 
8178  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
8179  {
8180  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
8181  json.BeginObject(true);
8182 
8183  json.WriteString("Type");
8184  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
8185 
8186  json.WriteString("Size");
8187  json.WriteNumber(hAlloc->GetSize());
8188 
8189  const void* pUserData = hAlloc->GetUserData();
8190  if(pUserData != VMA_NULL)
8191  {
8192  json.WriteString("UserData");
8193  if(hAlloc->IsUserDataString())
8194  {
8195  json.WriteString((const char*)pUserData);
8196  }
8197  else
8198  {
8199  json.BeginString();
8200  json.ContinueString_Pointer(pUserData);
8201  json.EndString();
8202  }
8203  }
8204 
8205  json.EndObject();
8206  }
8207 
8208  json.EndArray();
8209  }
8210  }
8211  if(dedicatedAllocationsStarted)
8212  {
8213  json.EndObject();
8214  }
8215 
8216  {
8217  bool allocationsStarted = false;
8218  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8219  {
8220  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
8221  {
8222  if(allocationsStarted == false)
8223  {
8224  allocationsStarted = true;
8225  json.WriteString("DefaultPools");
8226  json.BeginObject();
8227  }
8228 
8229  json.BeginString("Type ");
8230  json.ContinueString(memTypeIndex);
8231  json.EndString();
8232 
8233  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
8234  }
8235  }
8236  if(allocationsStarted)
8237  {
8238  json.EndObject();
8239  }
8240  }
8241 
8242  {
8243  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
8244  const size_t poolCount = m_Pools.size();
8245  if(poolCount > 0)
8246  {
8247  json.WriteString("Pools");
8248  json.BeginArray();
8249  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
8250  {
8251  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
8252  }
8253  json.EndArray();
8254  }
8255  }
8256 }
8257 
8258 #endif // #if VMA_STATS_STRING_ENABLED
8259 
8260 static VkResult AllocateMemoryForImage(
8261  VmaAllocator allocator,
8262  VkImage image,
8263  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8264  VmaSuballocationType suballocType,
8265  VmaAllocation* pAllocation)
8266 {
8267  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
8268 
8269  VkMemoryRequirements vkMemReq = {};
8270  bool requiresDedicatedAllocation = false;
8271  bool prefersDedicatedAllocation = false;
8272  allocator->GetImageMemoryRequirements(image, vkMemReq,
8273  requiresDedicatedAllocation, prefersDedicatedAllocation);
8274 
8275  return allocator->AllocateMemory(
8276  vkMemReq,
8277  requiresDedicatedAllocation,
8278  prefersDedicatedAllocation,
8279  VK_NULL_HANDLE, // dedicatedBuffer
8280  image, // dedicatedImage
8281  *pAllocationCreateInfo,
8282  suballocType,
8283  pAllocation);
8284 }
8285 
8287 // Public interface
8288 
8289 VkResult vmaCreateAllocator(
8290  const VmaAllocatorCreateInfo* pCreateInfo,
8291  VmaAllocator* pAllocator)
8292 {
8293  VMA_ASSERT(pCreateInfo && pAllocator);
8294  VMA_DEBUG_LOG("vmaCreateAllocator");
8295  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
8296  return VK_SUCCESS;
8297 }
8298 
8299 void vmaDestroyAllocator(
8300  VmaAllocator allocator)
8301 {
8302  if(allocator != VK_NULL_HANDLE)
8303  {
8304  VMA_DEBUG_LOG("vmaDestroyAllocator");
8305  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
8306  vma_delete(&allocationCallbacks, allocator);
8307  }
8308 }
8309 
8310 void vmaGetPhysicalDeviceProperties(
8311  VmaAllocator allocator,
8312  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
8313 {
8314  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
8315  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
8316 }
8317 
8318 void vmaGetMemoryProperties(
8319  VmaAllocator allocator,
8320  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
8321 {
8322  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
8323  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
8324 }
8325 
8326 void vmaGetMemoryTypeProperties(
8327  VmaAllocator allocator,
8328  uint32_t memoryTypeIndex,
8329  VkMemoryPropertyFlags* pFlags)
8330 {
8331  VMA_ASSERT(allocator && pFlags);
8332  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
8333  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
8334 }
8335 
8336 void vmaSetCurrentFrameIndex(
8337  VmaAllocator allocator,
8338  uint32_t frameIndex)
8339 {
8340  VMA_ASSERT(allocator);
8341  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
8342 
8343  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8344 
8345  allocator->SetCurrentFrameIndex(frameIndex);
8346 }
8347 
8348 void vmaCalculateStats(
8349  VmaAllocator allocator,
8350  VmaStats* pStats)
8351 {
8352  VMA_ASSERT(allocator && pStats);
8353  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8354  allocator->CalculateStats(pStats);
8355 }
8356 
8357 #if VMA_STATS_STRING_ENABLED
8358 
8359 void vmaBuildStatsString(
8360  VmaAllocator allocator,
8361  char** ppStatsString,
8362  VkBool32 detailedMap)
8363 {
8364  VMA_ASSERT(allocator && ppStatsString);
8365  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8366 
8367  VmaStringBuilder sb(allocator);
8368  {
8369  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
8370  json.BeginObject();
8371 
8372  VmaStats stats;
8373  allocator->CalculateStats(&stats);
8374 
8375  json.WriteString("Total");
8376  VmaPrintStatInfo(json, stats.total);
8377 
8378  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
8379  {
8380  json.BeginString("Heap ");
8381  json.ContinueString(heapIndex);
8382  json.EndString();
8383  json.BeginObject();
8384 
8385  json.WriteString("Size");
8386  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
8387 
8388  json.WriteString("Flags");
8389  json.BeginArray(true);
8390  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
8391  {
8392  json.WriteString("DEVICE_LOCAL");
8393  }
8394  json.EndArray();
8395 
8396  if(stats.memoryHeap[heapIndex].blockCount > 0)
8397  {
8398  json.WriteString("Stats");
8399  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
8400  }
8401 
8402  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
8403  {
8404  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
8405  {
8406  json.BeginString("Type ");
8407  json.ContinueString(typeIndex);
8408  json.EndString();
8409 
8410  json.BeginObject();
8411 
8412  json.WriteString("Flags");
8413  json.BeginArray(true);
8414  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
8415  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
8416  {
8417  json.WriteString("DEVICE_LOCAL");
8418  }
8419  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
8420  {
8421  json.WriteString("HOST_VISIBLE");
8422  }
8423  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
8424  {
8425  json.WriteString("HOST_COHERENT");
8426  }
8427  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
8428  {
8429  json.WriteString("HOST_CACHED");
8430  }
8431  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
8432  {
8433  json.WriteString("LAZILY_ALLOCATED");
8434  }
8435  json.EndArray();
8436 
8437  if(stats.memoryType[typeIndex].blockCount > 0)
8438  {
8439  json.WriteString("Stats");
8440  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
8441  }
8442 
8443  json.EndObject();
8444  }
8445  }
8446 
8447  json.EndObject();
8448  }
8449  if(detailedMap == VK_TRUE)
8450  {
8451  allocator->PrintDetailedMap(json);
8452  }
8453 
8454  json.EndObject();
8455  }
8456 
8457  const size_t len = sb.GetLength();
8458  char* const pChars = vma_new_array(allocator, char, len + 1);
8459  if(len > 0)
8460  {
8461  memcpy(pChars, sb.GetData(), len);
8462  }
8463  pChars[len] = '\0';
8464  *ppStatsString = pChars;
8465 }
8466 
8467 void vmaFreeStatsString(
8468  VmaAllocator allocator,
8469  char* pStatsString)
8470 {
8471  if(pStatsString != VMA_NULL)
8472  {
8473  VMA_ASSERT(allocator);
8474  size_t len = strlen(pStatsString);
8475  vma_delete_array(allocator, pStatsString, len + 1);
8476  }
8477 }
8478 
8479 #endif // #if VMA_STATS_STRING_ENABLED
8480 
8481 /*
8482 This function is not protected by any mutex because it just reads immutable data.
8483 */
8484 VkResult vmaFindMemoryTypeIndex(
8485  VmaAllocator allocator,
8486  uint32_t memoryTypeBits,
8487  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8488  uint32_t* pMemoryTypeIndex)
8489 {
8490  VMA_ASSERT(allocator != VK_NULL_HANDLE);
8491  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8492  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8493 
8494  if(pAllocationCreateInfo->memoryTypeBits != 0)
8495  {
8496  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
8497  }
8498 
8499  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
8500  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
8501 
8502  // Convert usage to requiredFlags and preferredFlags.
8503  switch(pAllocationCreateInfo->usage)
8504  {
8505  case VMA_MEMORY_USAGE_UNKNOWN:
8506  break;
8507  case VMA_MEMORY_USAGE_GPU_ONLY:
8508  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8509  break;
8510  case VMA_MEMORY_USAGE_CPU_ONLY:
8511  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
8512  break;
8513  case VMA_MEMORY_USAGE_CPU_TO_GPU:
8514  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8515  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8516  break;
8517  case VMA_MEMORY_USAGE_GPU_TO_CPU:
8518  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8519  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
8520  break;
8521  default:
8522  break;
8523  }
8524 
8525  *pMemoryTypeIndex = UINT32_MAX;
8526  uint32_t minCost = UINT32_MAX;
8527  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
8528  memTypeIndex < allocator->GetMemoryTypeCount();
8529  ++memTypeIndex, memTypeBit <<= 1)
8530  {
8531  // This memory type is acceptable according to memoryTypeBits bitmask.
8532  if((memTypeBit & memoryTypeBits) != 0)
8533  {
8534  const VkMemoryPropertyFlags currFlags =
8535  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
8536  // This memory type contains requiredFlags.
8537  if((requiredFlags & ~currFlags) == 0)
8538  {
8539  // Calculate cost as number of bits from preferredFlags not present in this memory type.
8540  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
8541  // Remember memory type with lowest cost.
8542  if(currCost < minCost)
8543  {
8544  *pMemoryTypeIndex = memTypeIndex;
8545  if(currCost == 0)
8546  {
8547  return VK_SUCCESS;
8548  }
8549  minCost = currCost;
8550  }
8551  }
8552  }
8553  }
8554  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
8555 }
8556 
8557 VkResult vmaFindMemoryTypeIndexForBufferInfo(
8558  VmaAllocator allocator,
8559  const VkBufferCreateInfo* pBufferCreateInfo,
8560  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8561  uint32_t* pMemoryTypeIndex)
8562 {
8563  VMA_ASSERT(allocator != VK_NULL_HANDLE);
8564  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
8565  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8566  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8567 
8568  const VkDevice hDev = allocator->m_hDevice;
8569  VkBuffer hBuffer = VK_NULL_HANDLE;
8570  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
8571  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
8572  if(res == VK_SUCCESS)
8573  {
8574  VkMemoryRequirements memReq = {};
8575  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
8576  hDev, hBuffer, &memReq);
8577 
8578  res = vmaFindMemoryTypeIndex(
8579  allocator,
8580  memReq.memoryTypeBits,
8581  pAllocationCreateInfo,
8582  pMemoryTypeIndex);
8583 
8584  allocator->GetVulkanFunctions().vkDestroyBuffer(
8585  hDev, hBuffer, allocator->GetAllocationCallbacks());
8586  }
8587  return res;
8588 }
8589 
8590 VkResult vmaFindMemoryTypeIndexForImageInfo(
8591  VmaAllocator allocator,
8592  const VkImageCreateInfo* pImageCreateInfo,
8593  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8594  uint32_t* pMemoryTypeIndex)
8595 {
8596  VMA_ASSERT(allocator != VK_NULL_HANDLE);
8597  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
8598  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8599  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8600 
8601  const VkDevice hDev = allocator->m_hDevice;
8602  VkImage hImage = VK_NULL_HANDLE;
8603  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
8604  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
8605  if(res == VK_SUCCESS)
8606  {
8607  VkMemoryRequirements memReq = {};
8608  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
8609  hDev, hImage, &memReq);
8610 
8611  res = vmaFindMemoryTypeIndex(
8612  allocator,
8613  memReq.memoryTypeBits,
8614  pAllocationCreateInfo,
8615  pMemoryTypeIndex);
8616 
8617  allocator->GetVulkanFunctions().vkDestroyImage(
8618  hDev, hImage, allocator->GetAllocationCallbacks());
8619  }
8620  return res;
8621 }
8622 
8623 VkResult vmaCreatePool(
8624  VmaAllocator allocator,
8625  const VmaPoolCreateInfo* pCreateInfo,
8626  VmaPool* pPool)
8627 {
8628  VMA_ASSERT(allocator && pCreateInfo && pPool);
8629 
8630  VMA_DEBUG_LOG("vmaCreatePool");
8631 
8632  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8633 
8634  return allocator->CreatePool(pCreateInfo, pPool);
8635 }
8636 
8637 void vmaDestroyPool(
8638  VmaAllocator allocator,
8639  VmaPool pool)
8640 {
8641  VMA_ASSERT(allocator);
8642 
8643  if(pool == VK_NULL_HANDLE)
8644  {
8645  return;
8646  }
8647 
8648  VMA_DEBUG_LOG("vmaDestroyPool");
8649 
8650  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8651 
8652  allocator->DestroyPool(pool);
8653 }
8654 
8655 void vmaGetPoolStats(
8656  VmaAllocator allocator,
8657  VmaPool pool,
8658  VmaPoolStats* pPoolStats)
8659 {
8660  VMA_ASSERT(allocator && pool && pPoolStats);
8661 
8662  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8663 
8664  allocator->GetPoolStats(pool, pPoolStats);
8665 }
8666 
8667 void vmaMakePoolAllocationsLost(
8668  VmaAllocator allocator,
8669  VmaPool pool,
8670  size_t* pLostAllocationCount)
8671 {
8672  VMA_ASSERT(allocator && pool);
8673 
8674  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8675 
8676  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
8677 }
8678 
8679 VkResult vmaAllocateMemory(
8680  VmaAllocator allocator,
8681  const VkMemoryRequirements* pVkMemoryRequirements,
8682  const VmaAllocationCreateInfo* pCreateInfo,
8683  VmaAllocation* pAllocation,
8684  VmaAllocationInfo* pAllocationInfo)
8685 {
8686  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
8687 
8688  VMA_DEBUG_LOG("vmaAllocateMemory");
8689 
8690  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8691 
8692  VkResult result = allocator->AllocateMemory(
8693  *pVkMemoryRequirements,
8694  false, // requiresDedicatedAllocation
8695  false, // prefersDedicatedAllocation
8696  VK_NULL_HANDLE, // dedicatedBuffer
8697  VK_NULL_HANDLE, // dedicatedImage
8698  *pCreateInfo,
8699  VMA_SUBALLOCATION_TYPE_UNKNOWN,
8700  pAllocation);
8701 
8702  if(pAllocationInfo && result == VK_SUCCESS)
8703  {
8704  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8705  }
8706 
8707  return result;
8708 }
8709 
8710 VkResult vmaAllocateMemoryForBuffer(
8711  VmaAllocator allocator,
8712  VkBuffer buffer,
8713  const VmaAllocationCreateInfo* pCreateInfo,
8714  VmaAllocation* pAllocation,
8715  VmaAllocationInfo* pAllocationInfo)
8716 {
8717  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8718 
8719  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
8720 
8721  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8722 
8723  VkMemoryRequirements vkMemReq = {};
8724  bool requiresDedicatedAllocation = false;
8725  bool prefersDedicatedAllocation = false;
8726  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
8727  requiresDedicatedAllocation,
8728  prefersDedicatedAllocation);
8729 
8730  VkResult result = allocator->AllocateMemory(
8731  vkMemReq,
8732  requiresDedicatedAllocation,
8733  prefersDedicatedAllocation,
8734  buffer, // dedicatedBuffer
8735  VK_NULL_HANDLE, // dedicatedImage
8736  *pCreateInfo,
8737  VMA_SUBALLOCATION_TYPE_BUFFER,
8738  pAllocation);
8739 
8740  if(pAllocationInfo && result == VK_SUCCESS)
8741  {
8742  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8743  }
8744 
8745  return result;
8746 }
8747 
8748 VkResult vmaAllocateMemoryForImage(
8749  VmaAllocator allocator,
8750  VkImage image,
8751  const VmaAllocationCreateInfo* pCreateInfo,
8752  VmaAllocation* pAllocation,
8753  VmaAllocationInfo* pAllocationInfo)
8754 {
8755  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8756 
8757  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
8758 
8759  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8760 
8761  VkResult result = AllocateMemoryForImage(
8762  allocator,
8763  image,
8764  pCreateInfo,
8765  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
8766  pAllocation);
8767 
8768  if(pAllocationInfo && result == VK_SUCCESS)
8769  {
8770  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8771  }
8772 
8773  return result;
8774 }
8775 
8776 void vmaFreeMemory(
8777  VmaAllocator allocator,
8778  VmaAllocation allocation)
8779 {
8780  VMA_ASSERT(allocator && allocation);
8781 
8782  VMA_DEBUG_LOG("vmaFreeMemory");
8783 
8784  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8785 
8786  allocator->FreeMemory(allocation);
8787 }
8788 
8789 void vmaGetAllocationInfo(
8790  VmaAllocator allocator,
8791  VmaAllocation allocation,
8792  VmaAllocationInfo* pAllocationInfo)
8793 {
8794  VMA_ASSERT(allocator && allocation && pAllocationInfo);
8795 
8796  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8797 
8798  allocator->GetAllocationInfo(allocation, pAllocationInfo);
8799 }
8800 
8801 VkBool32 vmaTouchAllocation(
8802  VmaAllocator allocator,
8803  VmaAllocation allocation)
8804 {
8805  VMA_ASSERT(allocator && allocation);
8806 
8807  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8808 
8809  return allocator->TouchAllocation(allocation);
8810 }
8811 
8812 void vmaSetAllocationUserData(
8813  VmaAllocator allocator,
8814  VmaAllocation allocation,
8815  void* pUserData)
8816 {
8817  VMA_ASSERT(allocator && allocation);
8818 
8819  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8820 
8821  allocation->SetUserData(allocator, pUserData);
8822 }
8823 
8824 void vmaCreateLostAllocation(
8825  VmaAllocator allocator,
8826  VmaAllocation* pAllocation)
8827 {
8828  VMA_ASSERT(allocator && pAllocation);
8829 
8830  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
8831 
8832  allocator->CreateLostAllocation(pAllocation);
8833 }
8834 
8835 VkResult vmaMapMemory(
8836  VmaAllocator allocator,
8837  VmaAllocation allocation,
8838  void** ppData)
8839 {
8840  VMA_ASSERT(allocator && allocation && ppData);
8841 
8842  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8843 
8844  return allocator->Map(allocation, ppData);
8845 }
8846 
8847 void vmaUnmapMemory(
8848  VmaAllocator allocator,
8849  VmaAllocation allocation)
8850 {
8851  VMA_ASSERT(allocator && allocation);
8852 
8853  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8854 
8855  allocator->Unmap(allocation);
8856 }
8857 
8858 VkResult vmaDefragment(
8859  VmaAllocator allocator,
8860  VmaAllocation* pAllocations,
8861  size_t allocationCount,
8862  VkBool32* pAllocationsChanged,
8863  const VmaDefragmentationInfo *pDefragmentationInfo,
8864  VmaDefragmentationStats* pDefragmentationStats)
8865 {
8866  VMA_ASSERT(allocator && pAllocations);
8867 
8868  VMA_DEBUG_LOG("vmaDefragment");
8869 
8870  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8871 
8872  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
8873 }
8874 
8875 VkResult vmaCreateBuffer(
8876  VmaAllocator allocator,
8877  const VkBufferCreateInfo* pBufferCreateInfo,
8878  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8879  VkBuffer* pBuffer,
8880  VmaAllocation* pAllocation,
8881  VmaAllocationInfo* pAllocationInfo)
8882 {
8883  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
8884 
8885  VMA_DEBUG_LOG("vmaCreateBuffer");
8886 
8887  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8888 
8889  *pBuffer = VK_NULL_HANDLE;
8890  *pAllocation = VK_NULL_HANDLE;
8891 
8892  // 1. Create VkBuffer.
8893  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
8894  allocator->m_hDevice,
8895  pBufferCreateInfo,
8896  allocator->GetAllocationCallbacks(),
8897  pBuffer);
8898  if(res >= 0)
8899  {
8900  // 2. vkGetBufferMemoryRequirements.
8901  VkMemoryRequirements vkMemReq = {};
8902  bool requiresDedicatedAllocation = false;
8903  bool prefersDedicatedAllocation = false;
8904  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
8905  requiresDedicatedAllocation, prefersDedicatedAllocation);
8906 
8907  // Make sure alignment requirements for specific buffer usages reported
8908  // in Physical Device Properties are included in alignment reported by memory requirements.
8909  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
8910  {
8911  VMA_ASSERT(vkMemReq.alignment %
8912  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
8913  }
8914  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
8915  {
8916  VMA_ASSERT(vkMemReq.alignment %
8917  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
8918  }
8919  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
8920  {
8921  VMA_ASSERT(vkMemReq.alignment %
8922  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
8923  }
8924 
8925  // 3. Allocate memory using allocator.
8926  res = allocator->AllocateMemory(
8927  vkMemReq,
8928  requiresDedicatedAllocation,
8929  prefersDedicatedAllocation,
8930  *pBuffer, // dedicatedBuffer
8931  VK_NULL_HANDLE, // dedicatedImage
8932  *pAllocationCreateInfo,
8933  VMA_SUBALLOCATION_TYPE_BUFFER,
8934  pAllocation);
8935  if(res >= 0)
8936  {
8937  // 3. Bind buffer with memory.
8938  res = (*allocator->GetVulkanFunctions().vkBindBufferMemory)(
8939  allocator->m_hDevice,
8940  *pBuffer,
8941  (*pAllocation)->GetMemory(),
8942  (*pAllocation)->GetOffset());
8943  if(res >= 0)
8944  {
8945  // All steps succeeded.
8946  if(pAllocationInfo != VMA_NULL)
8947  {
8948  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8949  }
8950  return VK_SUCCESS;
8951  }
8952  allocator->FreeMemory(*pAllocation);
8953  *pAllocation = VK_NULL_HANDLE;
8954  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8955  *pBuffer = VK_NULL_HANDLE;
8956  return res;
8957  }
8958  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8959  *pBuffer = VK_NULL_HANDLE;
8960  return res;
8961  }
8962  return res;
8963 }
8964 
8965 void vmaDestroyBuffer(
8966  VmaAllocator allocator,
8967  VkBuffer buffer,
8968  VmaAllocation allocation)
8969 {
8970  if(buffer != VK_NULL_HANDLE)
8971  {
8972  VMA_ASSERT(allocator);
8973 
8974  VMA_DEBUG_LOG("vmaDestroyBuffer");
8975 
8976  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8977 
8978  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
8979 
8980  allocator->FreeMemory(allocation);
8981  }
8982 }
8983 
8984 VkResult vmaCreateImage(
8985  VmaAllocator allocator,
8986  const VkImageCreateInfo* pImageCreateInfo,
8987  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8988  VkImage* pImage,
8989  VmaAllocation* pAllocation,
8990  VmaAllocationInfo* pAllocationInfo)
8991 {
8992  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
8993 
8994  VMA_DEBUG_LOG("vmaCreateImage");
8995 
8996  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8997 
8998  *pImage = VK_NULL_HANDLE;
8999  *pAllocation = VK_NULL_HANDLE;
9000 
9001  // 1. Create VkImage.
9002  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
9003  allocator->m_hDevice,
9004  pImageCreateInfo,
9005  allocator->GetAllocationCallbacks(),
9006  pImage);
9007  if(res >= 0)
9008  {
9009  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
9010  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
9011  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
9012 
9013  // 2. Allocate memory using allocator.
9014  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
9015  if(res >= 0)
9016  {
9017  // 3. Bind image with memory.
9018  res = (*allocator->GetVulkanFunctions().vkBindImageMemory)(
9019  allocator->m_hDevice,
9020  *pImage,
9021  (*pAllocation)->GetMemory(),
9022  (*pAllocation)->GetOffset());
9023  if(res >= 0)
9024  {
9025  // All steps succeeded.
9026  if(pAllocationInfo != VMA_NULL)
9027  {
9028  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
9029  }
9030  return VK_SUCCESS;
9031  }
9032  allocator->FreeMemory(*pAllocation);
9033  *pAllocation = VK_NULL_HANDLE;
9034  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
9035  *pImage = VK_NULL_HANDLE;
9036  return res;
9037  }
9038  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
9039  *pImage = VK_NULL_HANDLE;
9040  return res;
9041  }
9042  return res;
9043 }
9044 
9045 void vmaDestroyImage(
9046  VmaAllocator allocator,
9047  VkImage image,
9048  VmaAllocation allocation)
9049 {
9050  if(image != VK_NULL_HANDLE)
9051  {
9052  VMA_ASSERT(allocator);
9053 
9054  VMA_DEBUG_LOG("vmaDestroyImage");
9055 
9056  VMA_DEBUG_GLOBAL_MUTEX_LOCK
9057 
9058  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
9059 
9060  allocator->FreeMemory(allocation);
9061  }
9062 }
9063 
9064 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:1005
-
VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:1259
+Go to the documentation of this file.
1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
943 #include <vulkan/vulkan.h>
944 
945 VK_DEFINE_HANDLE(VmaAllocator)
946 
947 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
949  VmaAllocator allocator,
950  uint32_t memoryType,
951  VkDeviceMemory memory,
952  VkDeviceSize size);
954 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
955  VmaAllocator allocator,
956  uint32_t memoryType,
957  VkDeviceMemory memory,
958  VkDeviceSize size);
959 
967 typedef struct VmaDeviceMemoryCallbacks {
969  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
971  PFN_vmaFreeDeviceMemoryFunction pfnFree;
972 } VmaDeviceMemoryCallbacks;
973 
975 typedef enum VmaAllocatorCreateFlagBits {
980  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
1002  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
1003 
1004  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1005 } VmaAllocatorCreateFlagBits;
1006 typedef VkFlags VmaAllocatorCreateFlags;
1007 
1012 typedef struct VmaVulkanFunctions {
1013  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
1014  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
1015  PFN_vkAllocateMemory vkAllocateMemory;
1016  PFN_vkFreeMemory vkFreeMemory;
1017  PFN_vkMapMemory vkMapMemory;
1018  PFN_vkUnmapMemory vkUnmapMemory;
1019  PFN_vkBindBufferMemory vkBindBufferMemory;
1020  PFN_vkBindImageMemory vkBindImageMemory;
1021  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
1022  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
1023  PFN_vkCreateBuffer vkCreateBuffer;
1024  PFN_vkDestroyBuffer vkDestroyBuffer;
1025  PFN_vkCreateImage vkCreateImage;
1026  PFN_vkDestroyImage vkDestroyImage;
1027  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
1028  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
1029 } VmaVulkanFunctions;
1030 
1032 typedef struct VmaAllocatorCreateInfo
1033 {
1035  VmaAllocatorCreateFlags flags;
1037 
1038  VkPhysicalDevice physicalDevice;
1040 
1041  VkDevice device;
1043 
1044  VkDeviceSize preferredLargeHeapBlockSize;
1046 
1047  const VkAllocationCallbacks* pAllocationCallbacks;
1049 
1050  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
1064  uint32_t frameInUseCount;
1088  const VkDeviceSize* pHeapSizeLimit;
1100  const VmaVulkanFunctions* pVulkanFunctions;
1101 } VmaAllocatorCreateInfo;
1102 
1104 VkResult vmaCreateAllocator(
1105  const VmaAllocatorCreateInfo* pCreateInfo,
1106  VmaAllocator* pAllocator);
1107 
1109 void vmaDestroyAllocator(
1110  VmaAllocator allocator);
1111 
1116 void vmaGetPhysicalDeviceProperties(
1117  VmaAllocator allocator,
1118  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
1119 
1124 void vmaGetMemoryProperties(
1125  VmaAllocator allocator,
1126  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
1127 
1134 void vmaGetMemoryTypeProperties(
1135  VmaAllocator allocator,
1136  uint32_t memoryTypeIndex,
1137  VkMemoryPropertyFlags* pFlags);
1138 
1147 void vmaSetCurrentFrameIndex(
1148  VmaAllocator allocator,
1149  uint32_t frameIndex);
1150 
1153 typedef struct VmaStatInfo
1154 {
1156  uint32_t blockCount;
1158  uint32_t allocationCount;
1160  uint32_t unusedRangeCount;
1162  VkDeviceSize usedBytes;
1164  VkDeviceSize unusedBytes;
1165  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1166  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1167 } VmaStatInfo;
1168 
1170 typedef struct VmaStats
1171 {
1172  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1173  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1174  VmaStatInfo total;
1175 } VmaStats;
1176 
1178 void vmaCalculateStats(
1179  VmaAllocator allocator,
1180  VmaStats* pStats);
1181 
1182 #define VMA_STATS_STRING_ENABLED 1
1183 
1184 #if VMA_STATS_STRING_ENABLED
1185 
1187 
1189 void vmaBuildStatsString(
1190  VmaAllocator allocator,
1191  char** ppStatsString,
1192  VkBool32 detailedMap);
1193 
1194 void vmaFreeStatsString(
1195  VmaAllocator allocator,
1196  char* pStatsString);
1197 
1198 #endif // #if VMA_STATS_STRING_ENABLED
1199 
1200 VK_DEFINE_HANDLE(VmaPool)
1201 
1202 typedef enum VmaMemoryUsage
1203 {
1207  VMA_MEMORY_USAGE_UNKNOWN = 0,
1224  VMA_MEMORY_USAGE_GPU_ONLY = 1,
1234  VMA_MEMORY_USAGE_CPU_ONLY = 2,
1241  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1250  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1251  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1252 } VmaMemoryUsage;
1253 
1255 typedef enum VmaAllocationCreateFlagBits {
1267  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1268 
1278  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1291  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1304  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1311  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1317  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1318 
1319  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1320 } VmaAllocationCreateFlagBits;
1321 typedef VkFlags VmaAllocationCreateFlags;
1322 
1323 typedef struct VmaAllocationCreateInfo
1324 {
1326  VmaAllocationCreateFlags flags;
1332  VmaMemoryUsage usage;
1337  VkMemoryPropertyFlags requiredFlags;
1342  VkMemoryPropertyFlags preferredFlags;
1350  uint32_t memoryTypeBits;
1356  VmaPool pool;
1363  void* pUserData;
1364 } VmaAllocationCreateInfo;
1365 
1382 VkResult vmaFindMemoryTypeIndex(
1383  VmaAllocator allocator,
1384  uint32_t memoryTypeBits,
1385  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1386  uint32_t* pMemoryTypeIndex);
1387 
1400 VkResult vmaFindMemoryTypeIndexForBufferInfo(
1401  VmaAllocator allocator,
1402  const VkBufferCreateInfo* pBufferCreateInfo,
1403  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1404  uint32_t* pMemoryTypeIndex);
1405 
1418 VkResult vmaFindMemoryTypeIndexForImageInfo(
1419  VmaAllocator allocator,
1420  const VkImageCreateInfo* pImageCreateInfo,
1421  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1422  uint32_t* pMemoryTypeIndex);
1423 
1425 typedef enum VmaPoolCreateFlagBits {
1443  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1444 
1445  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1446 } VmaPoolCreateFlagBits;
1447 typedef VkFlags VmaPoolCreateFlags;
1448 
1451 typedef struct VmaPoolCreateInfo {
1454  uint32_t memoryTypeIndex;
1457  VmaPoolCreateFlags flags;
1462  VkDeviceSize blockSize;
1467  size_t minBlockCount;
1475  size_t maxBlockCount;
1489  uint32_t frameInUseCount;
1490 } VmaPoolCreateInfo;
1491 
1494 typedef struct VmaPoolStats {
1497  VkDeviceSize size;
1500  VkDeviceSize unusedSize;
1503  size_t allocationCount;
1506  size_t unusedRangeCount;
1513  VkDeviceSize unusedRangeSizeMax;
1514 } VmaPoolStats;
1515 
1522 VkResult vmaCreatePool(
1523  VmaAllocator allocator,
1524  const VmaPoolCreateInfo* pCreateInfo,
1525  VmaPool* pPool);
1526 
1529 void vmaDestroyPool(
1530  VmaAllocator allocator,
1531  VmaPool pool);
1532 
1539 void vmaGetPoolStats(
1540  VmaAllocator allocator,
1541  VmaPool pool,
1542  VmaPoolStats* pPoolStats);
1543 
1550 void vmaMakePoolAllocationsLost(
1551  VmaAllocator allocator,
1552  VmaPool pool,
1553  size_t* pLostAllocationCount);
1554 
1555 VK_DEFINE_HANDLE(VmaAllocation)
1556 
1557 
1559 typedef struct VmaAllocationInfo {
1564  uint32_t memoryType;
1573  VkDeviceMemory deviceMemory;
1578  VkDeviceSize offset;
1583  VkDeviceSize size;
1592  void* pMappedData;
1597  void* pUserData;
1598 } VmaAllocationInfo;
1599 
1610 VkResult vmaAllocateMemory(
1611  VmaAllocator allocator,
1612  const VkMemoryRequirements* pVkMemoryRequirements,
1613  const VmaAllocationCreateInfo* pCreateInfo,
1614  VmaAllocation* pAllocation,
1615  VmaAllocationInfo* pAllocationInfo);
1616 
1623 VkResult vmaAllocateMemoryForBuffer(
1624  VmaAllocator allocator,
1625  VkBuffer buffer,
1626  const VmaAllocationCreateInfo* pCreateInfo,
1627  VmaAllocation* pAllocation,
1628  VmaAllocationInfo* pAllocationInfo);
1629 
1631 VkResult vmaAllocateMemoryForImage(
1632  VmaAllocator allocator,
1633  VkImage image,
1634  const VmaAllocationCreateInfo* pCreateInfo,
1635  VmaAllocation* pAllocation,
1636  VmaAllocationInfo* pAllocationInfo);
1637 
1639 void vmaFreeMemory(
1640  VmaAllocator allocator,
1641  VmaAllocation allocation);
1642 
1659 void vmaGetAllocationInfo(
1660  VmaAllocator allocator,
1661  VmaAllocation allocation,
1662  VmaAllocationInfo* pAllocationInfo);
1663 
1678 VkBool32 vmaTouchAllocation(
1679  VmaAllocator allocator,
1680  VmaAllocation allocation);
1681 
1695 void vmaSetAllocationUserData(
1696  VmaAllocator allocator,
1697  VmaAllocation allocation,
1698  void* pUserData);
1699 
1710 void vmaCreateLostAllocation(
1711  VmaAllocator allocator,
1712  VmaAllocation* pAllocation);
1713 
1748 VkResult vmaMapMemory(
1749  VmaAllocator allocator,
1750  VmaAllocation allocation,
1751  void** ppData);
1752 
1757 void vmaUnmapMemory(
1758  VmaAllocator allocator,
1759  VmaAllocation allocation);
1760 
1762 typedef struct VmaDefragmentationInfo {
1767  VkDeviceSize maxBytesToMove;
1772  uint32_t maxAllocationsToMove;
1773 } VmaDefragmentationInfo;
1774 
1776 typedef struct VmaDefragmentationStats {
1778  VkDeviceSize bytesMoved;
1780  VkDeviceSize bytesFreed;
1782  uint32_t allocationsMoved;
1784  uint32_t deviceMemoryBlocksFreed;
1785 } VmaDefragmentationStats;
1786 
1869 VkResult vmaDefragment(
1870  VmaAllocator allocator,
1871  VmaAllocation* pAllocations,
1872  size_t allocationCount,
1873  VkBool32* pAllocationsChanged,
1874  const VmaDefragmentationInfo *pDefragmentationInfo,
1875  VmaDefragmentationStats* pDefragmentationStats);
1876 
1903 VkResult vmaCreateBuffer(
1904  VmaAllocator allocator,
1905  const VkBufferCreateInfo* pBufferCreateInfo,
1906  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1907  VkBuffer* pBuffer,
1908  VmaAllocation* pAllocation,
1909  VmaAllocationInfo* pAllocationInfo);
1910 
1922 void vmaDestroyBuffer(
1923  VmaAllocator allocator,
1924  VkBuffer buffer,
1925  VmaAllocation allocation);
1926 
1928 VkResult vmaCreateImage(
1929  VmaAllocator allocator,
1930  const VkImageCreateInfo* pImageCreateInfo,
1931  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1932  VkImage* pImage,
1933  VmaAllocation* pAllocation,
1934  VmaAllocationInfo* pAllocationInfo);
1935 
1947 void vmaDestroyImage(
1948  VmaAllocator allocator,
1949  VkImage image,
1950  VmaAllocation allocation);
1951 
1952 #ifdef __cplusplus
1953 }
1954 #endif
1955 
1956 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
1957 
1958 // For Visual Studio IntelliSense.
1959 #ifdef __INTELLISENSE__
1960 #define VMA_IMPLEMENTATION
1961 #endif
1962 
1963 #ifdef VMA_IMPLEMENTATION
1964 #undef VMA_IMPLEMENTATION
1965 
1966 #include <cstdint>
1967 #include <cstdlib>
1968 #include <cstring>
1969 
1970 /*******************************************************************************
1971 CONFIGURATION SECTION
1972 
1973 Define some of these macros before each #include of this header or change them
1974 here if you need other then default behavior depending on your environment.
1975 */
1976 
1977 /*
1978 Define this macro to 1 to make the library fetch pointers to Vulkan functions
1979 internally, like:
1980 
1981  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1982 
1983 Define to 0 if you are going to provide you own pointers to Vulkan functions via
1984 VmaAllocatorCreateInfo::pVulkanFunctions.
1985 */
1986 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
1987 #define VMA_STATIC_VULKAN_FUNCTIONS 1
1988 #endif
1989 
1990 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
1991 //#define VMA_USE_STL_CONTAINERS 1
1992 
1993 /* Set this macro to 1 to make the library including and using STL containers:
1994 std::pair, std::vector, std::list, std::unordered_map.
1995 
1996 Set it to 0 or undefined to make the library using its own implementation of
1997 the containers.
1998 */
1999 #if VMA_USE_STL_CONTAINERS
2000  #define VMA_USE_STL_VECTOR 1
2001  #define VMA_USE_STL_UNORDERED_MAP 1
2002  #define VMA_USE_STL_LIST 1
2003 #endif
2004 
2005 #if VMA_USE_STL_VECTOR
2006  #include <vector>
2007 #endif
2008 
2009 #if VMA_USE_STL_UNORDERED_MAP
2010  #include <unordered_map>
2011 #endif
2012 
2013 #if VMA_USE_STL_LIST
2014  #include <list>
2015 #endif
2016 
2017 /*
2018 Following headers are used in this CONFIGURATION section only, so feel free to
2019 remove them if not needed.
2020 */
2021 #include <cassert> // for assert
2022 #include <algorithm> // for min, max
2023 #include <mutex> // for std::mutex
2024 #include <atomic> // for std::atomic
2025 
2026 #if !defined(_WIN32) && !defined(__APPLE__)
2027  #include <malloc.h> // for aligned_alloc()
2028 #endif
2029 
2030 #ifndef VMA_NULL
2031  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
2032  #define VMA_NULL nullptr
2033 #endif
2034 
2035 #if defined(__APPLE__) || defined(__ANDROID__)
2036 #include <cstdlib>
2037 void *aligned_alloc(size_t alignment, size_t size)
2038 {
2039  // alignment must be >= sizeof(void*)
2040  if(alignment < sizeof(void*))
2041  {
2042  alignment = sizeof(void*);
2043  }
2044 
2045  void *pointer;
2046  if(posix_memalign(&pointer, alignment, size) == 0)
2047  return pointer;
2048  return VMA_NULL;
2049 }
2050 #endif
2051 
2052 // Normal assert to check for programmer's errors, especially in Debug configuration.
2053 #ifndef VMA_ASSERT
2054  #ifdef _DEBUG
2055  #define VMA_ASSERT(expr) assert(expr)
2056  #else
2057  #define VMA_ASSERT(expr)
2058  #endif
2059 #endif
2060 
2061 // Assert that will be called very often, like inside data structures e.g. operator[].
2062 // Making it non-empty can make program slow.
2063 #ifndef VMA_HEAVY_ASSERT
2064  #ifdef _DEBUG
2065  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
2066  #else
2067  #define VMA_HEAVY_ASSERT(expr)
2068  #endif
2069 #endif
2070 
2071 #ifndef VMA_ALIGN_OF
2072  #define VMA_ALIGN_OF(type) (__alignof(type))
2073 #endif
2074 
2075 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
2076  #if defined(_WIN32)
2077  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
2078  #else
2079  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
2080  #endif
2081 #endif
2082 
2083 #ifndef VMA_SYSTEM_FREE
2084  #if defined(_WIN32)
2085  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
2086  #else
2087  #define VMA_SYSTEM_FREE(ptr) free(ptr)
2088  #endif
2089 #endif
2090 
2091 #ifndef VMA_MIN
2092  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
2093 #endif
2094 
2095 #ifndef VMA_MAX
2096  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
2097 #endif
2098 
2099 #ifndef VMA_SWAP
2100  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
2101 #endif
2102 
2103 #ifndef VMA_SORT
2104  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
2105 #endif
2106 
2107 #ifndef VMA_DEBUG_LOG
2108  #define VMA_DEBUG_LOG(format, ...)
2109  /*
2110  #define VMA_DEBUG_LOG(format, ...) do { \
2111  printf(format, __VA_ARGS__); \
2112  printf("\n"); \
2113  } while(false)
2114  */
2115 #endif
2116 
2117 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
2118 #if VMA_STATS_STRING_ENABLED
2119  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
2120  {
2121  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
2122  }
2123  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
2124  {
2125  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
2126  }
2127  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
2128  {
2129  snprintf(outStr, strLen, "%p", ptr);
2130  }
2131 #endif
2132 
2133 #ifndef VMA_MUTEX
2134  class VmaMutex
2135  {
2136  public:
2137  VmaMutex() { }
2138  ~VmaMutex() { }
2139  void Lock() { m_Mutex.lock(); }
2140  void Unlock() { m_Mutex.unlock(); }
2141  private:
2142  std::mutex m_Mutex;
2143  };
2144  #define VMA_MUTEX VmaMutex
2145 #endif
2146 
2147 /*
2148 If providing your own implementation, you need to implement a subset of std::atomic:
2149 
2150 - Constructor(uint32_t desired)
2151 - uint32_t load() const
2152 - void store(uint32_t desired)
2153 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
2154 */
2155 #ifndef VMA_ATOMIC_UINT32
2156  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
2157 #endif
2158 
2159 #ifndef VMA_BEST_FIT
2160 
2172  #define VMA_BEST_FIT (1)
2173 #endif
2174 
2175 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
2176 
2180  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
2181 #endif
2182 
2183 #ifndef VMA_DEBUG_ALIGNMENT
2184 
2188  #define VMA_DEBUG_ALIGNMENT (1)
2189 #endif
2190 
2191 #ifndef VMA_DEBUG_MARGIN
2192 
2196  #define VMA_DEBUG_MARGIN (0)
2197 #endif
2198 
2199 #ifndef VMA_DEBUG_GLOBAL_MUTEX
2200 
2204  #define VMA_DEBUG_GLOBAL_MUTEX (0)
2205 #endif
2206 
2207 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
2208 
2212  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
2213 #endif
2214 
2215 #ifndef VMA_SMALL_HEAP_MAX_SIZE
2216  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
2218 #endif
2219 
2220 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
2221  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
2223 #endif
2224 
2225 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
2226 
2227 /*******************************************************************************
2228 END OF CONFIGURATION
2229 */
2230 
2231 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
2232  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
2233 
2234 // Returns number of bits set to 1 in (v).
2235 static inline uint32_t VmaCountBitsSet(uint32_t v)
2236 {
2237  uint32_t c = v - ((v >> 1) & 0x55555555);
2238  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
2239  c = ((c >> 4) + c) & 0x0F0F0F0F;
2240  c = ((c >> 8) + c) & 0x00FF00FF;
2241  c = ((c >> 16) + c) & 0x0000FFFF;
2242  return c;
2243 }
2244 
2245 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2246 // Use types like uint32_t, uint64_t as T.
2247 template <typename T>
2248 static inline T VmaAlignUp(T val, T align)
2249 {
2250  return (val + align - 1) / align * align;
2251 }
2252 
2253 // Division with mathematical rounding to nearest number.
2254 template <typename T>
2255 inline T VmaRoundDiv(T x, T y)
2256 {
2257  return (x + (y / (T)2)) / y;
2258 }
2259 
2260 #ifndef VMA_SORT
2261 
2262 template<typename Iterator, typename Compare>
2263 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2264 {
2265  Iterator centerValue = end; --centerValue;
2266  Iterator insertIndex = beg;
2267  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2268  {
2269  if(cmp(*memTypeIndex, *centerValue))
2270  {
2271  if(insertIndex != memTypeIndex)
2272  {
2273  VMA_SWAP(*memTypeIndex, *insertIndex);
2274  }
2275  ++insertIndex;
2276  }
2277  }
2278  if(insertIndex != centerValue)
2279  {
2280  VMA_SWAP(*insertIndex, *centerValue);
2281  }
2282  return insertIndex;
2283 }
2284 
2285 template<typename Iterator, typename Compare>
2286 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2287 {
2288  if(beg < end)
2289  {
2290  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2291  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2292  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2293  }
2294 }
2295 
2296 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2297 
2298 #endif // #ifndef VMA_SORT
2299 
2300 /*
2301 Returns true if two memory blocks occupy overlapping pages.
2302 ResourceA must be in less memory offset than ResourceB.
2303 
2304 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2305 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2306 */
2307 static inline bool VmaBlocksOnSamePage(
2308  VkDeviceSize resourceAOffset,
2309  VkDeviceSize resourceASize,
2310  VkDeviceSize resourceBOffset,
2311  VkDeviceSize pageSize)
2312 {
2313  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2314  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2315  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2316  VkDeviceSize resourceBStart = resourceBOffset;
2317  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2318  return resourceAEndPage == resourceBStartPage;
2319 }
2320 
2321 enum VmaSuballocationType
2322 {
2323  VMA_SUBALLOCATION_TYPE_FREE = 0,
2324  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2325  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2326  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2327  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2328  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2329  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2330 };
2331 
2332 /*
2333 Returns true if given suballocation types could conflict and must respect
2334 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2335 or linear image and another one is optimal image. If type is unknown, behave
2336 conservatively.
2337 */
2338 static inline bool VmaIsBufferImageGranularityConflict(
2339  VmaSuballocationType suballocType1,
2340  VmaSuballocationType suballocType2)
2341 {
2342  if(suballocType1 > suballocType2)
2343  {
2344  VMA_SWAP(suballocType1, suballocType2);
2345  }
2346 
2347  switch(suballocType1)
2348  {
2349  case VMA_SUBALLOCATION_TYPE_FREE:
2350  return false;
2351  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
2352  return true;
2353  case VMA_SUBALLOCATION_TYPE_BUFFER:
2354  return
2355  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2356  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2357  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
2358  return
2359  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2360  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
2361  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2362  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
2363  return
2364  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2365  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
2366  return false;
2367  default:
2368  VMA_ASSERT(0);
2369  return true;
2370  }
2371 }
2372 
2373 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
2374 struct VmaMutexLock
2375 {
2376 public:
2377  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
2378  m_pMutex(useMutex ? &mutex : VMA_NULL)
2379  {
2380  if(m_pMutex)
2381  {
2382  m_pMutex->Lock();
2383  }
2384  }
2385 
2386  ~VmaMutexLock()
2387  {
2388  if(m_pMutex)
2389  {
2390  m_pMutex->Unlock();
2391  }
2392  }
2393 
2394 private:
2395  VMA_MUTEX* m_pMutex;
2396 };
2397 
2398 #if VMA_DEBUG_GLOBAL_MUTEX
2399  static VMA_MUTEX gDebugGlobalMutex;
2400  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
2401 #else
2402  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
2403 #endif
2404 
2405 // Minimum size of a free suballocation to register it in the free suballocation collection.
2406 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
2407 
2408 /*
2409 Performs binary search and returns iterator to first element that is greater or
2410 equal to (key), according to comparison (cmp).
2411 
2412 Cmp should return true if first argument is less than second argument.
2413 
2414 Returned value is the found element, if present in the collection or place where
2415 new element with value (key) should be inserted.
2416 */
2417 template <typename IterT, typename KeyT, typename CmpT>
2418 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
2419 {
2420  size_t down = 0, up = (end - beg);
2421  while(down < up)
2422  {
2423  const size_t mid = (down + up) / 2;
2424  if(cmp(*(beg+mid), key))
2425  {
2426  down = mid + 1;
2427  }
2428  else
2429  {
2430  up = mid;
2431  }
2432  }
2433  return beg + down;
2434 }
2435 
2437 // Memory allocation
2438 
2439 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
2440 {
2441  if((pAllocationCallbacks != VMA_NULL) &&
2442  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
2443  {
2444  return (*pAllocationCallbacks->pfnAllocation)(
2445  pAllocationCallbacks->pUserData,
2446  size,
2447  alignment,
2448  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2449  }
2450  else
2451  {
2452  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
2453  }
2454 }
2455 
2456 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
2457 {
2458  if((pAllocationCallbacks != VMA_NULL) &&
2459  (pAllocationCallbacks->pfnFree != VMA_NULL))
2460  {
2461  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
2462  }
2463  else
2464  {
2465  VMA_SYSTEM_FREE(ptr);
2466  }
2467 }
2468 
2469 template<typename T>
2470 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
2471 {
2472  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
2473 }
2474 
2475 template<typename T>
2476 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
2477 {
2478  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
2479 }
2480 
2481 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
2482 
2483 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
2484 
2485 template<typename T>
2486 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
2487 {
2488  ptr->~T();
2489  VmaFree(pAllocationCallbacks, ptr);
2490 }
2491 
2492 template<typename T>
2493 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
2494 {
2495  if(ptr != VMA_NULL)
2496  {
2497  for(size_t i = count; i--; )
2498  {
2499  ptr[i].~T();
2500  }
2501  VmaFree(pAllocationCallbacks, ptr);
2502  }
2503 }
2504 
2505 // STL-compatible allocator.
2506 template<typename T>
2507 class VmaStlAllocator
2508 {
2509 public:
2510  const VkAllocationCallbacks* const m_pCallbacks;
2511  typedef T value_type;
2512 
2513  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
2514  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
2515 
2516  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
2517  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
2518 
2519  template<typename U>
2520  bool operator==(const VmaStlAllocator<U>& rhs) const
2521  {
2522  return m_pCallbacks == rhs.m_pCallbacks;
2523  }
2524  template<typename U>
2525  bool operator!=(const VmaStlAllocator<U>& rhs) const
2526  {
2527  return m_pCallbacks != rhs.m_pCallbacks;
2528  }
2529 
2530  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
2531 };
2532 
2533 #if VMA_USE_STL_VECTOR
2534 
2535 #define VmaVector std::vector
2536 
2537 template<typename T, typename allocatorT>
2538 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
2539 {
2540  vec.insert(vec.begin() + index, item);
2541 }
2542 
2543 template<typename T, typename allocatorT>
2544 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
2545 {
2546  vec.erase(vec.begin() + index);
2547 }
2548 
2549 #else // #if VMA_USE_STL_VECTOR
2550 
2551 /* Class with interface compatible with subset of std::vector.
2552 T must be POD because constructors and destructors are not called and memcpy is
2553 used for these objects. */
2554 template<typename T, typename AllocatorT>
2555 class VmaVector
2556 {
2557 public:
2558  typedef T value_type;
2559 
2560  VmaVector(const AllocatorT& allocator) :
2561  m_Allocator(allocator),
2562  m_pArray(VMA_NULL),
2563  m_Count(0),
2564  m_Capacity(0)
2565  {
2566  }
2567 
2568  VmaVector(size_t count, const AllocatorT& allocator) :
2569  m_Allocator(allocator),
2570  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
2571  m_Count(count),
2572  m_Capacity(count)
2573  {
2574  }
2575 
2576  VmaVector(const VmaVector<T, AllocatorT>& src) :
2577  m_Allocator(src.m_Allocator),
2578  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
2579  m_Count(src.m_Count),
2580  m_Capacity(src.m_Count)
2581  {
2582  if(m_Count != 0)
2583  {
2584  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
2585  }
2586  }
2587 
2588  ~VmaVector()
2589  {
2590  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2591  }
2592 
2593  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
2594  {
2595  if(&rhs != this)
2596  {
2597  resize(rhs.m_Count);
2598  if(m_Count != 0)
2599  {
2600  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
2601  }
2602  }
2603  return *this;
2604  }
2605 
2606  bool empty() const { return m_Count == 0; }
2607  size_t size() const { return m_Count; }
2608  T* data() { return m_pArray; }
2609  const T* data() const { return m_pArray; }
2610 
2611  T& operator[](size_t index)
2612  {
2613  VMA_HEAVY_ASSERT(index < m_Count);
2614  return m_pArray[index];
2615  }
2616  const T& operator[](size_t index) const
2617  {
2618  VMA_HEAVY_ASSERT(index < m_Count);
2619  return m_pArray[index];
2620  }
2621 
2622  T& front()
2623  {
2624  VMA_HEAVY_ASSERT(m_Count > 0);
2625  return m_pArray[0];
2626  }
2627  const T& front() const
2628  {
2629  VMA_HEAVY_ASSERT(m_Count > 0);
2630  return m_pArray[0];
2631  }
2632  T& back()
2633  {
2634  VMA_HEAVY_ASSERT(m_Count > 0);
2635  return m_pArray[m_Count - 1];
2636  }
2637  const T& back() const
2638  {
2639  VMA_HEAVY_ASSERT(m_Count > 0);
2640  return m_pArray[m_Count - 1];
2641  }
2642 
2643  void reserve(size_t newCapacity, bool freeMemory = false)
2644  {
2645  newCapacity = VMA_MAX(newCapacity, m_Count);
2646 
2647  if((newCapacity < m_Capacity) && !freeMemory)
2648  {
2649  newCapacity = m_Capacity;
2650  }
2651 
2652  if(newCapacity != m_Capacity)
2653  {
2654  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
2655  if(m_Count != 0)
2656  {
2657  memcpy(newArray, m_pArray, m_Count * sizeof(T));
2658  }
2659  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2660  m_Capacity = newCapacity;
2661  m_pArray = newArray;
2662  }
2663  }
2664 
2665  void resize(size_t newCount, bool freeMemory = false)
2666  {
2667  size_t newCapacity = m_Capacity;
2668  if(newCount > m_Capacity)
2669  {
2670  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
2671  }
2672  else if(freeMemory)
2673  {
2674  newCapacity = newCount;
2675  }
2676 
2677  if(newCapacity != m_Capacity)
2678  {
2679  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
2680  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
2681  if(elementsToCopy != 0)
2682  {
2683  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
2684  }
2685  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2686  m_Capacity = newCapacity;
2687  m_pArray = newArray;
2688  }
2689 
2690  m_Count = newCount;
2691  }
2692 
2693  void clear(bool freeMemory = false)
2694  {
2695  resize(0, freeMemory);
2696  }
2697 
2698  void insert(size_t index, const T& src)
2699  {
2700  VMA_HEAVY_ASSERT(index <= m_Count);
2701  const size_t oldCount = size();
2702  resize(oldCount + 1);
2703  if(index < oldCount)
2704  {
2705  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
2706  }
2707  m_pArray[index] = src;
2708  }
2709 
2710  void remove(size_t index)
2711  {
2712  VMA_HEAVY_ASSERT(index < m_Count);
2713  const size_t oldCount = size();
2714  if(index < oldCount - 1)
2715  {
2716  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
2717  }
2718  resize(oldCount - 1);
2719  }
2720 
2721  void push_back(const T& src)
2722  {
2723  const size_t newIndex = size();
2724  resize(newIndex + 1);
2725  m_pArray[newIndex] = src;
2726  }
2727 
2728  void pop_back()
2729  {
2730  VMA_HEAVY_ASSERT(m_Count > 0);
2731  resize(size() - 1);
2732  }
2733 
2734  void push_front(const T& src)
2735  {
2736  insert(0, src);
2737  }
2738 
2739  void pop_front()
2740  {
2741  VMA_HEAVY_ASSERT(m_Count > 0);
2742  remove(0);
2743  }
2744 
2745  typedef T* iterator;
2746 
2747  iterator begin() { return m_pArray; }
2748  iterator end() { return m_pArray + m_Count; }
2749 
2750 private:
2751  AllocatorT m_Allocator;
2752  T* m_pArray;
2753  size_t m_Count;
2754  size_t m_Capacity;
2755 };
2756 
2757 template<typename T, typename allocatorT>
2758 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2759 {
2760  vec.insert(index, item);
2761 }
2762 
2763 template<typename T, typename allocatorT>
2764 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2765 {
2766  vec.remove(index);
2767 }
2768 
2769 #endif // #if VMA_USE_STL_VECTOR
2770 
2771 template<typename CmpLess, typename VectorT>
2772 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2773 {
2774  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2775  vector.data(),
2776  vector.data() + vector.size(),
2777  value,
2778  CmpLess()) - vector.data();
2779  VmaVectorInsert(vector, indexToInsert, value);
2780  return indexToInsert;
2781 }
2782 
2783 template<typename CmpLess, typename VectorT>
2784 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2785 {
2786  CmpLess comparator;
2787  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2788  vector.begin(),
2789  vector.end(),
2790  value,
2791  comparator);
2792  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
2793  {
2794  size_t indexToRemove = it - vector.begin();
2795  VmaVectorRemove(vector, indexToRemove);
2796  return true;
2797  }
2798  return false;
2799 }
2800 
2801 template<typename CmpLess, typename VectorT>
2802 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
2803 {
2804  CmpLess comparator;
2805  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2806  vector.data(),
2807  vector.data() + vector.size(),
2808  value,
2809  comparator);
2810  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
2811  {
2812  return it - vector.begin();
2813  }
2814  else
2815  {
2816  return vector.size();
2817  }
2818 }
2819 
2821 // class VmaPoolAllocator
2822 
2823 /*
2824 Allocator for objects of type T using a list of arrays (pools) to speed up
2825 allocation. Number of elements that can be allocated is not bounded because
2826 allocator can create multiple blocks.
2827 */
2828 template<typename T>
2829 class VmaPoolAllocator
2830 {
2831 public:
2832  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
2833  ~VmaPoolAllocator();
2834  void Clear();
2835  T* Alloc();
2836  void Free(T* ptr);
2837 
2838 private:
2839  union Item
2840  {
2841  uint32_t NextFreeIndex;
2842  T Value;
2843  };
2844 
2845  struct ItemBlock
2846  {
2847  Item* pItems;
2848  uint32_t FirstFreeIndex;
2849  };
2850 
2851  const VkAllocationCallbacks* m_pAllocationCallbacks;
2852  size_t m_ItemsPerBlock;
2853  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
2854 
2855  ItemBlock& CreateNewBlock();
2856 };
2857 
2858 template<typename T>
2859 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
2860  m_pAllocationCallbacks(pAllocationCallbacks),
2861  m_ItemsPerBlock(itemsPerBlock),
2862  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
2863 {
2864  VMA_ASSERT(itemsPerBlock > 0);
2865 }
2866 
2867 template<typename T>
2868 VmaPoolAllocator<T>::~VmaPoolAllocator()
2869 {
2870  Clear();
2871 }
2872 
2873 template<typename T>
2874 void VmaPoolAllocator<T>::Clear()
2875 {
2876  for(size_t i = m_ItemBlocks.size(); i--; )
2877  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
2878  m_ItemBlocks.clear();
2879 }
2880 
2881 template<typename T>
2882 T* VmaPoolAllocator<T>::Alloc()
2883 {
2884  for(size_t i = m_ItemBlocks.size(); i--; )
2885  {
2886  ItemBlock& block = m_ItemBlocks[i];
2887  // This block has some free items: Use first one.
2888  if(block.FirstFreeIndex != UINT32_MAX)
2889  {
2890  Item* const pItem = &block.pItems[block.FirstFreeIndex];
2891  block.FirstFreeIndex = pItem->NextFreeIndex;
2892  return &pItem->Value;
2893  }
2894  }
2895 
2896  // No block has free item: Create new one and use it.
2897  ItemBlock& newBlock = CreateNewBlock();
2898  Item* const pItem = &newBlock.pItems[0];
2899  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
2900  return &pItem->Value;
2901 }
2902 
2903 template<typename T>
2904 void VmaPoolAllocator<T>::Free(T* ptr)
2905 {
2906  // Search all memory blocks to find ptr.
2907  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
2908  {
2909  ItemBlock& block = m_ItemBlocks[i];
2910 
2911  // Casting to union.
2912  Item* pItemPtr;
2913  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
2914 
2915  // Check if pItemPtr is in address range of this block.
2916  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
2917  {
2918  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
2919  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
2920  block.FirstFreeIndex = index;
2921  return;
2922  }
2923  }
2924  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
2925 }
2926 
2927 template<typename T>
2928 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
2929 {
2930  ItemBlock newBlock = {
2931  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
2932 
2933  m_ItemBlocks.push_back(newBlock);
2934 
2935  // Setup singly-linked list of all free items in this block.
2936  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
2937  newBlock.pItems[i].NextFreeIndex = i + 1;
2938  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
2939  return m_ItemBlocks.back();
2940 }
2941 
2943 // class VmaRawList, VmaList
2944 
2945 #if VMA_USE_STL_LIST
2946 
2947 #define VmaList std::list
2948 
2949 #else // #if VMA_USE_STL_LIST
2950 
2951 template<typename T>
2952 struct VmaListItem
2953 {
2954  VmaListItem* pPrev;
2955  VmaListItem* pNext;
2956  T Value;
2957 };
2958 
2959 // Doubly linked list.
2960 template<typename T>
2961 class VmaRawList
2962 {
2963 public:
2964  typedef VmaListItem<T> ItemType;
2965 
2966  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
2967  ~VmaRawList();
2968  void Clear();
2969 
2970  size_t GetCount() const { return m_Count; }
2971  bool IsEmpty() const { return m_Count == 0; }
2972 
2973  ItemType* Front() { return m_pFront; }
2974  const ItemType* Front() const { return m_pFront; }
2975  ItemType* Back() { return m_pBack; }
2976  const ItemType* Back() const { return m_pBack; }
2977 
2978  ItemType* PushBack();
2979  ItemType* PushFront();
2980  ItemType* PushBack(const T& value);
2981  ItemType* PushFront(const T& value);
2982  void PopBack();
2983  void PopFront();
2984 
2985  // Item can be null - it means PushBack.
2986  ItemType* InsertBefore(ItemType* pItem);
2987  // Item can be null - it means PushFront.
2988  ItemType* InsertAfter(ItemType* pItem);
2989 
2990  ItemType* InsertBefore(ItemType* pItem, const T& value);
2991  ItemType* InsertAfter(ItemType* pItem, const T& value);
2992 
2993  void Remove(ItemType* pItem);
2994 
2995 private:
2996  const VkAllocationCallbacks* const m_pAllocationCallbacks;
2997  VmaPoolAllocator<ItemType> m_ItemAllocator;
2998  ItemType* m_pFront;
2999  ItemType* m_pBack;
3000  size_t m_Count;
3001 
3002  // Declared not defined, to block copy constructor and assignment operator.
3003  VmaRawList(const VmaRawList<T>& src);
3004  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
3005 };
3006 
3007 template<typename T>
3008 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
3009  m_pAllocationCallbacks(pAllocationCallbacks),
3010  m_ItemAllocator(pAllocationCallbacks, 128),
3011  m_pFront(VMA_NULL),
3012  m_pBack(VMA_NULL),
3013  m_Count(0)
3014 {
3015 }
3016 
3017 template<typename T>
3018 VmaRawList<T>::~VmaRawList()
3019 {
3020  // Intentionally not calling Clear, because that would be unnecessary
3021  // computations to return all items to m_ItemAllocator as free.
3022 }
3023 
3024 template<typename T>
3025 void VmaRawList<T>::Clear()
3026 {
3027  if(IsEmpty() == false)
3028  {
3029  ItemType* pItem = m_pBack;
3030  while(pItem != VMA_NULL)
3031  {
3032  ItemType* const pPrevItem = pItem->pPrev;
3033  m_ItemAllocator.Free(pItem);
3034  pItem = pPrevItem;
3035  }
3036  m_pFront = VMA_NULL;
3037  m_pBack = VMA_NULL;
3038  m_Count = 0;
3039  }
3040 }
3041 
3042 template<typename T>
3043 VmaListItem<T>* VmaRawList<T>::PushBack()
3044 {
3045  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3046  pNewItem->pNext = VMA_NULL;
3047  if(IsEmpty())
3048  {
3049  pNewItem->pPrev = VMA_NULL;
3050  m_pFront = pNewItem;
3051  m_pBack = pNewItem;
3052  m_Count = 1;
3053  }
3054  else
3055  {
3056  pNewItem->pPrev = m_pBack;
3057  m_pBack->pNext = pNewItem;
3058  m_pBack = pNewItem;
3059  ++m_Count;
3060  }
3061  return pNewItem;
3062 }
3063 
3064 template<typename T>
3065 VmaListItem<T>* VmaRawList<T>::PushFront()
3066 {
3067  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3068  pNewItem->pPrev = VMA_NULL;
3069  if(IsEmpty())
3070  {
3071  pNewItem->pNext = VMA_NULL;
3072  m_pFront = pNewItem;
3073  m_pBack = pNewItem;
3074  m_Count = 1;
3075  }
3076  else
3077  {
3078  pNewItem->pNext = m_pFront;
3079  m_pFront->pPrev = pNewItem;
3080  m_pFront = pNewItem;
3081  ++m_Count;
3082  }
3083  return pNewItem;
3084 }
3085 
3086 template<typename T>
3087 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
3088 {
3089  ItemType* const pNewItem = PushBack();
3090  pNewItem->Value = value;
3091  return pNewItem;
3092 }
3093 
3094 template<typename T>
3095 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
3096 {
3097  ItemType* const pNewItem = PushFront();
3098  pNewItem->Value = value;
3099  return pNewItem;
3100 }
3101 
3102 template<typename T>
3103 void VmaRawList<T>::PopBack()
3104 {
3105  VMA_HEAVY_ASSERT(m_Count > 0);
3106  ItemType* const pBackItem = m_pBack;
3107  ItemType* const pPrevItem = pBackItem->pPrev;
3108  if(pPrevItem != VMA_NULL)
3109  {
3110  pPrevItem->pNext = VMA_NULL;
3111  }
3112  m_pBack = pPrevItem;
3113  m_ItemAllocator.Free(pBackItem);
3114  --m_Count;
3115 }
3116 
3117 template<typename T>
3118 void VmaRawList<T>::PopFront()
3119 {
3120  VMA_HEAVY_ASSERT(m_Count > 0);
3121  ItemType* const pFrontItem = m_pFront;
3122  ItemType* const pNextItem = pFrontItem->pNext;
3123  if(pNextItem != VMA_NULL)
3124  {
3125  pNextItem->pPrev = VMA_NULL;
3126  }
3127  m_pFront = pNextItem;
3128  m_ItemAllocator.Free(pFrontItem);
3129  --m_Count;
3130 }
3131 
3132 template<typename T>
3133 void VmaRawList<T>::Remove(ItemType* pItem)
3134 {
3135  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
3136  VMA_HEAVY_ASSERT(m_Count > 0);
3137 
3138  if(pItem->pPrev != VMA_NULL)
3139  {
3140  pItem->pPrev->pNext = pItem->pNext;
3141  }
3142  else
3143  {
3144  VMA_HEAVY_ASSERT(m_pFront == pItem);
3145  m_pFront = pItem->pNext;
3146  }
3147 
3148  if(pItem->pNext != VMA_NULL)
3149  {
3150  pItem->pNext->pPrev = pItem->pPrev;
3151  }
3152  else
3153  {
3154  VMA_HEAVY_ASSERT(m_pBack == pItem);
3155  m_pBack = pItem->pPrev;
3156  }
3157 
3158  m_ItemAllocator.Free(pItem);
3159  --m_Count;
3160 }
3161 
3162 template<typename T>
3163 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
3164 {
3165  if(pItem != VMA_NULL)
3166  {
3167  ItemType* const prevItem = pItem->pPrev;
3168  ItemType* const newItem = m_ItemAllocator.Alloc();
3169  newItem->pPrev = prevItem;
3170  newItem->pNext = pItem;
3171  pItem->pPrev = newItem;
3172  if(prevItem != VMA_NULL)
3173  {
3174  prevItem->pNext = newItem;
3175  }
3176  else
3177  {
3178  VMA_HEAVY_ASSERT(m_pFront == pItem);
3179  m_pFront = newItem;
3180  }
3181  ++m_Count;
3182  return newItem;
3183  }
3184  else
3185  return PushBack();
3186 }
3187 
3188 template<typename T>
3189 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
3190 {
3191  if(pItem != VMA_NULL)
3192  {
3193  ItemType* const nextItem = pItem->pNext;
3194  ItemType* const newItem = m_ItemAllocator.Alloc();
3195  newItem->pNext = nextItem;
3196  newItem->pPrev = pItem;
3197  pItem->pNext = newItem;
3198  if(nextItem != VMA_NULL)
3199  {
3200  nextItem->pPrev = newItem;
3201  }
3202  else
3203  {
3204  VMA_HEAVY_ASSERT(m_pBack == pItem);
3205  m_pBack = newItem;
3206  }
3207  ++m_Count;
3208  return newItem;
3209  }
3210  else
3211  return PushFront();
3212 }
3213 
3214 template<typename T>
3215 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
3216 {
3217  ItemType* const newItem = InsertBefore(pItem);
3218  newItem->Value = value;
3219  return newItem;
3220 }
3221 
3222 template<typename T>
3223 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
3224 {
3225  ItemType* const newItem = InsertAfter(pItem);
3226  newItem->Value = value;
3227  return newItem;
3228 }
3229 
3230 template<typename T, typename AllocatorT>
3231 class VmaList
3232 {
3233 public:
3234  class iterator
3235  {
3236  public:
3237  iterator() :
3238  m_pList(VMA_NULL),
3239  m_pItem(VMA_NULL)
3240  {
3241  }
3242 
3243  T& operator*() const
3244  {
3245  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3246  return m_pItem->Value;
3247  }
3248  T* operator->() const
3249  {
3250  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3251  return &m_pItem->Value;
3252  }
3253 
3254  iterator& operator++()
3255  {
3256  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3257  m_pItem = m_pItem->pNext;
3258  return *this;
3259  }
3260  iterator& operator--()
3261  {
3262  if(m_pItem != VMA_NULL)
3263  {
3264  m_pItem = m_pItem->pPrev;
3265  }
3266  else
3267  {
3268  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3269  m_pItem = m_pList->Back();
3270  }
3271  return *this;
3272  }
3273 
3274  iterator operator++(int)
3275  {
3276  iterator result = *this;
3277  ++*this;
3278  return result;
3279  }
3280  iterator operator--(int)
3281  {
3282  iterator result = *this;
3283  --*this;
3284  return result;
3285  }
3286 
3287  bool operator==(const iterator& rhs) const
3288  {
3289  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3290  return m_pItem == rhs.m_pItem;
3291  }
3292  bool operator!=(const iterator& rhs) const
3293  {
3294  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3295  return m_pItem != rhs.m_pItem;
3296  }
3297 
3298  private:
3299  VmaRawList<T>* m_pList;
3300  VmaListItem<T>* m_pItem;
3301 
3302  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3303  m_pList(pList),
3304  m_pItem(pItem)
3305  {
3306  }
3307 
3308  friend class VmaList<T, AllocatorT>;
3309  };
3310 
3311  class const_iterator
3312  {
3313  public:
3314  const_iterator() :
3315  m_pList(VMA_NULL),
3316  m_pItem(VMA_NULL)
3317  {
3318  }
3319 
3320  const_iterator(const iterator& src) :
3321  m_pList(src.m_pList),
3322  m_pItem(src.m_pItem)
3323  {
3324  }
3325 
3326  const T& operator*() const
3327  {
3328  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3329  return m_pItem->Value;
3330  }
3331  const T* operator->() const
3332  {
3333  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3334  return &m_pItem->Value;
3335  }
3336 
3337  const_iterator& operator++()
3338  {
3339  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3340  m_pItem = m_pItem->pNext;
3341  return *this;
3342  }
3343  const_iterator& operator--()
3344  {
3345  if(m_pItem != VMA_NULL)
3346  {
3347  m_pItem = m_pItem->pPrev;
3348  }
3349  else
3350  {
3351  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3352  m_pItem = m_pList->Back();
3353  }
3354  return *this;
3355  }
3356 
3357  const_iterator operator++(int)
3358  {
3359  const_iterator result = *this;
3360  ++*this;
3361  return result;
3362  }
3363  const_iterator operator--(int)
3364  {
3365  const_iterator result = *this;
3366  --*this;
3367  return result;
3368  }
3369 
3370  bool operator==(const const_iterator& rhs) const
3371  {
3372  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3373  return m_pItem == rhs.m_pItem;
3374  }
3375  bool operator!=(const const_iterator& rhs) const
3376  {
3377  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3378  return m_pItem != rhs.m_pItem;
3379  }
3380 
3381  private:
3382  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
3383  m_pList(pList),
3384  m_pItem(pItem)
3385  {
3386  }
3387 
3388  const VmaRawList<T>* m_pList;
3389  const VmaListItem<T>* m_pItem;
3390 
3391  friend class VmaList<T, AllocatorT>;
3392  };
3393 
3394  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
3395 
3396  bool empty() const { return m_RawList.IsEmpty(); }
3397  size_t size() const { return m_RawList.GetCount(); }
3398 
3399  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
3400  iterator end() { return iterator(&m_RawList, VMA_NULL); }
3401 
3402  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
3403  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
3404 
3405  void clear() { m_RawList.Clear(); }
3406  void push_back(const T& value) { m_RawList.PushBack(value); }
3407  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
3408  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
3409 
3410 private:
3411  VmaRawList<T> m_RawList;
3412 };
3413 
3414 #endif // #if VMA_USE_STL_LIST
3415 
3417 // class VmaMap
3418 
3419 // Unused in this version.
3420 #if 0
3421 
3422 #if VMA_USE_STL_UNORDERED_MAP
3423 
3424 #define VmaPair std::pair
3425 
3426 #define VMA_MAP_TYPE(KeyT, ValueT) \
3427  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
3428 
3429 #else // #if VMA_USE_STL_UNORDERED_MAP
3430 
3431 template<typename T1, typename T2>
3432 struct VmaPair
3433 {
3434  T1 first;
3435  T2 second;
3436 
3437  VmaPair() : first(), second() { }
3438  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
3439 };
3440 
3441 /* Class compatible with subset of interface of std::unordered_map.
3442 KeyT, ValueT must be POD because they will be stored in VmaVector.
3443 */
3444 template<typename KeyT, typename ValueT>
3445 class VmaMap
3446 {
3447 public:
3448  typedef VmaPair<KeyT, ValueT> PairType;
3449  typedef PairType* iterator;
3450 
3451  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
3452 
3453  iterator begin() { return m_Vector.begin(); }
3454  iterator end() { return m_Vector.end(); }
3455 
3456  void insert(const PairType& pair);
3457  iterator find(const KeyT& key);
3458  void erase(iterator it);
3459 
3460 private:
3461  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
3462 };
3463 
3464 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
3465 
3466 template<typename FirstT, typename SecondT>
3467 struct VmaPairFirstLess
3468 {
3469  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
3470  {
3471  return lhs.first < rhs.first;
3472  }
3473  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
3474  {
3475  return lhs.first < rhsFirst;
3476  }
3477 };
3478 
3479 template<typename KeyT, typename ValueT>
3480 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
3481 {
3482  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3483  m_Vector.data(),
3484  m_Vector.data() + m_Vector.size(),
3485  pair,
3486  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
3487  VmaVectorInsert(m_Vector, indexToInsert, pair);
3488 }
3489 
3490 template<typename KeyT, typename ValueT>
3491 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
3492 {
3493  PairType* it = VmaBinaryFindFirstNotLess(
3494  m_Vector.data(),
3495  m_Vector.data() + m_Vector.size(),
3496  key,
3497  VmaPairFirstLess<KeyT, ValueT>());
3498  if((it != m_Vector.end()) && (it->first == key))
3499  {
3500  return it;
3501  }
3502  else
3503  {
3504  return m_Vector.end();
3505  }
3506 }
3507 
3508 template<typename KeyT, typename ValueT>
3509 void VmaMap<KeyT, ValueT>::erase(iterator it)
3510 {
3511  VmaVectorRemove(m_Vector, it - m_Vector.begin());
3512 }
3513 
3514 #endif // #if VMA_USE_STL_UNORDERED_MAP
3515 
3516 #endif // #if 0
3517 
3519 
3520 class VmaDeviceMemoryBlock;
3521 
3522 struct VmaAllocation_T
3523 {
3524 private:
3525  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
3526 
3527  enum FLAGS
3528  {
3529  FLAG_USER_DATA_STRING = 0x01,
3530  };
3531 
3532 public:
3533  enum ALLOCATION_TYPE
3534  {
3535  ALLOCATION_TYPE_NONE,
3536  ALLOCATION_TYPE_BLOCK,
3537  ALLOCATION_TYPE_DEDICATED,
3538  };
3539 
3540  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
3541  m_Alignment(1),
3542  m_Size(0),
3543  m_pUserData(VMA_NULL),
3544  m_LastUseFrameIndex(currentFrameIndex),
3545  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
3546  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
3547  m_MapCount(0),
3548  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
3549  {
3550  }
3551 
3552  ~VmaAllocation_T()
3553  {
3554  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
3555 
3556  // Check if owned string was freed.
3557  VMA_ASSERT(m_pUserData == VMA_NULL);
3558  }
3559 
3560  void InitBlockAllocation(
3561  VmaPool hPool,
3562  VmaDeviceMemoryBlock* block,
3563  VkDeviceSize offset,
3564  VkDeviceSize alignment,
3565  VkDeviceSize size,
3566  VmaSuballocationType suballocationType,
3567  bool mapped,
3568  bool canBecomeLost)
3569  {
3570  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3571  VMA_ASSERT(block != VMA_NULL);
3572  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3573  m_Alignment = alignment;
3574  m_Size = size;
3575  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3576  m_SuballocationType = (uint8_t)suballocationType;
3577  m_BlockAllocation.m_hPool = hPool;
3578  m_BlockAllocation.m_Block = block;
3579  m_BlockAllocation.m_Offset = offset;
3580  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
3581  }
3582 
3583  void InitLost()
3584  {
3585  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3586  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
3587  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3588  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
3589  m_BlockAllocation.m_Block = VMA_NULL;
3590  m_BlockAllocation.m_Offset = 0;
3591  m_BlockAllocation.m_CanBecomeLost = true;
3592  }
3593 
3594  void ChangeBlockAllocation(
3595  VmaAllocator hAllocator,
3596  VmaDeviceMemoryBlock* block,
3597  VkDeviceSize offset);
3598 
3599  // pMappedData not null means allocation is created with MAPPED flag.
3600  void InitDedicatedAllocation(
3601  uint32_t memoryTypeIndex,
3602  VkDeviceMemory hMemory,
3603  VmaSuballocationType suballocationType,
3604  void* pMappedData,
3605  VkDeviceSize size)
3606  {
3607  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3608  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
3609  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
3610  m_Alignment = 0;
3611  m_Size = size;
3612  m_SuballocationType = (uint8_t)suballocationType;
3613  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3614  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
3615  m_DedicatedAllocation.m_hMemory = hMemory;
3616  m_DedicatedAllocation.m_pMappedData = pMappedData;
3617  }
3618 
3619  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
3620  VkDeviceSize GetAlignment() const { return m_Alignment; }
3621  VkDeviceSize GetSize() const { return m_Size; }
3622  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
3623  void* GetUserData() const { return m_pUserData; }
3624  void SetUserData(VmaAllocator hAllocator, void* pUserData);
3625  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
3626 
3627  VmaDeviceMemoryBlock* GetBlock() const
3628  {
3629  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
3630  return m_BlockAllocation.m_Block;
3631  }
3632  VkDeviceSize GetOffset() const;
3633  VkDeviceMemory GetMemory() const;
3634  uint32_t GetMemoryTypeIndex() const;
3635  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
3636  void* GetMappedData() const;
3637  bool CanBecomeLost() const;
3638  VmaPool GetPool() const;
3639 
3640  uint32_t GetLastUseFrameIndex() const
3641  {
3642  return m_LastUseFrameIndex.load();
3643  }
3644  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
3645  {
3646  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
3647  }
3648  /*
3649  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
3650  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
3651  - Else, returns false.
3652 
3653  If hAllocation is already lost, assert - you should not call it then.
3654  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
3655  */
3656  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3657 
3658  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
3659  {
3660  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
3661  outInfo.blockCount = 1;
3662  outInfo.allocationCount = 1;
3663  outInfo.unusedRangeCount = 0;
3664  outInfo.usedBytes = m_Size;
3665  outInfo.unusedBytes = 0;
3666  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
3667  outInfo.unusedRangeSizeMin = UINT64_MAX;
3668  outInfo.unusedRangeSizeMax = 0;
3669  }
3670 
3671  void BlockAllocMap();
3672  void BlockAllocUnmap();
3673  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
3674  void DedicatedAllocUnmap(VmaAllocator hAllocator);
3675 
3676 private:
3677  VkDeviceSize m_Alignment;
3678  VkDeviceSize m_Size;
3679  void* m_pUserData;
3680  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
3681  uint8_t m_Type; // ALLOCATION_TYPE
3682  uint8_t m_SuballocationType; // VmaSuballocationType
3683  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
3684  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
3685  uint8_t m_MapCount;
3686  uint8_t m_Flags; // enum FLAGS
3687 
3688  // Allocation out of VmaDeviceMemoryBlock.
3689  struct BlockAllocation
3690  {
3691  VmaPool m_hPool; // Null if belongs to general memory.
3692  VmaDeviceMemoryBlock* m_Block;
3693  VkDeviceSize m_Offset;
3694  bool m_CanBecomeLost;
3695  };
3696 
3697  // Allocation for an object that has its own private VkDeviceMemory.
3698  struct DedicatedAllocation
3699  {
3700  uint32_t m_MemoryTypeIndex;
3701  VkDeviceMemory m_hMemory;
3702  void* m_pMappedData; // Not null means memory is mapped.
3703  };
3704 
3705  union
3706  {
3707  // Allocation out of VmaDeviceMemoryBlock.
3708  BlockAllocation m_BlockAllocation;
3709  // Allocation for an object that has its own private VkDeviceMemory.
3710  DedicatedAllocation m_DedicatedAllocation;
3711  };
3712 
3713  void FreeUserDataString(VmaAllocator hAllocator);
3714 };
3715 
3716 /*
3717 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
3718 allocated memory block or free.
3719 */
3720 struct VmaSuballocation
3721 {
3722  VkDeviceSize offset;
3723  VkDeviceSize size;
3724  VmaAllocation hAllocation;
3725  VmaSuballocationType type;
3726 };
3727 
3728 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
3729 
3730 // Cost of one additional allocation lost, as equivalent in bytes.
3731 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
3732 
3733 /*
3734 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3735 
3736 If canMakeOtherLost was false:
3737 - item points to a FREE suballocation.
3738 - itemsToMakeLostCount is 0.
3739 
3740 If canMakeOtherLost was true:
3741 - item points to first of sequence of suballocations, which are either FREE,
3742  or point to VmaAllocations that can become lost.
3743 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3744  the requested allocation to succeed.
3745 */
3746 struct VmaAllocationRequest
3747 {
3748  VkDeviceSize offset;
3749  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3750  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3751  VmaSuballocationList::iterator item;
3752  size_t itemsToMakeLostCount;
3753 
3754  VkDeviceSize CalcCost() const
3755  {
3756  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3757  }
3758 };
3759 
3760 /*
3761 Data structure used for bookkeeping of allocations and unused ranges of memory
3762 in a single VkDeviceMemory block.
3763 */
3764 class VmaBlockMetadata
3765 {
3766 public:
3767  VmaBlockMetadata(VmaAllocator hAllocator);
3768  ~VmaBlockMetadata();
3769  void Init(VkDeviceSize size);
3770 
3771  // Validates all data structures inside this object. If not valid, returns false.
3772  bool Validate() const;
3773  VkDeviceSize GetSize() const { return m_Size; }
3774  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
3775  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3776  VkDeviceSize GetUnusedRangeSizeMax() const;
3777  // Returns true if this block is empty - contains only single free suballocation.
3778  bool IsEmpty() const;
3779 
3780  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3781  void AddPoolStats(VmaPoolStats& inoutStats) const;
3782 
3783 #if VMA_STATS_STRING_ENABLED
3784  void PrintDetailedMap(class VmaJsonWriter& json) const;
3785 #endif
3786 
3787  // Creates trivial request for case when block is empty.
3788  void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3789 
3790  // Tries to find a place for suballocation with given parameters inside this block.
3791  // If succeeded, fills pAllocationRequest and returns true.
3792  // If failed, returns false.
3793  bool CreateAllocationRequest(
3794  uint32_t currentFrameIndex,
3795  uint32_t frameInUseCount,
3796  VkDeviceSize bufferImageGranularity,
3797  VkDeviceSize allocSize,
3798  VkDeviceSize allocAlignment,
3799  VmaSuballocationType allocType,
3800  bool canMakeOtherLost,
3801  VmaAllocationRequest* pAllocationRequest);
3802 
3803  bool MakeRequestedAllocationsLost(
3804  uint32_t currentFrameIndex,
3805  uint32_t frameInUseCount,
3806  VmaAllocationRequest* pAllocationRequest);
3807 
3808  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3809 
3810  // Makes actual allocation based on request. Request must already be checked and valid.
3811  void Alloc(
3812  const VmaAllocationRequest& request,
3813  VmaSuballocationType type,
3814  VkDeviceSize allocSize,
3815  VmaAllocation hAllocation);
3816 
3817  // Frees suballocation assigned to given memory region.
3818  void Free(const VmaAllocation allocation);
3819  void FreeAtOffset(VkDeviceSize offset);
3820 
3821 private:
3822  VkDeviceSize m_Size;
3823  uint32_t m_FreeCount;
3824  VkDeviceSize m_SumFreeSize;
3825  VmaSuballocationList m_Suballocations;
3826  // Suballocations that are free and have size greater than certain threshold.
3827  // Sorted by size, ascending.
3828  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
3829 
3830  bool ValidateFreeSuballocationList() const;
3831 
3832  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
3833  // If yes, fills pOffset and returns true. If no, returns false.
3834  bool CheckAllocation(
3835  uint32_t currentFrameIndex,
3836  uint32_t frameInUseCount,
3837  VkDeviceSize bufferImageGranularity,
3838  VkDeviceSize allocSize,
3839  VkDeviceSize allocAlignment,
3840  VmaSuballocationType allocType,
3841  VmaSuballocationList::const_iterator suballocItem,
3842  bool canMakeOtherLost,
3843  VkDeviceSize* pOffset,
3844  size_t* itemsToMakeLostCount,
3845  VkDeviceSize* pSumFreeSize,
3846  VkDeviceSize* pSumItemSize) const;
3847  // Given free suballocation, it merges it with following one, which must also be free.
3848  void MergeFreeWithNext(VmaSuballocationList::iterator item);
3849  // Releases given suballocation, making it free.
3850  // Merges it with adjacent free suballocations if applicable.
3851  // Returns iterator to new free suballocation at this place.
3852  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
3853  // Given free suballocation, it inserts it into sorted list of
3854  // m_FreeSuballocationsBySize if it's suitable.
3855  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
3856  // Given free suballocation, it removes it from sorted list of
3857  // m_FreeSuballocationsBySize if it's suitable.
3858  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
3859 };
3860 
3861 // Helper class that represents mapped memory. Synchronized internally.
3862 class VmaDeviceMemoryMapping
3863 {
3864 public:
3865  VmaDeviceMemoryMapping();
3866  ~VmaDeviceMemoryMapping();
3867 
3868  void* GetMappedData() const { return m_pMappedData; }
3869 
3870  // ppData can be null.
3871  VkResult Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData);
3872  void Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count);
3873 
3874 private:
3875  VMA_MUTEX m_Mutex;
3876  uint32_t m_MapCount;
3877  void* m_pMappedData;
3878 };
3879 
3880 /*
3881 Represents a single block of device memory (`VkDeviceMemory`) with all the
3882 data about its regions (aka suballocations, `VmaAllocation`), assigned and free.
3883 
3884 Thread-safety: This class must be externally synchronized.
3885 */
3886 class VmaDeviceMemoryBlock
3887 {
3888 public:
3889  uint32_t m_MemoryTypeIndex;
3890  VkDeviceMemory m_hMemory;
3891  VmaDeviceMemoryMapping m_Mapping;
3892  VmaBlockMetadata m_Metadata;
3893 
3894  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
3895 
3896  ~VmaDeviceMemoryBlock()
3897  {
3898  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
3899  }
3900 
3901  // Always call after construction.
3902  void Init(
3903  uint32_t newMemoryTypeIndex,
3904  VkDeviceMemory newMemory,
3905  VkDeviceSize newSize);
3906  // Always call before destruction.
3907  void Destroy(VmaAllocator allocator);
3908 
3909  // Validates all data structures inside this object. If not valid, returns false.
3910  bool Validate() const;
3911 
3912  // ppData can be null.
3913  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
3914  void Unmap(VmaAllocator hAllocator, uint32_t count);
3915 };
3916 
3917 struct VmaPointerLess
3918 {
3919  bool operator()(const void* lhs, const void* rhs) const
3920  {
3921  return lhs < rhs;
3922  }
3923 };
3924 
3925 class VmaDefragmentator;
3926 
3927 /*
3928 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
3929 Vulkan memory type.
3930 
3931 Synchronized internally with a mutex.
3932 */
3933 struct VmaBlockVector
3934 {
3935  VmaBlockVector(
3936  VmaAllocator hAllocator,
3937  uint32_t memoryTypeIndex,
3938  VkDeviceSize preferredBlockSize,
3939  size_t minBlockCount,
3940  size_t maxBlockCount,
3941  VkDeviceSize bufferImageGranularity,
3942  uint32_t frameInUseCount,
3943  bool isCustomPool);
3944  ~VmaBlockVector();
3945 
3946  VkResult CreateMinBlocks();
3947 
3948  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
3949  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
3950  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
3951  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
3952 
3953  void GetPoolStats(VmaPoolStats* pStats);
3954 
3955  bool IsEmpty() const { return m_Blocks.empty(); }
3956 
3957  VkResult Allocate(
3958  VmaPool hCurrentPool,
3959  uint32_t currentFrameIndex,
3960  const VkMemoryRequirements& vkMemReq,
3961  const VmaAllocationCreateInfo& createInfo,
3962  VmaSuballocationType suballocType,
3963  VmaAllocation* pAllocation);
3964 
3965  void Free(
3966  VmaAllocation hAllocation);
3967 
3968  // Adds statistics of this BlockVector to pStats.
3969  void AddStats(VmaStats* pStats);
3970 
3971 #if VMA_STATS_STRING_ENABLED
3972  void PrintDetailedMap(class VmaJsonWriter& json);
3973 #endif
3974 
3975  void MakePoolAllocationsLost(
3976  uint32_t currentFrameIndex,
3977  size_t* pLostAllocationCount);
3978 
3979  VmaDefragmentator* EnsureDefragmentator(
3980  VmaAllocator hAllocator,
3981  uint32_t currentFrameIndex);
3982 
3983  VkResult Defragment(
3984  VmaDefragmentationStats* pDefragmentationStats,
3985  VkDeviceSize& maxBytesToMove,
3986  uint32_t& maxAllocationsToMove);
3987 
3988  void DestroyDefragmentator();
3989 
3990 private:
3991  friend class VmaDefragmentator;
3992 
3993  const VmaAllocator m_hAllocator;
3994  const uint32_t m_MemoryTypeIndex;
3995  const VkDeviceSize m_PreferredBlockSize;
3996  const size_t m_MinBlockCount;
3997  const size_t m_MaxBlockCount;
3998  const VkDeviceSize m_BufferImageGranularity;
3999  const uint32_t m_FrameInUseCount;
4000  const bool m_IsCustomPool;
4001  VMA_MUTEX m_Mutex;
4002  // Incrementally sorted by sumFreeSize, ascending.
4003  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
4004  /* There can be at most one allocation that is completely empty - a
4005  hysteresis to avoid pessimistic case of alternating creation and destruction
4006  of a VkDeviceMemory. */
4007  bool m_HasEmptyBlock;
4008  VmaDefragmentator* m_pDefragmentator;
4009 
4010  size_t CalcMaxBlockSize() const;
4011 
4012  // Finds and removes given block from vector.
4013  void Remove(VmaDeviceMemoryBlock* pBlock);
4014 
4015  // Performs single step in sorting m_Blocks. They may not be fully sorted
4016  // after this call.
4017  void IncrementallySortBlocks();
4018 
4019  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
4020 };
4021 
4022 struct VmaPool_T
4023 {
4024 public:
4025  VmaBlockVector m_BlockVector;
4026 
4027  // Takes ownership.
4028  VmaPool_T(
4029  VmaAllocator hAllocator,
4030  const VmaPoolCreateInfo& createInfo);
4031  ~VmaPool_T();
4032 
4033  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
4034 
4035 #if VMA_STATS_STRING_ENABLED
4036  //void PrintDetailedMap(class VmaStringBuilder& sb);
4037 #endif
4038 };
4039 
4040 class VmaDefragmentator
4041 {
4042  const VmaAllocator m_hAllocator;
4043  VmaBlockVector* const m_pBlockVector;
4044  uint32_t m_CurrentFrameIndex;
4045  VkDeviceSize m_BytesMoved;
4046  uint32_t m_AllocationsMoved;
4047 
4048  struct AllocationInfo
4049  {
4050  VmaAllocation m_hAllocation;
4051  VkBool32* m_pChanged;
4052 
4053  AllocationInfo() :
4054  m_hAllocation(VK_NULL_HANDLE),
4055  m_pChanged(VMA_NULL)
4056  {
4057  }
4058  };
4059 
4060  struct AllocationInfoSizeGreater
4061  {
4062  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
4063  {
4064  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
4065  }
4066  };
4067 
4068  // Used between AddAllocation and Defragment.
4069  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
4070 
4071  struct BlockInfo
4072  {
4073  VmaDeviceMemoryBlock* m_pBlock;
4074  bool m_HasNonMovableAllocations;
4075  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
4076 
4077  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
4078  m_pBlock(VMA_NULL),
4079  m_HasNonMovableAllocations(true),
4080  m_Allocations(pAllocationCallbacks),
4081  m_pMappedDataForDefragmentation(VMA_NULL)
4082  {
4083  }
4084 
4085  void CalcHasNonMovableAllocations()
4086  {
4087  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
4088  const size_t defragmentAllocCount = m_Allocations.size();
4089  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
4090  }
4091 
4092  void SortAllocationsBySizeDescecnding()
4093  {
4094  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
4095  }
4096 
4097  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
4098  void Unmap(VmaAllocator hAllocator);
4099 
4100  private:
4101  // Not null if mapped for defragmentation only, not originally mapped.
4102  void* m_pMappedDataForDefragmentation;
4103  };
4104 
4105  struct BlockPointerLess
4106  {
4107  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
4108  {
4109  return pLhsBlockInfo->m_pBlock < pRhsBlock;
4110  }
4111  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
4112  {
4113  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
4114  }
4115  };
4116 
4117  // 1. Blocks with some non-movable allocations go first.
4118  // 2. Blocks with smaller sumFreeSize go first.
4119  struct BlockInfoCompareMoveDestination
4120  {
4121  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
4122  {
4123  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
4124  {
4125  return true;
4126  }
4127  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
4128  {
4129  return false;
4130  }
4131  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
4132  {
4133  return true;
4134  }
4135  return false;
4136  }
4137  };
4138 
4139  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
4140  BlockInfoVector m_Blocks;
4141 
4142  VkResult DefragmentRound(
4143  VkDeviceSize maxBytesToMove,
4144  uint32_t maxAllocationsToMove);
4145 
4146  static bool MoveMakesSense(
4147  size_t dstBlockIndex, VkDeviceSize dstOffset,
4148  size_t srcBlockIndex, VkDeviceSize srcOffset);
4149 
4150 public:
4151  VmaDefragmentator(
4152  VmaAllocator hAllocator,
4153  VmaBlockVector* pBlockVector,
4154  uint32_t currentFrameIndex);
4155 
4156  ~VmaDefragmentator();
4157 
4158  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
4159  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
4160 
4161  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
4162 
4163  VkResult Defragment(
4164  VkDeviceSize maxBytesToMove,
4165  uint32_t maxAllocationsToMove);
4166 };
4167 
4168 // Main allocator object.
4169 struct VmaAllocator_T
4170 {
4171  bool m_UseMutex;
4172  bool m_UseKhrDedicatedAllocation;
4173  VkDevice m_hDevice;
4174  bool m_AllocationCallbacksSpecified;
4175  VkAllocationCallbacks m_AllocationCallbacks;
4176  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
4177 
4178  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
4179  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
4180  VMA_MUTEX m_HeapSizeLimitMutex;
4181 
4182  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
4183  VkPhysicalDeviceMemoryProperties m_MemProps;
4184 
4185  // Default pools.
4186  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
4187 
4188  // Each vector is sorted by memory (handle value).
4189  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
4190  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
4191  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
4192 
4193  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
4194  ~VmaAllocator_T();
4195 
4196  const VkAllocationCallbacks* GetAllocationCallbacks() const
4197  {
4198  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
4199  }
4200  const VmaVulkanFunctions& GetVulkanFunctions() const
4201  {
4202  return m_VulkanFunctions;
4203  }
4204 
4205  VkDeviceSize GetBufferImageGranularity() const
4206  {
4207  return VMA_MAX(
4208  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
4209  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
4210  }
4211 
4212  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
4213  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
4214 
4215  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
4216  {
4217  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
4218  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
4219  }
4220 
4221  void GetBufferMemoryRequirements(
4222  VkBuffer hBuffer,
4223  VkMemoryRequirements& memReq,
4224  bool& requiresDedicatedAllocation,
4225  bool& prefersDedicatedAllocation) const;
4226  void GetImageMemoryRequirements(
4227  VkImage hImage,
4228  VkMemoryRequirements& memReq,
4229  bool& requiresDedicatedAllocation,
4230  bool& prefersDedicatedAllocation) const;
4231 
4232  // Main allocation function.
4233  VkResult AllocateMemory(
4234  const VkMemoryRequirements& vkMemReq,
4235  bool requiresDedicatedAllocation,
4236  bool prefersDedicatedAllocation,
4237  VkBuffer dedicatedBuffer,
4238  VkImage dedicatedImage,
4239  const VmaAllocationCreateInfo& createInfo,
4240  VmaSuballocationType suballocType,
4241  VmaAllocation* pAllocation);
4242 
4243  // Main deallocation function.
4244  void FreeMemory(const VmaAllocation allocation);
4245 
4246  void CalculateStats(VmaStats* pStats);
4247 
4248 #if VMA_STATS_STRING_ENABLED
4249  void PrintDetailedMap(class VmaJsonWriter& json);
4250 #endif
4251 
4252  VkResult Defragment(
4253  VmaAllocation* pAllocations,
4254  size_t allocationCount,
4255  VkBool32* pAllocationsChanged,
4256  const VmaDefragmentationInfo* pDefragmentationInfo,
4257  VmaDefragmentationStats* pDefragmentationStats);
4258 
4259  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
4260  bool TouchAllocation(VmaAllocation hAllocation);
4261 
4262  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
4263  void DestroyPool(VmaPool pool);
4264  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
4265 
4266  void SetCurrentFrameIndex(uint32_t frameIndex);
4267 
4268  void MakePoolAllocationsLost(
4269  VmaPool hPool,
4270  size_t* pLostAllocationCount);
4271 
4272  void CreateLostAllocation(VmaAllocation* pAllocation);
4273 
4274  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
4275  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
4276 
4277  VkResult Map(VmaAllocation hAllocation, void** ppData);
4278  void Unmap(VmaAllocation hAllocation);
4279 
4280 private:
4281  VkDeviceSize m_PreferredLargeHeapBlockSize;
4282 
4283  VkPhysicalDevice m_PhysicalDevice;
4284  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
4285 
4286  VMA_MUTEX m_PoolsMutex;
4287  // Protected by m_PoolsMutex. Sorted by pointer value.
4288  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
4289 
4290  VmaVulkanFunctions m_VulkanFunctions;
4291 
4292  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
4293 
4294  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
4295 
4296  VkResult AllocateMemoryOfType(
4297  const VkMemoryRequirements& vkMemReq,
4298  bool dedicatedAllocation,
4299  VkBuffer dedicatedBuffer,
4300  VkImage dedicatedImage,
4301  const VmaAllocationCreateInfo& createInfo,
4302  uint32_t memTypeIndex,
4303  VmaSuballocationType suballocType,
4304  VmaAllocation* pAllocation);
4305 
4306  // Allocates and registers new VkDeviceMemory specifically for single allocation.
4307  VkResult AllocateDedicatedMemory(
4308  VkDeviceSize size,
4309  VmaSuballocationType suballocType,
4310  uint32_t memTypeIndex,
4311  bool map,
4312  bool isUserDataString,
4313  void* pUserData,
4314  VkBuffer dedicatedBuffer,
4315  VkImage dedicatedImage,
4316  VmaAllocation* pAllocation);
4317 
4318  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
4319  void FreeDedicatedMemory(VmaAllocation allocation);
4320 };
4321 
4323 // Memory allocation #2 after VmaAllocator_T definition
4324 
4325 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
4326 {
4327  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
4328 }
4329 
4330 static void VmaFree(VmaAllocator hAllocator, void* ptr)
4331 {
4332  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
4333 }
4334 
4335 template<typename T>
4336 static T* VmaAllocate(VmaAllocator hAllocator)
4337 {
4338  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
4339 }
4340 
4341 template<typename T>
4342 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
4343 {
4344  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
4345 }
4346 
4347 template<typename T>
4348 static void vma_delete(VmaAllocator hAllocator, T* ptr)
4349 {
4350  if(ptr != VMA_NULL)
4351  {
4352  ptr->~T();
4353  VmaFree(hAllocator, ptr);
4354  }
4355 }
4356 
4357 template<typename T>
4358 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
4359 {
4360  if(ptr != VMA_NULL)
4361  {
4362  for(size_t i = count; i--; )
4363  ptr[i].~T();
4364  VmaFree(hAllocator, ptr);
4365  }
4366 }
4367 
4369 // VmaStringBuilder
4370 
4371 #if VMA_STATS_STRING_ENABLED
4372 
4373 class VmaStringBuilder
4374 {
4375 public:
4376  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
4377  size_t GetLength() const { return m_Data.size(); }
4378  const char* GetData() const { return m_Data.data(); }
4379 
4380  void Add(char ch) { m_Data.push_back(ch); }
4381  void Add(const char* pStr);
4382  void AddNewLine() { Add('\n'); }
4383  void AddNumber(uint32_t num);
4384  void AddNumber(uint64_t num);
4385  void AddPointer(const void* ptr);
4386 
4387 private:
4388  VmaVector< char, VmaStlAllocator<char> > m_Data;
4389 };
4390 
4391 void VmaStringBuilder::Add(const char* pStr)
4392 {
4393  const size_t strLen = strlen(pStr);
4394  if(strLen > 0)
4395  {
4396  const size_t oldCount = m_Data.size();
4397  m_Data.resize(oldCount + strLen);
4398  memcpy(m_Data.data() + oldCount, pStr, strLen);
4399  }
4400 }
4401 
4402 void VmaStringBuilder::AddNumber(uint32_t num)
4403 {
4404  char buf[11];
4405  VmaUint32ToStr(buf, sizeof(buf), num);
4406  Add(buf);
4407 }
4408 
4409 void VmaStringBuilder::AddNumber(uint64_t num)
4410 {
4411  char buf[21];
4412  VmaUint64ToStr(buf, sizeof(buf), num);
4413  Add(buf);
4414 }
4415 
4416 void VmaStringBuilder::AddPointer(const void* ptr)
4417 {
4418  char buf[21];
4419  VmaPtrToStr(buf, sizeof(buf), ptr);
4420  Add(buf);
4421 }
4422 
4423 #endif // #if VMA_STATS_STRING_ENABLED
4424 
4426 // VmaJsonWriter
4427 
4428 #if VMA_STATS_STRING_ENABLED
4429 
4430 class VmaJsonWriter
4431 {
4432 public:
4433  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
4434  ~VmaJsonWriter();
4435 
4436  void BeginObject(bool singleLine = false);
4437  void EndObject();
4438 
4439  void BeginArray(bool singleLine = false);
4440  void EndArray();
4441 
4442  void WriteString(const char* pStr);
4443  void BeginString(const char* pStr = VMA_NULL);
4444  void ContinueString(const char* pStr);
4445  void ContinueString(uint32_t n);
4446  void ContinueString(uint64_t n);
4447  void ContinueString_Pointer(const void* ptr);
4448  void EndString(const char* pStr = VMA_NULL);
4449 
4450  void WriteNumber(uint32_t n);
4451  void WriteNumber(uint64_t n);
4452  void WriteBool(bool b);
4453  void WriteNull();
4454 
4455 private:
4456  static const char* const INDENT;
4457 
4458  enum COLLECTION_TYPE
4459  {
4460  COLLECTION_TYPE_OBJECT,
4461  COLLECTION_TYPE_ARRAY,
4462  };
4463  struct StackItem
4464  {
4465  COLLECTION_TYPE type;
4466  uint32_t valueCount;
4467  bool singleLineMode;
4468  };
4469 
4470  VmaStringBuilder& m_SB;
4471  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
4472  bool m_InsideString;
4473 
4474  void BeginValue(bool isString);
4475  void WriteIndent(bool oneLess = false);
4476 };
4477 
4478 const char* const VmaJsonWriter::INDENT = " ";
4479 
4480 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
4481  m_SB(sb),
4482  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
4483  m_InsideString(false)
4484 {
4485 }
4486 
4487 VmaJsonWriter::~VmaJsonWriter()
4488 {
4489  VMA_ASSERT(!m_InsideString);
4490  VMA_ASSERT(m_Stack.empty());
4491 }
4492 
4493 void VmaJsonWriter::BeginObject(bool singleLine)
4494 {
4495  VMA_ASSERT(!m_InsideString);
4496 
4497  BeginValue(false);
4498  m_SB.Add('{');
4499 
4500  StackItem item;
4501  item.type = COLLECTION_TYPE_OBJECT;
4502  item.valueCount = 0;
4503  item.singleLineMode = singleLine;
4504  m_Stack.push_back(item);
4505 }
4506 
4507 void VmaJsonWriter::EndObject()
4508 {
4509  VMA_ASSERT(!m_InsideString);
4510 
4511  WriteIndent(true);
4512  m_SB.Add('}');
4513 
4514  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
4515  m_Stack.pop_back();
4516 }
4517 
4518 void VmaJsonWriter::BeginArray(bool singleLine)
4519 {
4520  VMA_ASSERT(!m_InsideString);
4521 
4522  BeginValue(false);
4523  m_SB.Add('[');
4524 
4525  StackItem item;
4526  item.type = COLLECTION_TYPE_ARRAY;
4527  item.valueCount = 0;
4528  item.singleLineMode = singleLine;
4529  m_Stack.push_back(item);
4530 }
4531 
4532 void VmaJsonWriter::EndArray()
4533 {
4534  VMA_ASSERT(!m_InsideString);
4535 
4536  WriteIndent(true);
4537  m_SB.Add(']');
4538 
4539  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
4540  m_Stack.pop_back();
4541 }
4542 
4543 void VmaJsonWriter::WriteString(const char* pStr)
4544 {
4545  BeginString(pStr);
4546  EndString();
4547 }
4548 
4549 void VmaJsonWriter::BeginString(const char* pStr)
4550 {
4551  VMA_ASSERT(!m_InsideString);
4552 
4553  BeginValue(true);
4554  m_SB.Add('"');
4555  m_InsideString = true;
4556  if(pStr != VMA_NULL && pStr[0] != '\0')
4557  {
4558  ContinueString(pStr);
4559  }
4560 }
4561 
4562 void VmaJsonWriter::ContinueString(const char* pStr)
4563 {
4564  VMA_ASSERT(m_InsideString);
4565 
4566  const size_t strLen = strlen(pStr);
4567  for(size_t i = 0; i < strLen; ++i)
4568  {
4569  char ch = pStr[i];
4570  if(ch == '\'')
4571  {
4572  m_SB.Add("\\\\");
4573  }
4574  else if(ch == '"')
4575  {
4576  m_SB.Add("\\\"");
4577  }
4578  else if(ch >= 32)
4579  {
4580  m_SB.Add(ch);
4581  }
4582  else switch(ch)
4583  {
4584  case '\b':
4585  m_SB.Add("\\b");
4586  break;
4587  case '\f':
4588  m_SB.Add("\\f");
4589  break;
4590  case '\n':
4591  m_SB.Add("\\n");
4592  break;
4593  case '\r':
4594  m_SB.Add("\\r");
4595  break;
4596  case '\t':
4597  m_SB.Add("\\t");
4598  break;
4599  default:
4600  VMA_ASSERT(0 && "Character not currently supported.");
4601  break;
4602  }
4603  }
4604 }
4605 
4606 void VmaJsonWriter::ContinueString(uint32_t n)
4607 {
4608  VMA_ASSERT(m_InsideString);
4609  m_SB.AddNumber(n);
4610 }
4611 
4612 void VmaJsonWriter::ContinueString(uint64_t n)
4613 {
4614  VMA_ASSERT(m_InsideString);
4615  m_SB.AddNumber(n);
4616 }
4617 
4618 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
4619 {
4620  VMA_ASSERT(m_InsideString);
4621  m_SB.AddPointer(ptr);
4622 }
4623 
4624 void VmaJsonWriter::EndString(const char* pStr)
4625 {
4626  VMA_ASSERT(m_InsideString);
4627  if(pStr != VMA_NULL && pStr[0] != '\0')
4628  {
4629  ContinueString(pStr);
4630  }
4631  m_SB.Add('"');
4632  m_InsideString = false;
4633 }
4634 
4635 void VmaJsonWriter::WriteNumber(uint32_t n)
4636 {
4637  VMA_ASSERT(!m_InsideString);
4638  BeginValue(false);
4639  m_SB.AddNumber(n);
4640 }
4641 
4642 void VmaJsonWriter::WriteNumber(uint64_t n)
4643 {
4644  VMA_ASSERT(!m_InsideString);
4645  BeginValue(false);
4646  m_SB.AddNumber(n);
4647 }
4648 
4649 void VmaJsonWriter::WriteBool(bool b)
4650 {
4651  VMA_ASSERT(!m_InsideString);
4652  BeginValue(false);
4653  m_SB.Add(b ? "true" : "false");
4654 }
4655 
4656 void VmaJsonWriter::WriteNull()
4657 {
4658  VMA_ASSERT(!m_InsideString);
4659  BeginValue(false);
4660  m_SB.Add("null");
4661 }
4662 
4663 void VmaJsonWriter::BeginValue(bool isString)
4664 {
4665  if(!m_Stack.empty())
4666  {
4667  StackItem& currItem = m_Stack.back();
4668  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4669  currItem.valueCount % 2 == 0)
4670  {
4671  VMA_ASSERT(isString);
4672  }
4673 
4674  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4675  currItem.valueCount % 2 != 0)
4676  {
4677  m_SB.Add(": ");
4678  }
4679  else if(currItem.valueCount > 0)
4680  {
4681  m_SB.Add(", ");
4682  WriteIndent();
4683  }
4684  else
4685  {
4686  WriteIndent();
4687  }
4688  ++currItem.valueCount;
4689  }
4690 }
4691 
4692 void VmaJsonWriter::WriteIndent(bool oneLess)
4693 {
4694  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
4695  {
4696  m_SB.AddNewLine();
4697 
4698  size_t count = m_Stack.size();
4699  if(count > 0 && oneLess)
4700  {
4701  --count;
4702  }
4703  for(size_t i = 0; i < count; ++i)
4704  {
4705  m_SB.Add(INDENT);
4706  }
4707  }
4708 }
4709 
4710 #endif // #if VMA_STATS_STRING_ENABLED
4711 
4713 
4714 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
4715 {
4716  if(IsUserDataString())
4717  {
4718  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
4719 
4720  FreeUserDataString(hAllocator);
4721 
4722  if(pUserData != VMA_NULL)
4723  {
4724  const char* const newStrSrc = (char*)pUserData;
4725  const size_t newStrLen = strlen(newStrSrc);
4726  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
4727  memcpy(newStrDst, newStrSrc, newStrLen + 1);
4728  m_pUserData = newStrDst;
4729  }
4730  }
4731  else
4732  {
4733  m_pUserData = pUserData;
4734  }
4735 }
4736 
4737 void VmaAllocation_T::ChangeBlockAllocation(
4738  VmaAllocator hAllocator,
4739  VmaDeviceMemoryBlock* block,
4740  VkDeviceSize offset)
4741 {
4742  VMA_ASSERT(block != VMA_NULL);
4743  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4744 
4745  // Move mapping reference counter from old block to new block.
4746  if(block != m_BlockAllocation.m_Block)
4747  {
4748  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
4749  if(IsPersistentMap())
4750  ++mapRefCount;
4751  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
4752  block->Map(hAllocator, mapRefCount, VMA_NULL);
4753  }
4754 
4755  m_BlockAllocation.m_Block = block;
4756  m_BlockAllocation.m_Offset = offset;
4757 }
4758 
4759 VkDeviceSize VmaAllocation_T::GetOffset() const
4760 {
4761  switch(m_Type)
4762  {
4763  case ALLOCATION_TYPE_BLOCK:
4764  return m_BlockAllocation.m_Offset;
4765  case ALLOCATION_TYPE_DEDICATED:
4766  return 0;
4767  default:
4768  VMA_ASSERT(0);
4769  return 0;
4770  }
4771 }
4772 
4773 VkDeviceMemory VmaAllocation_T::GetMemory() const
4774 {
4775  switch(m_Type)
4776  {
4777  case ALLOCATION_TYPE_BLOCK:
4778  return m_BlockAllocation.m_Block->m_hMemory;
4779  case ALLOCATION_TYPE_DEDICATED:
4780  return m_DedicatedAllocation.m_hMemory;
4781  default:
4782  VMA_ASSERT(0);
4783  return VK_NULL_HANDLE;
4784  }
4785 }
4786 
4787 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
4788 {
4789  switch(m_Type)
4790  {
4791  case ALLOCATION_TYPE_BLOCK:
4792  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
4793  case ALLOCATION_TYPE_DEDICATED:
4794  return m_DedicatedAllocation.m_MemoryTypeIndex;
4795  default:
4796  VMA_ASSERT(0);
4797  return UINT32_MAX;
4798  }
4799 }
4800 
4801 void* VmaAllocation_T::GetMappedData() const
4802 {
4803  switch(m_Type)
4804  {
4805  case ALLOCATION_TYPE_BLOCK:
4806  if(m_MapCount != 0)
4807  {
4808  void* pBlockData = m_BlockAllocation.m_Block->m_Mapping.GetMappedData();
4809  VMA_ASSERT(pBlockData != VMA_NULL);
4810  return (char*)pBlockData + m_BlockAllocation.m_Offset;
4811  }
4812  else
4813  {
4814  return VMA_NULL;
4815  }
4816  break;
4817  case ALLOCATION_TYPE_DEDICATED:
4818  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
4819  return m_DedicatedAllocation.m_pMappedData;
4820  default:
4821  VMA_ASSERT(0);
4822  return VMA_NULL;
4823  }
4824 }
4825 
4826 bool VmaAllocation_T::CanBecomeLost() const
4827 {
4828  switch(m_Type)
4829  {
4830  case ALLOCATION_TYPE_BLOCK:
4831  return m_BlockAllocation.m_CanBecomeLost;
4832  case ALLOCATION_TYPE_DEDICATED:
4833  return false;
4834  default:
4835  VMA_ASSERT(0);
4836  return false;
4837  }
4838 }
4839 
4840 VmaPool VmaAllocation_T::GetPool() const
4841 {
4842  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4843  return m_BlockAllocation.m_hPool;
4844 }
4845 
4846 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4847 {
4848  VMA_ASSERT(CanBecomeLost());
4849 
4850  /*
4851  Warning: This is a carefully designed algorithm.
4852  Do not modify unless you really know what you're doing :)
4853  */
4854  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
4855  for(;;)
4856  {
4857  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
4858  {
4859  VMA_ASSERT(0);
4860  return false;
4861  }
4862  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
4863  {
4864  return false;
4865  }
4866  else // Last use time earlier than current time.
4867  {
4868  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
4869  {
4870  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
4871  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
4872  return true;
4873  }
4874  }
4875  }
4876 }
4877 
4878 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
4879 {
4880  VMA_ASSERT(IsUserDataString());
4881  if(m_pUserData != VMA_NULL)
4882  {
4883  char* const oldStr = (char*)m_pUserData;
4884  const size_t oldStrLen = strlen(oldStr);
4885  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
4886  m_pUserData = VMA_NULL;
4887  }
4888 }
4889 
4890 void VmaAllocation_T::BlockAllocMap()
4891 {
4892  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4893 
4894  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4895  {
4896  ++m_MapCount;
4897  }
4898  else
4899  {
4900  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
4901  }
4902 }
4903 
4904 void VmaAllocation_T::BlockAllocUnmap()
4905 {
4906  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4907 
4908  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4909  {
4910  --m_MapCount;
4911  }
4912  else
4913  {
4914  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
4915  }
4916 }
4917 
4918 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
4919 {
4920  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4921 
4922  if(m_MapCount != 0)
4923  {
4924  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4925  {
4926  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
4927  *ppData = m_DedicatedAllocation.m_pMappedData;
4928  ++m_MapCount;
4929  return VK_SUCCESS;
4930  }
4931  else
4932  {
4933  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
4934  return VK_ERROR_MEMORY_MAP_FAILED;
4935  }
4936  }
4937  else
4938  {
4939  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
4940  hAllocator->m_hDevice,
4941  m_DedicatedAllocation.m_hMemory,
4942  0, // offset
4943  VK_WHOLE_SIZE,
4944  0, // flags
4945  ppData);
4946  if(result == VK_SUCCESS)
4947  {
4948  m_DedicatedAllocation.m_pMappedData = *ppData;
4949  m_MapCount = 1;
4950  }
4951  return result;
4952  }
4953 }
4954 
4955 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
4956 {
4957  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4958 
4959  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4960  {
4961  --m_MapCount;
4962  if(m_MapCount == 0)
4963  {
4964  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
4965  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
4966  hAllocator->m_hDevice,
4967  m_DedicatedAllocation.m_hMemory);
4968  }
4969  }
4970  else
4971  {
4972  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
4973  }
4974 }
4975 
4976 #if VMA_STATS_STRING_ENABLED
4977 
4978 // Correspond to values of enum VmaSuballocationType.
4979 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
4980  "FREE",
4981  "UNKNOWN",
4982  "BUFFER",
4983  "IMAGE_UNKNOWN",
4984  "IMAGE_LINEAR",
4985  "IMAGE_OPTIMAL",
4986 };
4987 
4988 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
4989 {
4990  json.BeginObject();
4991 
4992  json.WriteString("Blocks");
4993  json.WriteNumber(stat.blockCount);
4994 
4995  json.WriteString("Allocations");
4996  json.WriteNumber(stat.allocationCount);
4997 
4998  json.WriteString("UnusedRanges");
4999  json.WriteNumber(stat.unusedRangeCount);
5000 
5001  json.WriteString("UsedBytes");
5002  json.WriteNumber(stat.usedBytes);
5003 
5004  json.WriteString("UnusedBytes");
5005  json.WriteNumber(stat.unusedBytes);
5006 
5007  if(stat.allocationCount > 1)
5008  {
5009  json.WriteString("AllocationSize");
5010  json.BeginObject(true);
5011  json.WriteString("Min");
5012  json.WriteNumber(stat.allocationSizeMin);
5013  json.WriteString("Avg");
5014  json.WriteNumber(stat.allocationSizeAvg);
5015  json.WriteString("Max");
5016  json.WriteNumber(stat.allocationSizeMax);
5017  json.EndObject();
5018  }
5019 
5020  if(stat.unusedRangeCount > 1)
5021  {
5022  json.WriteString("UnusedRangeSize");
5023  json.BeginObject(true);
5024  json.WriteString("Min");
5025  json.WriteNumber(stat.unusedRangeSizeMin);
5026  json.WriteString("Avg");
5027  json.WriteNumber(stat.unusedRangeSizeAvg);
5028  json.WriteString("Max");
5029  json.WriteNumber(stat.unusedRangeSizeMax);
5030  json.EndObject();
5031  }
5032 
5033  json.EndObject();
5034 }
5035 
5036 #endif // #if VMA_STATS_STRING_ENABLED
5037 
5038 struct VmaSuballocationItemSizeLess
5039 {
5040  bool operator()(
5041  const VmaSuballocationList::iterator lhs,
5042  const VmaSuballocationList::iterator rhs) const
5043  {
5044  return lhs->size < rhs->size;
5045  }
5046  bool operator()(
5047  const VmaSuballocationList::iterator lhs,
5048  VkDeviceSize rhsSize) const
5049  {
5050  return lhs->size < rhsSize;
5051  }
5052 };
5053 
5055 // class VmaBlockMetadata
5056 
5057 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
5058  m_Size(0),
5059  m_FreeCount(0),
5060  m_SumFreeSize(0),
5061  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
5062  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
5063 {
5064 }
5065 
5066 VmaBlockMetadata::~VmaBlockMetadata()
5067 {
5068 }
5069 
5070 void VmaBlockMetadata::Init(VkDeviceSize size)
5071 {
5072  m_Size = size;
5073  m_FreeCount = 1;
5074  m_SumFreeSize = size;
5075 
5076  VmaSuballocation suballoc = {};
5077  suballoc.offset = 0;
5078  suballoc.size = size;
5079  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5080  suballoc.hAllocation = VK_NULL_HANDLE;
5081 
5082  m_Suballocations.push_back(suballoc);
5083  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
5084  --suballocItem;
5085  m_FreeSuballocationsBySize.push_back(suballocItem);
5086 }
5087 
5088 bool VmaBlockMetadata::Validate() const
5089 {
5090  if(m_Suballocations.empty())
5091  {
5092  return false;
5093  }
5094 
5095  // Expected offset of new suballocation as calculates from previous ones.
5096  VkDeviceSize calculatedOffset = 0;
5097  // Expected number of free suballocations as calculated from traversing their list.
5098  uint32_t calculatedFreeCount = 0;
5099  // Expected sum size of free suballocations as calculated from traversing their list.
5100  VkDeviceSize calculatedSumFreeSize = 0;
5101  // Expected number of free suballocations that should be registered in
5102  // m_FreeSuballocationsBySize calculated from traversing their list.
5103  size_t freeSuballocationsToRegister = 0;
5104  // True if previous visisted suballocation was free.
5105  bool prevFree = false;
5106 
5107  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5108  suballocItem != m_Suballocations.cend();
5109  ++suballocItem)
5110  {
5111  const VmaSuballocation& subAlloc = *suballocItem;
5112 
5113  // Actual offset of this suballocation doesn't match expected one.
5114  if(subAlloc.offset != calculatedOffset)
5115  {
5116  return false;
5117  }
5118 
5119  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
5120  // Two adjacent free suballocations are invalid. They should be merged.
5121  if(prevFree && currFree)
5122  {
5123  return false;
5124  }
5125 
5126  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
5127  {
5128  return false;
5129  }
5130 
5131  if(currFree)
5132  {
5133  calculatedSumFreeSize += subAlloc.size;
5134  ++calculatedFreeCount;
5135  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5136  {
5137  ++freeSuballocationsToRegister;
5138  }
5139  }
5140  else
5141  {
5142  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
5143  {
5144  return false;
5145  }
5146  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
5147  {
5148  return false;
5149  }
5150  }
5151 
5152  calculatedOffset += subAlloc.size;
5153  prevFree = currFree;
5154  }
5155 
5156  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
5157  // match expected one.
5158  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
5159  {
5160  return false;
5161  }
5162 
5163  VkDeviceSize lastSize = 0;
5164  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
5165  {
5166  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
5167 
5168  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
5169  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5170  {
5171  return false;
5172  }
5173  // They must be sorted by size ascending.
5174  if(suballocItem->size < lastSize)
5175  {
5176  return false;
5177  }
5178 
5179  lastSize = suballocItem->size;
5180  }
5181 
5182  // Check if totals match calculacted values.
5183  if(!ValidateFreeSuballocationList() ||
5184  (calculatedOffset != m_Size) ||
5185  (calculatedSumFreeSize != m_SumFreeSize) ||
5186  (calculatedFreeCount != m_FreeCount))
5187  {
5188  return false;
5189  }
5190 
5191  return true;
5192 }
5193 
5194 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
5195 {
5196  if(!m_FreeSuballocationsBySize.empty())
5197  {
5198  return m_FreeSuballocationsBySize.back()->size;
5199  }
5200  else
5201  {
5202  return 0;
5203  }
5204 }
5205 
5206 bool VmaBlockMetadata::IsEmpty() const
5207 {
5208  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
5209 }
5210 
5211 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
5212 {
5213  outInfo.blockCount = 1;
5214 
5215  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5216  outInfo.allocationCount = rangeCount - m_FreeCount;
5217  outInfo.unusedRangeCount = m_FreeCount;
5218 
5219  outInfo.unusedBytes = m_SumFreeSize;
5220  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
5221 
5222  outInfo.allocationSizeMin = UINT64_MAX;
5223  outInfo.allocationSizeMax = 0;
5224  outInfo.unusedRangeSizeMin = UINT64_MAX;
5225  outInfo.unusedRangeSizeMax = 0;
5226 
5227  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5228  suballocItem != m_Suballocations.cend();
5229  ++suballocItem)
5230  {
5231  const VmaSuballocation& suballoc = *suballocItem;
5232  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
5233  {
5234  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
5235  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
5236  }
5237  else
5238  {
5239  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
5240  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
5241  }
5242  }
5243 }
5244 
5245 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
5246 {
5247  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5248 
5249  inoutStats.size += m_Size;
5250  inoutStats.unusedSize += m_SumFreeSize;
5251  inoutStats.allocationCount += rangeCount - m_FreeCount;
5252  inoutStats.unusedRangeCount += m_FreeCount;
5253  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
5254 }
5255 
5256 #if VMA_STATS_STRING_ENABLED
5257 
5258 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
5259 {
5260  json.BeginObject();
5261 
5262  json.WriteString("TotalBytes");
5263  json.WriteNumber(m_Size);
5264 
5265  json.WriteString("UnusedBytes");
5266  json.WriteNumber(m_SumFreeSize);
5267 
5268  json.WriteString("Allocations");
5269  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
5270 
5271  json.WriteString("UnusedRanges");
5272  json.WriteNumber(m_FreeCount);
5273 
5274  json.WriteString("Suballocations");
5275  json.BeginArray();
5276  size_t i = 0;
5277  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5278  suballocItem != m_Suballocations.cend();
5279  ++suballocItem, ++i)
5280  {
5281  json.BeginObject(true);
5282 
5283  json.WriteString("Type");
5284  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
5285 
5286  json.WriteString("Size");
5287  json.WriteNumber(suballocItem->size);
5288 
5289  json.WriteString("Offset");
5290  json.WriteNumber(suballocItem->offset);
5291 
5292  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5293  {
5294  const void* pUserData = suballocItem->hAllocation->GetUserData();
5295  if(pUserData != VMA_NULL)
5296  {
5297  json.WriteString("UserData");
5298  if(suballocItem->hAllocation->IsUserDataString())
5299  {
5300  json.WriteString((const char*)pUserData);
5301  }
5302  else
5303  {
5304  json.BeginString();
5305  json.ContinueString_Pointer(pUserData);
5306  json.EndString();
5307  }
5308  }
5309  }
5310 
5311  json.EndObject();
5312  }
5313  json.EndArray();
5314 
5315  json.EndObject();
5316 }
5317 
5318 #endif // #if VMA_STATS_STRING_ENABLED
5319 
5320 /*
5321 How many suitable free suballocations to analyze before choosing best one.
5322 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
5323  be chosen.
5324 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
5325  suballocations will be analized and best one will be chosen.
5326 - Any other value is also acceptable.
5327 */
5328 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
5329 
5330 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
5331 {
5332  VMA_ASSERT(IsEmpty());
5333  pAllocationRequest->offset = 0;
5334  pAllocationRequest->sumFreeSize = m_SumFreeSize;
5335  pAllocationRequest->sumItemSize = 0;
5336  pAllocationRequest->item = m_Suballocations.begin();
5337  pAllocationRequest->itemsToMakeLostCount = 0;
5338 }
5339 
5340 bool VmaBlockMetadata::CreateAllocationRequest(
5341  uint32_t currentFrameIndex,
5342  uint32_t frameInUseCount,
5343  VkDeviceSize bufferImageGranularity,
5344  VkDeviceSize allocSize,
5345  VkDeviceSize allocAlignment,
5346  VmaSuballocationType allocType,
5347  bool canMakeOtherLost,
5348  VmaAllocationRequest* pAllocationRequest)
5349 {
5350  VMA_ASSERT(allocSize > 0);
5351  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5352  VMA_ASSERT(pAllocationRequest != VMA_NULL);
5353  VMA_HEAVY_ASSERT(Validate());
5354 
5355  // There is not enough total free space in this block to fullfill the request: Early return.
5356  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
5357  {
5358  return false;
5359  }
5360 
5361  // New algorithm, efficiently searching freeSuballocationsBySize.
5362  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
5363  if(freeSuballocCount > 0)
5364  {
5365  if(VMA_BEST_FIT)
5366  {
5367  // Find first free suballocation with size not less than allocSize.
5368  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5369  m_FreeSuballocationsBySize.data(),
5370  m_FreeSuballocationsBySize.data() + freeSuballocCount,
5371  allocSize,
5372  VmaSuballocationItemSizeLess());
5373  size_t index = it - m_FreeSuballocationsBySize.data();
5374  for(; index < freeSuballocCount; ++index)
5375  {
5376  if(CheckAllocation(
5377  currentFrameIndex,
5378  frameInUseCount,
5379  bufferImageGranularity,
5380  allocSize,
5381  allocAlignment,
5382  allocType,
5383  m_FreeSuballocationsBySize[index],
5384  false, // canMakeOtherLost
5385  &pAllocationRequest->offset,
5386  &pAllocationRequest->itemsToMakeLostCount,
5387  &pAllocationRequest->sumFreeSize,
5388  &pAllocationRequest->sumItemSize))
5389  {
5390  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5391  return true;
5392  }
5393  }
5394  }
5395  else
5396  {
5397  // Search staring from biggest suballocations.
5398  for(size_t index = freeSuballocCount; index--; )
5399  {
5400  if(CheckAllocation(
5401  currentFrameIndex,
5402  frameInUseCount,
5403  bufferImageGranularity,
5404  allocSize,
5405  allocAlignment,
5406  allocType,
5407  m_FreeSuballocationsBySize[index],
5408  false, // canMakeOtherLost
5409  &pAllocationRequest->offset,
5410  &pAllocationRequest->itemsToMakeLostCount,
5411  &pAllocationRequest->sumFreeSize,
5412  &pAllocationRequest->sumItemSize))
5413  {
5414  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5415  return true;
5416  }
5417  }
5418  }
5419  }
5420 
5421  if(canMakeOtherLost)
5422  {
5423  // Brute-force algorithm. TODO: Come up with something better.
5424 
5425  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
5426  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
5427 
5428  VmaAllocationRequest tmpAllocRequest = {};
5429  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
5430  suballocIt != m_Suballocations.end();
5431  ++suballocIt)
5432  {
5433  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
5434  suballocIt->hAllocation->CanBecomeLost())
5435  {
5436  if(CheckAllocation(
5437  currentFrameIndex,
5438  frameInUseCount,
5439  bufferImageGranularity,
5440  allocSize,
5441  allocAlignment,
5442  allocType,
5443  suballocIt,
5444  canMakeOtherLost,
5445  &tmpAllocRequest.offset,
5446  &tmpAllocRequest.itemsToMakeLostCount,
5447  &tmpAllocRequest.sumFreeSize,
5448  &tmpAllocRequest.sumItemSize))
5449  {
5450  tmpAllocRequest.item = suballocIt;
5451 
5452  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
5453  {
5454  *pAllocationRequest = tmpAllocRequest;
5455  }
5456  }
5457  }
5458  }
5459 
5460  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
5461  {
5462  return true;
5463  }
5464  }
5465 
5466  return false;
5467 }
5468 
5469 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
5470  uint32_t currentFrameIndex,
5471  uint32_t frameInUseCount,
5472  VmaAllocationRequest* pAllocationRequest)
5473 {
5474  while(pAllocationRequest->itemsToMakeLostCount > 0)
5475  {
5476  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
5477  {
5478  ++pAllocationRequest->item;
5479  }
5480  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5481  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
5482  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
5483  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5484  {
5485  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
5486  --pAllocationRequest->itemsToMakeLostCount;
5487  }
5488  else
5489  {
5490  return false;
5491  }
5492  }
5493 
5494  VMA_HEAVY_ASSERT(Validate());
5495  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5496  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
5497 
5498  return true;
5499 }
5500 
5501 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5502 {
5503  uint32_t lostAllocationCount = 0;
5504  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
5505  it != m_Suballocations.end();
5506  ++it)
5507  {
5508  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
5509  it->hAllocation->CanBecomeLost() &&
5510  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5511  {
5512  it = FreeSuballocation(it);
5513  ++lostAllocationCount;
5514  }
5515  }
5516  return lostAllocationCount;
5517 }
5518 
5519 void VmaBlockMetadata::Alloc(
5520  const VmaAllocationRequest& request,
5521  VmaSuballocationType type,
5522  VkDeviceSize allocSize,
5523  VmaAllocation hAllocation)
5524 {
5525  VMA_ASSERT(request.item != m_Suballocations.end());
5526  VmaSuballocation& suballoc = *request.item;
5527  // Given suballocation is a free block.
5528  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5529  // Given offset is inside this suballocation.
5530  VMA_ASSERT(request.offset >= suballoc.offset);
5531  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
5532  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
5533  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
5534 
5535  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
5536  // it to become used.
5537  UnregisterFreeSuballocation(request.item);
5538 
5539  suballoc.offset = request.offset;
5540  suballoc.size = allocSize;
5541  suballoc.type = type;
5542  suballoc.hAllocation = hAllocation;
5543 
5544  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
5545  if(paddingEnd)
5546  {
5547  VmaSuballocation paddingSuballoc = {};
5548  paddingSuballoc.offset = request.offset + allocSize;
5549  paddingSuballoc.size = paddingEnd;
5550  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5551  VmaSuballocationList::iterator next = request.item;
5552  ++next;
5553  const VmaSuballocationList::iterator paddingEndItem =
5554  m_Suballocations.insert(next, paddingSuballoc);
5555  RegisterFreeSuballocation(paddingEndItem);
5556  }
5557 
5558  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
5559  if(paddingBegin)
5560  {
5561  VmaSuballocation paddingSuballoc = {};
5562  paddingSuballoc.offset = request.offset - paddingBegin;
5563  paddingSuballoc.size = paddingBegin;
5564  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5565  const VmaSuballocationList::iterator paddingBeginItem =
5566  m_Suballocations.insert(request.item, paddingSuballoc);
5567  RegisterFreeSuballocation(paddingBeginItem);
5568  }
5569 
5570  // Update totals.
5571  m_FreeCount = m_FreeCount - 1;
5572  if(paddingBegin > 0)
5573  {
5574  ++m_FreeCount;
5575  }
5576  if(paddingEnd > 0)
5577  {
5578  ++m_FreeCount;
5579  }
5580  m_SumFreeSize -= allocSize;
5581 }
5582 
5583 void VmaBlockMetadata::Free(const VmaAllocation allocation)
5584 {
5585  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5586  suballocItem != m_Suballocations.end();
5587  ++suballocItem)
5588  {
5589  VmaSuballocation& suballoc = *suballocItem;
5590  if(suballoc.hAllocation == allocation)
5591  {
5592  FreeSuballocation(suballocItem);
5593  VMA_HEAVY_ASSERT(Validate());
5594  return;
5595  }
5596  }
5597  VMA_ASSERT(0 && "Not found!");
5598 }
5599 
5600 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
5601 {
5602  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5603  suballocItem != m_Suballocations.end();
5604  ++suballocItem)
5605  {
5606  VmaSuballocation& suballoc = *suballocItem;
5607  if(suballoc.offset == offset)
5608  {
5609  FreeSuballocation(suballocItem);
5610  return;
5611  }
5612  }
5613  VMA_ASSERT(0 && "Not found!");
5614 }
5615 
5616 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
5617 {
5618  VkDeviceSize lastSize = 0;
5619  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
5620  {
5621  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
5622 
5623  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
5624  {
5625  VMA_ASSERT(0);
5626  return false;
5627  }
5628  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5629  {
5630  VMA_ASSERT(0);
5631  return false;
5632  }
5633  if(it->size < lastSize)
5634  {
5635  VMA_ASSERT(0);
5636  return false;
5637  }
5638 
5639  lastSize = it->size;
5640  }
5641  return true;
5642 }
5643 
5644 bool VmaBlockMetadata::CheckAllocation(
5645  uint32_t currentFrameIndex,
5646  uint32_t frameInUseCount,
5647  VkDeviceSize bufferImageGranularity,
5648  VkDeviceSize allocSize,
5649  VkDeviceSize allocAlignment,
5650  VmaSuballocationType allocType,
5651  VmaSuballocationList::const_iterator suballocItem,
5652  bool canMakeOtherLost,
5653  VkDeviceSize* pOffset,
5654  size_t* itemsToMakeLostCount,
5655  VkDeviceSize* pSumFreeSize,
5656  VkDeviceSize* pSumItemSize) const
5657 {
5658  VMA_ASSERT(allocSize > 0);
5659  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5660  VMA_ASSERT(suballocItem != m_Suballocations.cend());
5661  VMA_ASSERT(pOffset != VMA_NULL);
5662 
5663  *itemsToMakeLostCount = 0;
5664  *pSumFreeSize = 0;
5665  *pSumItemSize = 0;
5666 
5667  if(canMakeOtherLost)
5668  {
5669  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5670  {
5671  *pSumFreeSize = suballocItem->size;
5672  }
5673  else
5674  {
5675  if(suballocItem->hAllocation->CanBecomeLost() &&
5676  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5677  {
5678  ++*itemsToMakeLostCount;
5679  *pSumItemSize = suballocItem->size;
5680  }
5681  else
5682  {
5683  return false;
5684  }
5685  }
5686 
5687  // Remaining size is too small for this request: Early return.
5688  if(m_Size - suballocItem->offset < allocSize)
5689  {
5690  return false;
5691  }
5692 
5693  // Start from offset equal to beginning of this suballocation.
5694  *pOffset = suballocItem->offset;
5695 
5696  // Apply VMA_DEBUG_MARGIN at the beginning.
5697  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5698  {
5699  *pOffset += VMA_DEBUG_MARGIN;
5700  }
5701 
5702  // Apply alignment.
5703  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5704  *pOffset = VmaAlignUp(*pOffset, alignment);
5705 
5706  // Check previous suballocations for BufferImageGranularity conflicts.
5707  // Make bigger alignment if necessary.
5708  if(bufferImageGranularity > 1)
5709  {
5710  bool bufferImageGranularityConflict = false;
5711  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5712  while(prevSuballocItem != m_Suballocations.cbegin())
5713  {
5714  --prevSuballocItem;
5715  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5716  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5717  {
5718  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5719  {
5720  bufferImageGranularityConflict = true;
5721  break;
5722  }
5723  }
5724  else
5725  // Already on previous page.
5726  break;
5727  }
5728  if(bufferImageGranularityConflict)
5729  {
5730  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5731  }
5732  }
5733 
5734  // Now that we have final *pOffset, check if we are past suballocItem.
5735  // If yes, return false - this function should be called for another suballocItem as starting point.
5736  if(*pOffset >= suballocItem->offset + suballocItem->size)
5737  {
5738  return false;
5739  }
5740 
5741  // Calculate padding at the beginning based on current offset.
5742  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
5743 
5744  // Calculate required margin at the end if this is not last suballocation.
5745  VmaSuballocationList::const_iterator next = suballocItem;
5746  ++next;
5747  const VkDeviceSize requiredEndMargin =
5748  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5749 
5750  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
5751  // Another early return check.
5752  if(suballocItem->offset + totalSize > m_Size)
5753  {
5754  return false;
5755  }
5756 
5757  // Advance lastSuballocItem until desired size is reached.
5758  // Update itemsToMakeLostCount.
5759  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
5760  if(totalSize > suballocItem->size)
5761  {
5762  VkDeviceSize remainingSize = totalSize - suballocItem->size;
5763  while(remainingSize > 0)
5764  {
5765  ++lastSuballocItem;
5766  if(lastSuballocItem == m_Suballocations.cend())
5767  {
5768  return false;
5769  }
5770  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5771  {
5772  *pSumFreeSize += lastSuballocItem->size;
5773  }
5774  else
5775  {
5776  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
5777  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
5778  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5779  {
5780  ++*itemsToMakeLostCount;
5781  *pSumItemSize += lastSuballocItem->size;
5782  }
5783  else
5784  {
5785  return false;
5786  }
5787  }
5788  remainingSize = (lastSuballocItem->size < remainingSize) ?
5789  remainingSize - lastSuballocItem->size : 0;
5790  }
5791  }
5792 
5793  // Check next suballocations for BufferImageGranularity conflicts.
5794  // If conflict exists, we must mark more allocations lost or fail.
5795  if(bufferImageGranularity > 1)
5796  {
5797  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
5798  ++nextSuballocItem;
5799  while(nextSuballocItem != m_Suballocations.cend())
5800  {
5801  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5802  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5803  {
5804  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5805  {
5806  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
5807  if(nextSuballoc.hAllocation->CanBecomeLost() &&
5808  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5809  {
5810  ++*itemsToMakeLostCount;
5811  }
5812  else
5813  {
5814  return false;
5815  }
5816  }
5817  }
5818  else
5819  {
5820  // Already on next page.
5821  break;
5822  }
5823  ++nextSuballocItem;
5824  }
5825  }
5826  }
5827  else
5828  {
5829  const VmaSuballocation& suballoc = *suballocItem;
5830  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5831 
5832  *pSumFreeSize = suballoc.size;
5833 
5834  // Size of this suballocation is too small for this request: Early return.
5835  if(suballoc.size < allocSize)
5836  {
5837  return false;
5838  }
5839 
5840  // Start from offset equal to beginning of this suballocation.
5841  *pOffset = suballoc.offset;
5842 
5843  // Apply VMA_DEBUG_MARGIN at the beginning.
5844  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5845  {
5846  *pOffset += VMA_DEBUG_MARGIN;
5847  }
5848 
5849  // Apply alignment.
5850  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5851  *pOffset = VmaAlignUp(*pOffset, alignment);
5852 
5853  // Check previous suballocations for BufferImageGranularity conflicts.
5854  // Make bigger alignment if necessary.
5855  if(bufferImageGranularity > 1)
5856  {
5857  bool bufferImageGranularityConflict = false;
5858  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5859  while(prevSuballocItem != m_Suballocations.cbegin())
5860  {
5861  --prevSuballocItem;
5862  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5863  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5864  {
5865  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5866  {
5867  bufferImageGranularityConflict = true;
5868  break;
5869  }
5870  }
5871  else
5872  // Already on previous page.
5873  break;
5874  }
5875  if(bufferImageGranularityConflict)
5876  {
5877  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5878  }
5879  }
5880 
5881  // Calculate padding at the beginning based on current offset.
5882  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
5883 
5884  // Calculate required margin at the end if this is not last suballocation.
5885  VmaSuballocationList::const_iterator next = suballocItem;
5886  ++next;
5887  const VkDeviceSize requiredEndMargin =
5888  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5889 
5890  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
5891  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
5892  {
5893  return false;
5894  }
5895 
5896  // Check next suballocations for BufferImageGranularity conflicts.
5897  // If conflict exists, allocation cannot be made here.
5898  if(bufferImageGranularity > 1)
5899  {
5900  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
5901  ++nextSuballocItem;
5902  while(nextSuballocItem != m_Suballocations.cend())
5903  {
5904  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5905  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5906  {
5907  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5908  {
5909  return false;
5910  }
5911  }
5912  else
5913  {
5914  // Already on next page.
5915  break;
5916  }
5917  ++nextSuballocItem;
5918  }
5919  }
5920  }
5921 
5922  // All tests passed: Success. pOffset is already filled.
5923  return true;
5924 }
5925 
5926 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
5927 {
5928  VMA_ASSERT(item != m_Suballocations.end());
5929  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5930 
5931  VmaSuballocationList::iterator nextItem = item;
5932  ++nextItem;
5933  VMA_ASSERT(nextItem != m_Suballocations.end());
5934  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
5935 
5936  item->size += nextItem->size;
5937  --m_FreeCount;
5938  m_Suballocations.erase(nextItem);
5939 }
5940 
5941 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
5942 {
5943  // Change this suballocation to be marked as free.
5944  VmaSuballocation& suballoc = *suballocItem;
5945  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5946  suballoc.hAllocation = VK_NULL_HANDLE;
5947 
5948  // Update totals.
5949  ++m_FreeCount;
5950  m_SumFreeSize += suballoc.size;
5951 
5952  // Merge with previous and/or next suballocation if it's also free.
5953  bool mergeWithNext = false;
5954  bool mergeWithPrev = false;
5955 
5956  VmaSuballocationList::iterator nextItem = suballocItem;
5957  ++nextItem;
5958  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
5959  {
5960  mergeWithNext = true;
5961  }
5962 
5963  VmaSuballocationList::iterator prevItem = suballocItem;
5964  if(suballocItem != m_Suballocations.begin())
5965  {
5966  --prevItem;
5967  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5968  {
5969  mergeWithPrev = true;
5970  }
5971  }
5972 
5973  if(mergeWithNext)
5974  {
5975  UnregisterFreeSuballocation(nextItem);
5976  MergeFreeWithNext(suballocItem);
5977  }
5978 
5979  if(mergeWithPrev)
5980  {
5981  UnregisterFreeSuballocation(prevItem);
5982  MergeFreeWithNext(prevItem);
5983  RegisterFreeSuballocation(prevItem);
5984  return prevItem;
5985  }
5986  else
5987  {
5988  RegisterFreeSuballocation(suballocItem);
5989  return suballocItem;
5990  }
5991 }
5992 
5993 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
5994 {
5995  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5996  VMA_ASSERT(item->size > 0);
5997 
5998  // You may want to enable this validation at the beginning or at the end of
5999  // this function, depending on what do you want to check.
6000  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6001 
6002  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6003  {
6004  if(m_FreeSuballocationsBySize.empty())
6005  {
6006  m_FreeSuballocationsBySize.push_back(item);
6007  }
6008  else
6009  {
6010  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
6011  }
6012  }
6013 
6014  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6015 }
6016 
6017 
6018 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
6019 {
6020  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6021  VMA_ASSERT(item->size > 0);
6022 
6023  // You may want to enable this validation at the beginning or at the end of
6024  // this function, depending on what do you want to check.
6025  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6026 
6027  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6028  {
6029  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
6030  m_FreeSuballocationsBySize.data(),
6031  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
6032  item,
6033  VmaSuballocationItemSizeLess());
6034  for(size_t index = it - m_FreeSuballocationsBySize.data();
6035  index < m_FreeSuballocationsBySize.size();
6036  ++index)
6037  {
6038  if(m_FreeSuballocationsBySize[index] == item)
6039  {
6040  VmaVectorRemove(m_FreeSuballocationsBySize, index);
6041  return;
6042  }
6043  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
6044  }
6045  VMA_ASSERT(0 && "Not found.");
6046  }
6047 
6048  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6049 }
6050 
6052 // class VmaDeviceMemoryMapping
6053 
6054 VmaDeviceMemoryMapping::VmaDeviceMemoryMapping() :
6055  m_MapCount(0),
6056  m_pMappedData(VMA_NULL)
6057 {
6058 }
6059 
6060 VmaDeviceMemoryMapping::~VmaDeviceMemoryMapping()
6061 {
6062  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
6063 }
6064 
6065 VkResult VmaDeviceMemoryMapping::Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData)
6066 {
6067  if(count == 0)
6068  {
6069  return VK_SUCCESS;
6070  }
6071 
6072  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6073  if(m_MapCount != 0)
6074  {
6075  m_MapCount += count;
6076  VMA_ASSERT(m_pMappedData != VMA_NULL);
6077  if(ppData != VMA_NULL)
6078  {
6079  *ppData = m_pMappedData;
6080  }
6081  return VK_SUCCESS;
6082  }
6083  else
6084  {
6085  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
6086  hAllocator->m_hDevice,
6087  hMemory,
6088  0, // offset
6089  VK_WHOLE_SIZE,
6090  0, // flags
6091  &m_pMappedData);
6092  if(result == VK_SUCCESS)
6093  {
6094  if(ppData != VMA_NULL)
6095  {
6096  *ppData = m_pMappedData;
6097  }
6098  m_MapCount = count;
6099  }
6100  return result;
6101  }
6102 }
6103 
6104 void VmaDeviceMemoryMapping::Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count)
6105 {
6106  if(count == 0)
6107  {
6108  return;
6109  }
6110 
6111  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6112  if(m_MapCount >= count)
6113  {
6114  m_MapCount -= count;
6115  if(m_MapCount == 0)
6116  {
6117  m_pMappedData = VMA_NULL;
6118  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, hMemory);
6119  }
6120  }
6121  else
6122  {
6123  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
6124  }
6125 }
6126 
6128 // class VmaDeviceMemoryBlock
6129 
6130 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
6131  m_MemoryTypeIndex(UINT32_MAX),
6132  m_hMemory(VK_NULL_HANDLE),
6133  m_Metadata(hAllocator)
6134 {
6135 }
6136 
6137 void VmaDeviceMemoryBlock::Init(
6138  uint32_t newMemoryTypeIndex,
6139  VkDeviceMemory newMemory,
6140  VkDeviceSize newSize)
6141 {
6142  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
6143 
6144  m_MemoryTypeIndex = newMemoryTypeIndex;
6145  m_hMemory = newMemory;
6146 
6147  m_Metadata.Init(newSize);
6148 }
6149 
6150 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
6151 {
6152  // This is the most important assert in the entire library.
6153  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
6154  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
6155 
6156  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
6157  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
6158  m_hMemory = VK_NULL_HANDLE;
6159 }
6160 
6161 bool VmaDeviceMemoryBlock::Validate() const
6162 {
6163  if((m_hMemory == VK_NULL_HANDLE) ||
6164  (m_Metadata.GetSize() == 0))
6165  {
6166  return false;
6167  }
6168 
6169  return m_Metadata.Validate();
6170 }
6171 
6172 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
6173 {
6174  return m_Mapping.Map(hAllocator, m_hMemory, count, ppData);
6175 }
6176 
6177 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
6178 {
6179  m_Mapping.Unmap(hAllocator, m_hMemory, count);
6180 }
6181 
6182 static void InitStatInfo(VmaStatInfo& outInfo)
6183 {
6184  memset(&outInfo, 0, sizeof(outInfo));
6185  outInfo.allocationSizeMin = UINT64_MAX;
6186  outInfo.unusedRangeSizeMin = UINT64_MAX;
6187 }
6188 
6189 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
6190 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
6191 {
6192  inoutInfo.blockCount += srcInfo.blockCount;
6193  inoutInfo.allocationCount += srcInfo.allocationCount;
6194  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
6195  inoutInfo.usedBytes += srcInfo.usedBytes;
6196  inoutInfo.unusedBytes += srcInfo.unusedBytes;
6197  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
6198  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
6199  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
6200  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
6201 }
6202 
6203 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
6204 {
6205  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
6206  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
6207  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
6208  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
6209 }
6210 
6211 VmaPool_T::VmaPool_T(
6212  VmaAllocator hAllocator,
6213  const VmaPoolCreateInfo& createInfo) :
6214  m_BlockVector(
6215  hAllocator,
6216  createInfo.memoryTypeIndex,
6217  createInfo.blockSize,
6218  createInfo.minBlockCount,
6219  createInfo.maxBlockCount,
6220  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
6221  createInfo.frameInUseCount,
6222  true) // isCustomPool
6223 {
6224 }
6225 
6226 VmaPool_T::~VmaPool_T()
6227 {
6228 }
6229 
6230 #if VMA_STATS_STRING_ENABLED
6231 
6232 #endif // #if VMA_STATS_STRING_ENABLED
6233 
6234 VmaBlockVector::VmaBlockVector(
6235  VmaAllocator hAllocator,
6236  uint32_t memoryTypeIndex,
6237  VkDeviceSize preferredBlockSize,
6238  size_t minBlockCount,
6239  size_t maxBlockCount,
6240  VkDeviceSize bufferImageGranularity,
6241  uint32_t frameInUseCount,
6242  bool isCustomPool) :
6243  m_hAllocator(hAllocator),
6244  m_MemoryTypeIndex(memoryTypeIndex),
6245  m_PreferredBlockSize(preferredBlockSize),
6246  m_MinBlockCount(minBlockCount),
6247  m_MaxBlockCount(maxBlockCount),
6248  m_BufferImageGranularity(bufferImageGranularity),
6249  m_FrameInUseCount(frameInUseCount),
6250  m_IsCustomPool(isCustomPool),
6251  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
6252  m_HasEmptyBlock(false),
6253  m_pDefragmentator(VMA_NULL)
6254 {
6255 }
6256 
6257 VmaBlockVector::~VmaBlockVector()
6258 {
6259  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
6260 
6261  for(size_t i = m_Blocks.size(); i--; )
6262  {
6263  m_Blocks[i]->Destroy(m_hAllocator);
6264  vma_delete(m_hAllocator, m_Blocks[i]);
6265  }
6266 }
6267 
6268 VkResult VmaBlockVector::CreateMinBlocks()
6269 {
6270  for(size_t i = 0; i < m_MinBlockCount; ++i)
6271  {
6272  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
6273  if(res != VK_SUCCESS)
6274  {
6275  return res;
6276  }
6277  }
6278  return VK_SUCCESS;
6279 }
6280 
6281 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
6282 {
6283  pStats->size = 0;
6284  pStats->unusedSize = 0;
6285  pStats->allocationCount = 0;
6286  pStats->unusedRangeCount = 0;
6287  pStats->unusedRangeSizeMax = 0;
6288 
6289  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6290 
6291  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6292  {
6293  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6294  VMA_ASSERT(pBlock);
6295  VMA_HEAVY_ASSERT(pBlock->Validate());
6296  pBlock->m_Metadata.AddPoolStats(*pStats);
6297  }
6298 }
6299 
6300 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
6301 
6302 VkResult VmaBlockVector::Allocate(
6303  VmaPool hCurrentPool,
6304  uint32_t currentFrameIndex,
6305  const VkMemoryRequirements& vkMemReq,
6306  const VmaAllocationCreateInfo& createInfo,
6307  VmaSuballocationType suballocType,
6308  VmaAllocation* pAllocation)
6309 {
6310  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
6311  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
6312 
6313  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6314 
6315  // 1. Search existing allocations. Try to allocate without making other allocations lost.
6316  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6317  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6318  {
6319  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6320  VMA_ASSERT(pCurrBlock);
6321  VmaAllocationRequest currRequest = {};
6322  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6323  currentFrameIndex,
6324  m_FrameInUseCount,
6325  m_BufferImageGranularity,
6326  vkMemReq.size,
6327  vkMemReq.alignment,
6328  suballocType,
6329  false, // canMakeOtherLost
6330  &currRequest))
6331  {
6332  // Allocate from pCurrBlock.
6333  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
6334 
6335  if(mapped)
6336  {
6337  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
6338  if(res != VK_SUCCESS)
6339  {
6340  return res;
6341  }
6342  }
6343 
6344  // We no longer have an empty Allocation.
6345  if(pCurrBlock->m_Metadata.IsEmpty())
6346  {
6347  m_HasEmptyBlock = false;
6348  }
6349 
6350  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6351  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
6352  (*pAllocation)->InitBlockAllocation(
6353  hCurrentPool,
6354  pCurrBlock,
6355  currRequest.offset,
6356  vkMemReq.alignment,
6357  vkMemReq.size,
6358  suballocType,
6359  mapped,
6360  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6361  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
6362  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6363  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6364  return VK_SUCCESS;
6365  }
6366  }
6367 
6368  const bool canCreateNewBlock =
6369  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
6370  (m_Blocks.size() < m_MaxBlockCount);
6371 
6372  // 2. Try to create new block.
6373  if(canCreateNewBlock)
6374  {
6375  // Calculate optimal size for new block.
6376  VkDeviceSize newBlockSize = m_PreferredBlockSize;
6377  uint32_t newBlockSizeShift = 0;
6378  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
6379 
6380  // Allocating blocks of other sizes is allowed only in default pools.
6381  // In custom pools block size is fixed.
6382  if(m_IsCustomPool == false)
6383  {
6384  // Allocate 1/8, 1/4, 1/2 as first blocks.
6385  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
6386  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
6387  {
6388  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6389  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= vkMemReq.size * 2)
6390  {
6391  newBlockSize = smallerNewBlockSize;
6392  ++newBlockSizeShift;
6393  }
6394  else
6395  {
6396  break;
6397  }
6398  }
6399  }
6400 
6401  size_t newBlockIndex = 0;
6402  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
6403  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
6404  if(m_IsCustomPool == false)
6405  {
6406  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
6407  {
6408  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6409  if(smallerNewBlockSize >= vkMemReq.size)
6410  {
6411  newBlockSize = smallerNewBlockSize;
6412  ++newBlockSizeShift;
6413  res = CreateBlock(newBlockSize, &newBlockIndex);
6414  }
6415  else
6416  {
6417  break;
6418  }
6419  }
6420  }
6421 
6422  if(res == VK_SUCCESS)
6423  {
6424  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
6425  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= vkMemReq.size);
6426 
6427  if(mapped)
6428  {
6429  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
6430  if(res != VK_SUCCESS)
6431  {
6432  return res;
6433  }
6434  }
6435 
6436  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
6437  VmaAllocationRequest allocRequest;
6438  pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
6439  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6440  pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
6441  (*pAllocation)->InitBlockAllocation(
6442  hCurrentPool,
6443  pBlock,
6444  allocRequest.offset,
6445  vkMemReq.alignment,
6446  vkMemReq.size,
6447  suballocType,
6448  mapped,
6449  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6450  VMA_HEAVY_ASSERT(pBlock->Validate());
6451  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
6452  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6453  return VK_SUCCESS;
6454  }
6455  }
6456 
6457  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
6458 
6459  // 3. Try to allocate from existing blocks with making other allocations lost.
6460  if(canMakeOtherLost)
6461  {
6462  uint32_t tryIndex = 0;
6463  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
6464  {
6465  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
6466  VmaAllocationRequest bestRequest = {};
6467  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
6468 
6469  // 1. Search existing allocations.
6470  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6471  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6472  {
6473  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6474  VMA_ASSERT(pCurrBlock);
6475  VmaAllocationRequest currRequest = {};
6476  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6477  currentFrameIndex,
6478  m_FrameInUseCount,
6479  m_BufferImageGranularity,
6480  vkMemReq.size,
6481  vkMemReq.alignment,
6482  suballocType,
6483  canMakeOtherLost,
6484  &currRequest))
6485  {
6486  const VkDeviceSize currRequestCost = currRequest.CalcCost();
6487  if(pBestRequestBlock == VMA_NULL ||
6488  currRequestCost < bestRequestCost)
6489  {
6490  pBestRequestBlock = pCurrBlock;
6491  bestRequest = currRequest;
6492  bestRequestCost = currRequestCost;
6493 
6494  if(bestRequestCost == 0)
6495  {
6496  break;
6497  }
6498  }
6499  }
6500  }
6501 
6502  if(pBestRequestBlock != VMA_NULL)
6503  {
6504  if(mapped)
6505  {
6506  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
6507  if(res != VK_SUCCESS)
6508  {
6509  return res;
6510  }
6511  }
6512 
6513  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
6514  currentFrameIndex,
6515  m_FrameInUseCount,
6516  &bestRequest))
6517  {
6518  // We no longer have an empty Allocation.
6519  if(pBestRequestBlock->m_Metadata.IsEmpty())
6520  {
6521  m_HasEmptyBlock = false;
6522  }
6523  // Allocate from this pBlock.
6524  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6525  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
6526  (*pAllocation)->InitBlockAllocation(
6527  hCurrentPool,
6528  pBestRequestBlock,
6529  bestRequest.offset,
6530  vkMemReq.alignment,
6531  vkMemReq.size,
6532  suballocType,
6533  mapped,
6534  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6535  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
6536  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6537  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6538  return VK_SUCCESS;
6539  }
6540  // else: Some allocations must have been touched while we are here. Next try.
6541  }
6542  else
6543  {
6544  // Could not find place in any of the blocks - break outer loop.
6545  break;
6546  }
6547  }
6548  /* Maximum number of tries exceeded - a very unlike event when many other
6549  threads are simultaneously touching allocations making it impossible to make
6550  lost at the same time as we try to allocate. */
6551  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
6552  {
6553  return VK_ERROR_TOO_MANY_OBJECTS;
6554  }
6555  }
6556 
6557  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6558 }
6559 
6560 void VmaBlockVector::Free(
6561  VmaAllocation hAllocation)
6562 {
6563  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
6564 
6565  // Scope for lock.
6566  {
6567  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6568 
6569  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
6570 
6571  if(hAllocation->IsPersistentMap())
6572  {
6573  pBlock->m_Mapping.Unmap(m_hAllocator, pBlock->m_hMemory, 1);
6574  }
6575 
6576  pBlock->m_Metadata.Free(hAllocation);
6577  VMA_HEAVY_ASSERT(pBlock->Validate());
6578 
6579  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
6580 
6581  // pBlock became empty after this deallocation.
6582  if(pBlock->m_Metadata.IsEmpty())
6583  {
6584  // Already has empty Allocation. We don't want to have two, so delete this one.
6585  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
6586  {
6587  pBlockToDelete = pBlock;
6588  Remove(pBlock);
6589  }
6590  // We now have first empty Allocation.
6591  else
6592  {
6593  m_HasEmptyBlock = true;
6594  }
6595  }
6596  // pBlock didn't become empty, but we have another empty block - find and free that one.
6597  // (This is optional, heuristics.)
6598  else if(m_HasEmptyBlock)
6599  {
6600  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
6601  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
6602  {
6603  pBlockToDelete = pLastBlock;
6604  m_Blocks.pop_back();
6605  m_HasEmptyBlock = false;
6606  }
6607  }
6608 
6609  IncrementallySortBlocks();
6610  }
6611 
6612  // Destruction of a free Allocation. Deferred until this point, outside of mutex
6613  // lock, for performance reason.
6614  if(pBlockToDelete != VMA_NULL)
6615  {
6616  VMA_DEBUG_LOG(" Deleted empty allocation");
6617  pBlockToDelete->Destroy(m_hAllocator);
6618  vma_delete(m_hAllocator, pBlockToDelete);
6619  }
6620 }
6621 
6622 size_t VmaBlockVector::CalcMaxBlockSize() const
6623 {
6624  size_t result = 0;
6625  for(size_t i = m_Blocks.size(); i--; )
6626  {
6627  result = VMA_MAX((uint64_t)result, (uint64_t)m_Blocks[i]->m_Metadata.GetSize());
6628  if(result >= m_PreferredBlockSize)
6629  {
6630  break;
6631  }
6632  }
6633  return result;
6634 }
6635 
6636 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
6637 {
6638  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6639  {
6640  if(m_Blocks[blockIndex] == pBlock)
6641  {
6642  VmaVectorRemove(m_Blocks, blockIndex);
6643  return;
6644  }
6645  }
6646  VMA_ASSERT(0);
6647 }
6648 
6649 void VmaBlockVector::IncrementallySortBlocks()
6650 {
6651  // Bubble sort only until first swap.
6652  for(size_t i = 1; i < m_Blocks.size(); ++i)
6653  {
6654  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
6655  {
6656  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
6657  return;
6658  }
6659  }
6660 }
6661 
6662 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
6663 {
6664  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6665  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
6666  allocInfo.allocationSize = blockSize;
6667  VkDeviceMemory mem = VK_NULL_HANDLE;
6668  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
6669  if(res < 0)
6670  {
6671  return res;
6672  }
6673 
6674  // New VkDeviceMemory successfully created.
6675 
6676  // Create new Allocation for it.
6677  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
6678  pBlock->Init(
6679  m_MemoryTypeIndex,
6680  mem,
6681  allocInfo.allocationSize);
6682 
6683  m_Blocks.push_back(pBlock);
6684  if(pNewBlockIndex != VMA_NULL)
6685  {
6686  *pNewBlockIndex = m_Blocks.size() - 1;
6687  }
6688 
6689  return VK_SUCCESS;
6690 }
6691 
6692 #if VMA_STATS_STRING_ENABLED
6693 
6694 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
6695 {
6696  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6697 
6698  json.BeginObject();
6699 
6700  if(m_IsCustomPool)
6701  {
6702  json.WriteString("MemoryTypeIndex");
6703  json.WriteNumber(m_MemoryTypeIndex);
6704 
6705  json.WriteString("BlockSize");
6706  json.WriteNumber(m_PreferredBlockSize);
6707 
6708  json.WriteString("BlockCount");
6709  json.BeginObject(true);
6710  if(m_MinBlockCount > 0)
6711  {
6712  json.WriteString("Min");
6713  json.WriteNumber((uint64_t)m_MinBlockCount);
6714  }
6715  if(m_MaxBlockCount < SIZE_MAX)
6716  {
6717  json.WriteString("Max");
6718  json.WriteNumber((uint64_t)m_MaxBlockCount);
6719  }
6720  json.WriteString("Cur");
6721  json.WriteNumber((uint64_t)m_Blocks.size());
6722  json.EndObject();
6723 
6724  if(m_FrameInUseCount > 0)
6725  {
6726  json.WriteString("FrameInUseCount");
6727  json.WriteNumber(m_FrameInUseCount);
6728  }
6729  }
6730  else
6731  {
6732  json.WriteString("PreferredBlockSize");
6733  json.WriteNumber(m_PreferredBlockSize);
6734  }
6735 
6736  json.WriteString("Blocks");
6737  json.BeginArray();
6738  for(size_t i = 0; i < m_Blocks.size(); ++i)
6739  {
6740  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
6741  }
6742  json.EndArray();
6743 
6744  json.EndObject();
6745 }
6746 
6747 #endif // #if VMA_STATS_STRING_ENABLED
6748 
6749 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
6750  VmaAllocator hAllocator,
6751  uint32_t currentFrameIndex)
6752 {
6753  if(m_pDefragmentator == VMA_NULL)
6754  {
6755  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
6756  hAllocator,
6757  this,
6758  currentFrameIndex);
6759  }
6760 
6761  return m_pDefragmentator;
6762 }
6763 
6764 VkResult VmaBlockVector::Defragment(
6765  VmaDefragmentationStats* pDefragmentationStats,
6766  VkDeviceSize& maxBytesToMove,
6767  uint32_t& maxAllocationsToMove)
6768 {
6769  if(m_pDefragmentator == VMA_NULL)
6770  {
6771  return VK_SUCCESS;
6772  }
6773 
6774  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6775 
6776  // Defragment.
6777  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
6778 
6779  // Accumulate statistics.
6780  if(pDefragmentationStats != VMA_NULL)
6781  {
6782  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
6783  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
6784  pDefragmentationStats->bytesMoved += bytesMoved;
6785  pDefragmentationStats->allocationsMoved += allocationsMoved;
6786  VMA_ASSERT(bytesMoved <= maxBytesToMove);
6787  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
6788  maxBytesToMove -= bytesMoved;
6789  maxAllocationsToMove -= allocationsMoved;
6790  }
6791 
6792  // Free empty blocks.
6793  m_HasEmptyBlock = false;
6794  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
6795  {
6796  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
6797  if(pBlock->m_Metadata.IsEmpty())
6798  {
6799  if(m_Blocks.size() > m_MinBlockCount)
6800  {
6801  if(pDefragmentationStats != VMA_NULL)
6802  {
6803  ++pDefragmentationStats->deviceMemoryBlocksFreed;
6804  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
6805  }
6806 
6807  VmaVectorRemove(m_Blocks, blockIndex);
6808  pBlock->Destroy(m_hAllocator);
6809  vma_delete(m_hAllocator, pBlock);
6810  }
6811  else
6812  {
6813  m_HasEmptyBlock = true;
6814  }
6815  }
6816  }
6817 
6818  return result;
6819 }
6820 
6821 void VmaBlockVector::DestroyDefragmentator()
6822 {
6823  if(m_pDefragmentator != VMA_NULL)
6824  {
6825  vma_delete(m_hAllocator, m_pDefragmentator);
6826  m_pDefragmentator = VMA_NULL;
6827  }
6828 }
6829 
6830 void VmaBlockVector::MakePoolAllocationsLost(
6831  uint32_t currentFrameIndex,
6832  size_t* pLostAllocationCount)
6833 {
6834  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6835  size_t lostAllocationCount = 0;
6836  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6837  {
6838  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6839  VMA_ASSERT(pBlock);
6840  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
6841  }
6842  if(pLostAllocationCount != VMA_NULL)
6843  {
6844  *pLostAllocationCount = lostAllocationCount;
6845  }
6846 }
6847 
6848 void VmaBlockVector::AddStats(VmaStats* pStats)
6849 {
6850  const uint32_t memTypeIndex = m_MemoryTypeIndex;
6851  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
6852 
6853  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6854 
6855  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6856  {
6857  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6858  VMA_ASSERT(pBlock);
6859  VMA_HEAVY_ASSERT(pBlock->Validate());
6860  VmaStatInfo allocationStatInfo;
6861  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
6862  VmaAddStatInfo(pStats->total, allocationStatInfo);
6863  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
6864  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
6865  }
6866 }
6867 
6869 // VmaDefragmentator members definition
6870 
6871 VmaDefragmentator::VmaDefragmentator(
6872  VmaAllocator hAllocator,
6873  VmaBlockVector* pBlockVector,
6874  uint32_t currentFrameIndex) :
6875  m_hAllocator(hAllocator),
6876  m_pBlockVector(pBlockVector),
6877  m_CurrentFrameIndex(currentFrameIndex),
6878  m_BytesMoved(0),
6879  m_AllocationsMoved(0),
6880  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
6881  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
6882 {
6883 }
6884 
6885 VmaDefragmentator::~VmaDefragmentator()
6886 {
6887  for(size_t i = m_Blocks.size(); i--; )
6888  {
6889  vma_delete(m_hAllocator, m_Blocks[i]);
6890  }
6891 }
6892 
6893 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
6894 {
6895  AllocationInfo allocInfo;
6896  allocInfo.m_hAllocation = hAlloc;
6897  allocInfo.m_pChanged = pChanged;
6898  m_Allocations.push_back(allocInfo);
6899 }
6900 
6901 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
6902 {
6903  // It has already been mapped for defragmentation.
6904  if(m_pMappedDataForDefragmentation)
6905  {
6906  *ppMappedData = m_pMappedDataForDefragmentation;
6907  return VK_SUCCESS;
6908  }
6909 
6910  // It is originally mapped.
6911  if(m_pBlock->m_Mapping.GetMappedData())
6912  {
6913  *ppMappedData = m_pBlock->m_Mapping.GetMappedData();
6914  return VK_SUCCESS;
6915  }
6916 
6917  // Map on first usage.
6918  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
6919  *ppMappedData = m_pMappedDataForDefragmentation;
6920  return res;
6921 }
6922 
6923 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
6924 {
6925  if(m_pMappedDataForDefragmentation != VMA_NULL)
6926  {
6927  m_pBlock->Unmap(hAllocator, 1);
6928  }
6929 }
6930 
6931 VkResult VmaDefragmentator::DefragmentRound(
6932  VkDeviceSize maxBytesToMove,
6933  uint32_t maxAllocationsToMove)
6934 {
6935  if(m_Blocks.empty())
6936  {
6937  return VK_SUCCESS;
6938  }
6939 
6940  size_t srcBlockIndex = m_Blocks.size() - 1;
6941  size_t srcAllocIndex = SIZE_MAX;
6942  for(;;)
6943  {
6944  // 1. Find next allocation to move.
6945  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
6946  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
6947  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
6948  {
6949  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
6950  {
6951  // Finished: no more allocations to process.
6952  if(srcBlockIndex == 0)
6953  {
6954  return VK_SUCCESS;
6955  }
6956  else
6957  {
6958  --srcBlockIndex;
6959  srcAllocIndex = SIZE_MAX;
6960  }
6961  }
6962  else
6963  {
6964  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
6965  }
6966  }
6967 
6968  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
6969  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
6970 
6971  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
6972  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
6973  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
6974  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
6975 
6976  // 2. Try to find new place for this allocation in preceding or current block.
6977  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
6978  {
6979  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
6980  VmaAllocationRequest dstAllocRequest;
6981  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
6982  m_CurrentFrameIndex,
6983  m_pBlockVector->GetFrameInUseCount(),
6984  m_pBlockVector->GetBufferImageGranularity(),
6985  size,
6986  alignment,
6987  suballocType,
6988  false, // canMakeOtherLost
6989  &dstAllocRequest) &&
6990  MoveMakesSense(
6991  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
6992  {
6993  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
6994 
6995  // Reached limit on number of allocations or bytes to move.
6996  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
6997  (m_BytesMoved + size > maxBytesToMove))
6998  {
6999  return VK_INCOMPLETE;
7000  }
7001 
7002  void* pDstMappedData = VMA_NULL;
7003  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
7004  if(res != VK_SUCCESS)
7005  {
7006  return res;
7007  }
7008 
7009  void* pSrcMappedData = VMA_NULL;
7010  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
7011  if(res != VK_SUCCESS)
7012  {
7013  return res;
7014  }
7015 
7016  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
7017  memcpy(
7018  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
7019  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
7020  static_cast<size_t>(size));
7021 
7022  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
7023  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
7024 
7025  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
7026 
7027  if(allocInfo.m_pChanged != VMA_NULL)
7028  {
7029  *allocInfo.m_pChanged = VK_TRUE;
7030  }
7031 
7032  ++m_AllocationsMoved;
7033  m_BytesMoved += size;
7034 
7035  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
7036 
7037  break;
7038  }
7039  }
7040 
7041  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
7042 
7043  if(srcAllocIndex > 0)
7044  {
7045  --srcAllocIndex;
7046  }
7047  else
7048  {
7049  if(srcBlockIndex > 0)
7050  {
7051  --srcBlockIndex;
7052  srcAllocIndex = SIZE_MAX;
7053  }
7054  else
7055  {
7056  return VK_SUCCESS;
7057  }
7058  }
7059  }
7060 }
7061 
7062 VkResult VmaDefragmentator::Defragment(
7063  VkDeviceSize maxBytesToMove,
7064  uint32_t maxAllocationsToMove)
7065 {
7066  if(m_Allocations.empty())
7067  {
7068  return VK_SUCCESS;
7069  }
7070 
7071  // Create block info for each block.
7072  const size_t blockCount = m_pBlockVector->m_Blocks.size();
7073  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
7074  {
7075  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
7076  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
7077  m_Blocks.push_back(pBlockInfo);
7078  }
7079 
7080  // Sort them by m_pBlock pointer value.
7081  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
7082 
7083  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
7084  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
7085  {
7086  AllocationInfo& allocInfo = m_Allocations[blockIndex];
7087  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
7088  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
7089  {
7090  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
7091  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
7092  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
7093  {
7094  (*it)->m_Allocations.push_back(allocInfo);
7095  }
7096  else
7097  {
7098  VMA_ASSERT(0);
7099  }
7100  }
7101  }
7102  m_Allocations.clear();
7103 
7104  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
7105  {
7106  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
7107  pBlockInfo->CalcHasNonMovableAllocations();
7108  pBlockInfo->SortAllocationsBySizeDescecnding();
7109  }
7110 
7111  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
7112  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
7113 
7114  // Execute defragmentation rounds (the main part).
7115  VkResult result = VK_SUCCESS;
7116  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
7117  {
7118  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
7119  }
7120 
7121  // Unmap blocks that were mapped for defragmentation.
7122  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
7123  {
7124  m_Blocks[blockIndex]->Unmap(m_hAllocator);
7125  }
7126 
7127  return result;
7128 }
7129 
7130 bool VmaDefragmentator::MoveMakesSense(
7131  size_t dstBlockIndex, VkDeviceSize dstOffset,
7132  size_t srcBlockIndex, VkDeviceSize srcOffset)
7133 {
7134  if(dstBlockIndex < srcBlockIndex)
7135  {
7136  return true;
7137  }
7138  if(dstBlockIndex > srcBlockIndex)
7139  {
7140  return false;
7141  }
7142  if(dstOffset < srcOffset)
7143  {
7144  return true;
7145  }
7146  return false;
7147 }
7148 
7150 // VmaAllocator_T
7151 
7152 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
7153  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
7154  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
7155  m_hDevice(pCreateInfo->device),
7156  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
7157  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
7158  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
7159  m_PreferredLargeHeapBlockSize(0),
7160  m_PhysicalDevice(pCreateInfo->physicalDevice),
7161  m_CurrentFrameIndex(0),
7162  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
7163 {
7164  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
7165 
7166  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
7167  memset(&m_MemProps, 0, sizeof(m_MemProps));
7168  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
7169 
7170  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
7171  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
7172 
7173  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
7174  {
7175  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
7176  }
7177 
7178  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
7179  {
7180  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
7181  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
7182  }
7183 
7184  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
7185 
7186  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
7187  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
7188 
7189  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
7190  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
7191 
7192  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
7193  {
7194  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
7195  {
7196  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
7197  if(limit != VK_WHOLE_SIZE)
7198  {
7199  m_HeapSizeLimit[heapIndex] = limit;
7200  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
7201  {
7202  m_MemProps.memoryHeaps[heapIndex].size = limit;
7203  }
7204  }
7205  }
7206  }
7207 
7208  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7209  {
7210  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
7211 
7212  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
7213  this,
7214  memTypeIndex,
7215  preferredBlockSize,
7216  0,
7217  SIZE_MAX,
7218  GetBufferImageGranularity(),
7219  pCreateInfo->frameInUseCount,
7220  false); // isCustomPool
7221  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
7222  // becase minBlockCount is 0.
7223  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
7224  }
7225 }
7226 
7227 VmaAllocator_T::~VmaAllocator_T()
7228 {
7229  VMA_ASSERT(m_Pools.empty());
7230 
7231  for(size_t i = GetMemoryTypeCount(); i--; )
7232  {
7233  vma_delete(this, m_pDedicatedAllocations[i]);
7234  vma_delete(this, m_pBlockVectors[i]);
7235  }
7236 }
7237 
7238 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
7239 {
7240 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7241  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
7242  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
7243  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
7244  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
7245  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
7246  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
7247  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
7248  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
7249  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
7250  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
7251  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
7252  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
7253  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
7254  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
7255  if(m_UseKhrDedicatedAllocation)
7256  {
7257  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
7258  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
7259  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
7260  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
7261  }
7262 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7263 
7264 #define VMA_COPY_IF_NOT_NULL(funcName) \
7265  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
7266 
7267  if(pVulkanFunctions != VMA_NULL)
7268  {
7269  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
7270  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
7271  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
7272  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
7273  VMA_COPY_IF_NOT_NULL(vkMapMemory);
7274  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
7275  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
7276  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
7277  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
7278  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
7279  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
7280  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
7281  VMA_COPY_IF_NOT_NULL(vkCreateImage);
7282  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
7283  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
7284  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
7285  }
7286 
7287 #undef VMA_COPY_IF_NOT_NULL
7288 
7289  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
7290  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
7291  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
7292  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
7293  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
7294  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
7295  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
7296  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
7297  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
7298  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
7299  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
7300  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
7301  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
7302  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
7303  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
7304  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
7305  if(m_UseKhrDedicatedAllocation)
7306  {
7307  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
7308  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
7309  }
7310 }
7311 
7312 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
7313 {
7314  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7315  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
7316  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
7317  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
7318 }
7319 
7320 VkResult VmaAllocator_T::AllocateMemoryOfType(
7321  const VkMemoryRequirements& vkMemReq,
7322  bool dedicatedAllocation,
7323  VkBuffer dedicatedBuffer,
7324  VkImage dedicatedImage,
7325  const VmaAllocationCreateInfo& createInfo,
7326  uint32_t memTypeIndex,
7327  VmaSuballocationType suballocType,
7328  VmaAllocation* pAllocation)
7329 {
7330  VMA_ASSERT(pAllocation != VMA_NULL);
7331  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
7332 
7333  VmaAllocationCreateInfo finalCreateInfo = createInfo;
7334 
7335  // If memory type is not HOST_VISIBLE, disable MAPPED.
7336  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7337  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
7338  {
7339  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
7340  }
7341 
7342  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
7343  VMA_ASSERT(blockVector);
7344 
7345  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
7346  bool preferDedicatedMemory =
7347  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
7348  dedicatedAllocation ||
7349  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
7350  vkMemReq.size > preferredBlockSize / 2;
7351 
7352  if(preferDedicatedMemory &&
7353  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
7354  finalCreateInfo.pool == VK_NULL_HANDLE)
7355  {
7356  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
7357  }
7358 
7359  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
7360  {
7361  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7362  {
7363  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7364  }
7365  else
7366  {
7367  return AllocateDedicatedMemory(
7368  vkMemReq.size,
7369  suballocType,
7370  memTypeIndex,
7371  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7372  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7373  finalCreateInfo.pUserData,
7374  dedicatedBuffer,
7375  dedicatedImage,
7376  pAllocation);
7377  }
7378  }
7379  else
7380  {
7381  VkResult res = blockVector->Allocate(
7382  VK_NULL_HANDLE, // hCurrentPool
7383  m_CurrentFrameIndex.load(),
7384  vkMemReq,
7385  finalCreateInfo,
7386  suballocType,
7387  pAllocation);
7388  if(res == VK_SUCCESS)
7389  {
7390  return res;
7391  }
7392 
7393  // 5. Try dedicated memory.
7394  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7395  {
7396  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7397  }
7398  else
7399  {
7400  res = AllocateDedicatedMemory(
7401  vkMemReq.size,
7402  suballocType,
7403  memTypeIndex,
7404  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7405  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7406  finalCreateInfo.pUserData,
7407  dedicatedBuffer,
7408  dedicatedImage,
7409  pAllocation);
7410  if(res == VK_SUCCESS)
7411  {
7412  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
7413  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
7414  return VK_SUCCESS;
7415  }
7416  else
7417  {
7418  // Everything failed: Return error code.
7419  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7420  return res;
7421  }
7422  }
7423  }
7424 }
7425 
7426 VkResult VmaAllocator_T::AllocateDedicatedMemory(
7427  VkDeviceSize size,
7428  VmaSuballocationType suballocType,
7429  uint32_t memTypeIndex,
7430  bool map,
7431  bool isUserDataString,
7432  void* pUserData,
7433  VkBuffer dedicatedBuffer,
7434  VkImage dedicatedImage,
7435  VmaAllocation* pAllocation)
7436 {
7437  VMA_ASSERT(pAllocation);
7438 
7439  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
7440  allocInfo.memoryTypeIndex = memTypeIndex;
7441  allocInfo.allocationSize = size;
7442 
7443  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
7444  if(m_UseKhrDedicatedAllocation)
7445  {
7446  if(dedicatedBuffer != VK_NULL_HANDLE)
7447  {
7448  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
7449  dedicatedAllocInfo.buffer = dedicatedBuffer;
7450  allocInfo.pNext = &dedicatedAllocInfo;
7451  }
7452  else if(dedicatedImage != VK_NULL_HANDLE)
7453  {
7454  dedicatedAllocInfo.image = dedicatedImage;
7455  allocInfo.pNext = &dedicatedAllocInfo;
7456  }
7457  }
7458 
7459  // Allocate VkDeviceMemory.
7460  VkDeviceMemory hMemory = VK_NULL_HANDLE;
7461  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
7462  if(res < 0)
7463  {
7464  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7465  return res;
7466  }
7467 
7468  void* pMappedData = VMA_NULL;
7469  if(map)
7470  {
7471  res = (*m_VulkanFunctions.vkMapMemory)(
7472  m_hDevice,
7473  hMemory,
7474  0,
7475  VK_WHOLE_SIZE,
7476  0,
7477  &pMappedData);
7478  if(res < 0)
7479  {
7480  VMA_DEBUG_LOG(" vkMapMemory FAILED");
7481  FreeVulkanMemory(memTypeIndex, size, hMemory);
7482  return res;
7483  }
7484  }
7485 
7486  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
7487  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
7488  (*pAllocation)->SetUserData(this, pUserData);
7489 
7490  // Register it in m_pDedicatedAllocations.
7491  {
7492  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7493  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7494  VMA_ASSERT(pDedicatedAllocations);
7495  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
7496  }
7497 
7498  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
7499 
7500  return VK_SUCCESS;
7501 }
7502 
7503 void VmaAllocator_T::GetBufferMemoryRequirements(
7504  VkBuffer hBuffer,
7505  VkMemoryRequirements& memReq,
7506  bool& requiresDedicatedAllocation,
7507  bool& prefersDedicatedAllocation) const
7508 {
7509  if(m_UseKhrDedicatedAllocation)
7510  {
7511  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
7512  memReqInfo.buffer = hBuffer;
7513 
7514  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7515 
7516  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7517  memReq2.pNext = &memDedicatedReq;
7518 
7519  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7520 
7521  memReq = memReq2.memoryRequirements;
7522  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7523  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7524  }
7525  else
7526  {
7527  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
7528  requiresDedicatedAllocation = false;
7529  prefersDedicatedAllocation = false;
7530  }
7531 }
7532 
7533 void VmaAllocator_T::GetImageMemoryRequirements(
7534  VkImage hImage,
7535  VkMemoryRequirements& memReq,
7536  bool& requiresDedicatedAllocation,
7537  bool& prefersDedicatedAllocation) const
7538 {
7539  if(m_UseKhrDedicatedAllocation)
7540  {
7541  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
7542  memReqInfo.image = hImage;
7543 
7544  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7545 
7546  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7547  memReq2.pNext = &memDedicatedReq;
7548 
7549  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7550 
7551  memReq = memReq2.memoryRequirements;
7552  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7553  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7554  }
7555  else
7556  {
7557  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
7558  requiresDedicatedAllocation = false;
7559  prefersDedicatedAllocation = false;
7560  }
7561 }
7562 
7563 VkResult VmaAllocator_T::AllocateMemory(
7564  const VkMemoryRequirements& vkMemReq,
7565  bool requiresDedicatedAllocation,
7566  bool prefersDedicatedAllocation,
7567  VkBuffer dedicatedBuffer,
7568  VkImage dedicatedImage,
7569  const VmaAllocationCreateInfo& createInfo,
7570  VmaSuballocationType suballocType,
7571  VmaAllocation* pAllocation)
7572 {
7573  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
7574  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7575  {
7576  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
7577  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7578  }
7579  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7580  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
7581  {
7582  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
7583  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7584  }
7585  if(requiresDedicatedAllocation)
7586  {
7587  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7588  {
7589  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
7590  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7591  }
7592  if(createInfo.pool != VK_NULL_HANDLE)
7593  {
7594  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
7595  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7596  }
7597  }
7598  if((createInfo.pool != VK_NULL_HANDLE) &&
7599  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
7600  {
7601  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
7602  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7603  }
7604 
7605  if(createInfo.pool != VK_NULL_HANDLE)
7606  {
7607  return createInfo.pool->m_BlockVector.Allocate(
7608  createInfo.pool,
7609  m_CurrentFrameIndex.load(),
7610  vkMemReq,
7611  createInfo,
7612  suballocType,
7613  pAllocation);
7614  }
7615  else
7616  {
7617  // Bit mask of memory Vulkan types acceptable for this allocation.
7618  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
7619  uint32_t memTypeIndex = UINT32_MAX;
7620  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7621  if(res == VK_SUCCESS)
7622  {
7623  res = AllocateMemoryOfType(
7624  vkMemReq,
7625  requiresDedicatedAllocation || prefersDedicatedAllocation,
7626  dedicatedBuffer,
7627  dedicatedImage,
7628  createInfo,
7629  memTypeIndex,
7630  suballocType,
7631  pAllocation);
7632  // Succeeded on first try.
7633  if(res == VK_SUCCESS)
7634  {
7635  return res;
7636  }
7637  // Allocation from this memory type failed. Try other compatible memory types.
7638  else
7639  {
7640  for(;;)
7641  {
7642  // Remove old memTypeIndex from list of possibilities.
7643  memoryTypeBits &= ~(1u << memTypeIndex);
7644  // Find alternative memTypeIndex.
7645  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7646  if(res == VK_SUCCESS)
7647  {
7648  res = AllocateMemoryOfType(
7649  vkMemReq,
7650  requiresDedicatedAllocation || prefersDedicatedAllocation,
7651  dedicatedBuffer,
7652  dedicatedImage,
7653  createInfo,
7654  memTypeIndex,
7655  suballocType,
7656  pAllocation);
7657  // Allocation from this alternative memory type succeeded.
7658  if(res == VK_SUCCESS)
7659  {
7660  return res;
7661  }
7662  // else: Allocation from this memory type failed. Try next one - next loop iteration.
7663  }
7664  // No other matching memory type index could be found.
7665  else
7666  {
7667  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
7668  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7669  }
7670  }
7671  }
7672  }
7673  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
7674  else
7675  return res;
7676  }
7677 }
7678 
7679 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
7680 {
7681  VMA_ASSERT(allocation);
7682 
7683  if(allocation->CanBecomeLost() == false ||
7684  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
7685  {
7686  switch(allocation->GetType())
7687  {
7688  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7689  {
7690  VmaBlockVector* pBlockVector = VMA_NULL;
7691  VmaPool hPool = allocation->GetPool();
7692  if(hPool != VK_NULL_HANDLE)
7693  {
7694  pBlockVector = &hPool->m_BlockVector;
7695  }
7696  else
7697  {
7698  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7699  pBlockVector = m_pBlockVectors[memTypeIndex];
7700  }
7701  pBlockVector->Free(allocation);
7702  }
7703  break;
7704  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7705  FreeDedicatedMemory(allocation);
7706  break;
7707  default:
7708  VMA_ASSERT(0);
7709  }
7710  }
7711 
7712  allocation->SetUserData(this, VMA_NULL);
7713  vma_delete(this, allocation);
7714 }
7715 
7716 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
7717 {
7718  // Initialize.
7719  InitStatInfo(pStats->total);
7720  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
7721  InitStatInfo(pStats->memoryType[i]);
7722  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
7723  InitStatInfo(pStats->memoryHeap[i]);
7724 
7725  // Process default pools.
7726  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7727  {
7728  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
7729  VMA_ASSERT(pBlockVector);
7730  pBlockVector->AddStats(pStats);
7731  }
7732 
7733  // Process custom pools.
7734  {
7735  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7736  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
7737  {
7738  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
7739  }
7740  }
7741 
7742  // Process dedicated allocations.
7743  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7744  {
7745  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7746  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7747  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7748  VMA_ASSERT(pDedicatedAllocVector);
7749  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
7750  {
7751  VmaStatInfo allocationStatInfo;
7752  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
7753  VmaAddStatInfo(pStats->total, allocationStatInfo);
7754  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
7755  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
7756  }
7757  }
7758 
7759  // Postprocess.
7760  VmaPostprocessCalcStatInfo(pStats->total);
7761  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
7762  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
7763  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
7764  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
7765 }
7766 
7767 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
7768 
7769 VkResult VmaAllocator_T::Defragment(
7770  VmaAllocation* pAllocations,
7771  size_t allocationCount,
7772  VkBool32* pAllocationsChanged,
7773  const VmaDefragmentationInfo* pDefragmentationInfo,
7774  VmaDefragmentationStats* pDefragmentationStats)
7775 {
7776  if(pAllocationsChanged != VMA_NULL)
7777  {
7778  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
7779  }
7780  if(pDefragmentationStats != VMA_NULL)
7781  {
7782  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
7783  }
7784 
7785  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
7786 
7787  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
7788 
7789  const size_t poolCount = m_Pools.size();
7790 
7791  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
7792  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
7793  {
7794  VmaAllocation hAlloc = pAllocations[allocIndex];
7795  VMA_ASSERT(hAlloc);
7796  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
7797  // DedicatedAlloc cannot be defragmented.
7798  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
7799  // Only HOST_VISIBLE memory types can be defragmented.
7800  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
7801  // Lost allocation cannot be defragmented.
7802  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
7803  {
7804  VmaBlockVector* pAllocBlockVector = VMA_NULL;
7805 
7806  const VmaPool hAllocPool = hAlloc->GetPool();
7807  // This allocation belongs to custom pool.
7808  if(hAllocPool != VK_NULL_HANDLE)
7809  {
7810  pAllocBlockVector = &hAllocPool->GetBlockVector();
7811  }
7812  // This allocation belongs to general pool.
7813  else
7814  {
7815  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
7816  }
7817 
7818  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
7819 
7820  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
7821  &pAllocationsChanged[allocIndex] : VMA_NULL;
7822  pDefragmentator->AddAllocation(hAlloc, pChanged);
7823  }
7824  }
7825 
7826  VkResult result = VK_SUCCESS;
7827 
7828  // ======== Main processing.
7829 
7830  VkDeviceSize maxBytesToMove = SIZE_MAX;
7831  uint32_t maxAllocationsToMove = UINT32_MAX;
7832  if(pDefragmentationInfo != VMA_NULL)
7833  {
7834  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
7835  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
7836  }
7837 
7838  // Process standard memory.
7839  for(uint32_t memTypeIndex = 0;
7840  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
7841  ++memTypeIndex)
7842  {
7843  // Only HOST_VISIBLE memory types can be defragmented.
7844  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7845  {
7846  result = m_pBlockVectors[memTypeIndex]->Defragment(
7847  pDefragmentationStats,
7848  maxBytesToMove,
7849  maxAllocationsToMove);
7850  }
7851  }
7852 
7853  // Process custom pools.
7854  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
7855  {
7856  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
7857  pDefragmentationStats,
7858  maxBytesToMove,
7859  maxAllocationsToMove);
7860  }
7861 
7862  // ======== Destroy defragmentators.
7863 
7864  // Process custom pools.
7865  for(size_t poolIndex = poolCount; poolIndex--; )
7866  {
7867  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
7868  }
7869 
7870  // Process standard memory.
7871  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
7872  {
7873  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7874  {
7875  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
7876  }
7877  }
7878 
7879  return result;
7880 }
7881 
7882 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
7883 {
7884  if(hAllocation->CanBecomeLost())
7885  {
7886  /*
7887  Warning: This is a carefully designed algorithm.
7888  Do not modify unless you really know what you're doing :)
7889  */
7890  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
7891  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
7892  for(;;)
7893  {
7894  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
7895  {
7896  pAllocationInfo->memoryType = UINT32_MAX;
7897  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
7898  pAllocationInfo->offset = 0;
7899  pAllocationInfo->size = hAllocation->GetSize();
7900  pAllocationInfo->pMappedData = VMA_NULL;
7901  pAllocationInfo->pUserData = hAllocation->GetUserData();
7902  return;
7903  }
7904  else if(localLastUseFrameIndex == localCurrFrameIndex)
7905  {
7906  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7907  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7908  pAllocationInfo->offset = hAllocation->GetOffset();
7909  pAllocationInfo->size = hAllocation->GetSize();
7910  pAllocationInfo->pMappedData = VMA_NULL;
7911  pAllocationInfo->pUserData = hAllocation->GetUserData();
7912  return;
7913  }
7914  else // Last use time earlier than current time.
7915  {
7916  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
7917  {
7918  localLastUseFrameIndex = localCurrFrameIndex;
7919  }
7920  }
7921  }
7922  }
7923  else
7924  {
7925  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7926  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7927  pAllocationInfo->offset = hAllocation->GetOffset();
7928  pAllocationInfo->size = hAllocation->GetSize();
7929  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
7930  pAllocationInfo->pUserData = hAllocation->GetUserData();
7931  }
7932 }
7933 
7934 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
7935 {
7936  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
7937  if(hAllocation->CanBecomeLost())
7938  {
7939  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
7940  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
7941  for(;;)
7942  {
7943  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
7944  {
7945  return false;
7946  }
7947  else if(localLastUseFrameIndex == localCurrFrameIndex)
7948  {
7949  return true;
7950  }
7951  else // Last use time earlier than current time.
7952  {
7953  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
7954  {
7955  localLastUseFrameIndex = localCurrFrameIndex;
7956  }
7957  }
7958  }
7959  }
7960  else
7961  {
7962  return true;
7963  }
7964 }
7965 
7966 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
7967 {
7968  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
7969 
7970  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
7971 
7972  if(newCreateInfo.maxBlockCount == 0)
7973  {
7974  newCreateInfo.maxBlockCount = SIZE_MAX;
7975  }
7976  if(newCreateInfo.blockSize == 0)
7977  {
7978  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
7979  }
7980 
7981  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
7982 
7983  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
7984  if(res != VK_SUCCESS)
7985  {
7986  vma_delete(this, *pPool);
7987  *pPool = VMA_NULL;
7988  return res;
7989  }
7990 
7991  // Add to m_Pools.
7992  {
7993  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7994  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
7995  }
7996 
7997  return VK_SUCCESS;
7998 }
7999 
8000 void VmaAllocator_T::DestroyPool(VmaPool pool)
8001 {
8002  // Remove from m_Pools.
8003  {
8004  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
8005  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
8006  VMA_ASSERT(success && "Pool not found in Allocator.");
8007  }
8008 
8009  vma_delete(this, pool);
8010 }
8011 
8012 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
8013 {
8014  pool->m_BlockVector.GetPoolStats(pPoolStats);
8015 }
8016 
8017 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
8018 {
8019  m_CurrentFrameIndex.store(frameIndex);
8020 }
8021 
8022 void VmaAllocator_T::MakePoolAllocationsLost(
8023  VmaPool hPool,
8024  size_t* pLostAllocationCount)
8025 {
8026  hPool->m_BlockVector.MakePoolAllocationsLost(
8027  m_CurrentFrameIndex.load(),
8028  pLostAllocationCount);
8029 }
8030 
8031 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
8032 {
8033  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
8034  (*pAllocation)->InitLost();
8035 }
8036 
8037 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
8038 {
8039  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
8040 
8041  VkResult res;
8042  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
8043  {
8044  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
8045  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
8046  {
8047  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
8048  if(res == VK_SUCCESS)
8049  {
8050  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
8051  }
8052  }
8053  else
8054  {
8055  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
8056  }
8057  }
8058  else
8059  {
8060  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
8061  }
8062 
8063  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
8064  {
8065  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
8066  }
8067 
8068  return res;
8069 }
8070 
8071 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
8072 {
8073  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
8074  {
8075  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
8076  }
8077 
8078  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
8079 
8080  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
8081  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
8082  {
8083  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
8084  m_HeapSizeLimit[heapIndex] += size;
8085  }
8086 }
8087 
8088 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
8089 {
8090  if(hAllocation->CanBecomeLost())
8091  {
8092  return VK_ERROR_MEMORY_MAP_FAILED;
8093  }
8094 
8095  switch(hAllocation->GetType())
8096  {
8097  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
8098  {
8099  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
8100  char *pBytes = VMA_NULL;
8101  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
8102  if(res == VK_SUCCESS)
8103  {
8104  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
8105  hAllocation->BlockAllocMap();
8106  }
8107  return res;
8108  }
8109  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
8110  return hAllocation->DedicatedAllocMap(this, ppData);
8111  default:
8112  VMA_ASSERT(0);
8113  return VK_ERROR_MEMORY_MAP_FAILED;
8114  }
8115 }
8116 
8117 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
8118 {
8119  switch(hAllocation->GetType())
8120  {
8121  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
8122  {
8123  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
8124  hAllocation->BlockAllocUnmap();
8125  pBlock->Unmap(this, 1);
8126  }
8127  break;
8128  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
8129  hAllocation->DedicatedAllocUnmap(this);
8130  break;
8131  default:
8132  VMA_ASSERT(0);
8133  }
8134 }
8135 
8136 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
8137 {
8138  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
8139 
8140  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
8141  {
8142  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
8143  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
8144  VMA_ASSERT(pDedicatedAllocations);
8145  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
8146  VMA_ASSERT(success);
8147  }
8148 
8149  VkDeviceMemory hMemory = allocation->GetMemory();
8150 
8151  if(allocation->GetMappedData() != VMA_NULL)
8152  {
8153  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
8154  }
8155 
8156  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
8157 
8158  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
8159 }
8160 
8161 #if VMA_STATS_STRING_ENABLED
8162 
8163 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
8164 {
8165  bool dedicatedAllocationsStarted = false;
8166  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8167  {
8168  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
8169  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
8170  VMA_ASSERT(pDedicatedAllocVector);
8171  if(pDedicatedAllocVector->empty() == false)
8172  {
8173  if(dedicatedAllocationsStarted == false)
8174  {
8175  dedicatedAllocationsStarted = true;
8176  json.WriteString("DedicatedAllocations");
8177  json.BeginObject();
8178  }
8179 
8180  json.BeginString("Type ");
8181  json.ContinueString(memTypeIndex);
8182  json.EndString();
8183 
8184  json.BeginArray();
8185 
8186  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
8187  {
8188  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
8189  json.BeginObject(true);
8190 
8191  json.WriteString("Type");
8192  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
8193 
8194  json.WriteString("Size");
8195  json.WriteNumber(hAlloc->GetSize());
8196 
8197  const void* pUserData = hAlloc->GetUserData();
8198  if(pUserData != VMA_NULL)
8199  {
8200  json.WriteString("UserData");
8201  if(hAlloc->IsUserDataString())
8202  {
8203  json.WriteString((const char*)pUserData);
8204  }
8205  else
8206  {
8207  json.BeginString();
8208  json.ContinueString_Pointer(pUserData);
8209  json.EndString();
8210  }
8211  }
8212 
8213  json.EndObject();
8214  }
8215 
8216  json.EndArray();
8217  }
8218  }
8219  if(dedicatedAllocationsStarted)
8220  {
8221  json.EndObject();
8222  }
8223 
8224  {
8225  bool allocationsStarted = false;
8226  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8227  {
8228  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
8229  {
8230  if(allocationsStarted == false)
8231  {
8232  allocationsStarted = true;
8233  json.WriteString("DefaultPools");
8234  json.BeginObject();
8235  }
8236 
8237  json.BeginString("Type ");
8238  json.ContinueString(memTypeIndex);
8239  json.EndString();
8240 
8241  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
8242  }
8243  }
8244  if(allocationsStarted)
8245  {
8246  json.EndObject();
8247  }
8248  }
8249 
8250  {
8251  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
8252  const size_t poolCount = m_Pools.size();
8253  if(poolCount > 0)
8254  {
8255  json.WriteString("Pools");
8256  json.BeginArray();
8257  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
8258  {
8259  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
8260  }
8261  json.EndArray();
8262  }
8263  }
8264 }
8265 
8266 #endif // #if VMA_STATS_STRING_ENABLED
8267 
8268 static VkResult AllocateMemoryForImage(
8269  VmaAllocator allocator,
8270  VkImage image,
8271  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8272  VmaSuballocationType suballocType,
8273  VmaAllocation* pAllocation)
8274 {
8275  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
8276 
8277  VkMemoryRequirements vkMemReq = {};
8278  bool requiresDedicatedAllocation = false;
8279  bool prefersDedicatedAllocation = false;
8280  allocator->GetImageMemoryRequirements(image, vkMemReq,
8281  requiresDedicatedAllocation, prefersDedicatedAllocation);
8282 
8283  return allocator->AllocateMemory(
8284  vkMemReq,
8285  requiresDedicatedAllocation,
8286  prefersDedicatedAllocation,
8287  VK_NULL_HANDLE, // dedicatedBuffer
8288  image, // dedicatedImage
8289  *pAllocationCreateInfo,
8290  suballocType,
8291  pAllocation);
8292 }
8293 
8295 // Public interface
8296 
8297 VkResult vmaCreateAllocator(
8298  const VmaAllocatorCreateInfo* pCreateInfo,
8299  VmaAllocator* pAllocator)
8300 {
8301  VMA_ASSERT(pCreateInfo && pAllocator);
8302  VMA_DEBUG_LOG("vmaCreateAllocator");
8303  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
8304  return VK_SUCCESS;
8305 }
8306 
8307 void vmaDestroyAllocator(
8308  VmaAllocator allocator)
8309 {
8310  if(allocator != VK_NULL_HANDLE)
8311  {
8312  VMA_DEBUG_LOG("vmaDestroyAllocator");
8313  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
8314  vma_delete(&allocationCallbacks, allocator);
8315  }
8316 }
8317 
8318 void vmaGetPhysicalDeviceProperties(
8319  VmaAllocator allocator,
8320  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
8321 {
8322  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
8323  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
8324 }
8325 
8326 void vmaGetMemoryProperties(
8327  VmaAllocator allocator,
8328  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
8329 {
8330  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
8331  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
8332 }
8333 
8334 void vmaGetMemoryTypeProperties(
8335  VmaAllocator allocator,
8336  uint32_t memoryTypeIndex,
8337  VkMemoryPropertyFlags* pFlags)
8338 {
8339  VMA_ASSERT(allocator && pFlags);
8340  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
8341  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
8342 }
8343 
8344 void vmaSetCurrentFrameIndex(
8345  VmaAllocator allocator,
8346  uint32_t frameIndex)
8347 {
8348  VMA_ASSERT(allocator);
8349  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
8350 
8351  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8352 
8353  allocator->SetCurrentFrameIndex(frameIndex);
8354 }
8355 
8356 void vmaCalculateStats(
8357  VmaAllocator allocator,
8358  VmaStats* pStats)
8359 {
8360  VMA_ASSERT(allocator && pStats);
8361  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8362  allocator->CalculateStats(pStats);
8363 }
8364 
8365 #if VMA_STATS_STRING_ENABLED
8366 
8367 void vmaBuildStatsString(
8368  VmaAllocator allocator,
8369  char** ppStatsString,
8370  VkBool32 detailedMap)
8371 {
8372  VMA_ASSERT(allocator && ppStatsString);
8373  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8374 
8375  VmaStringBuilder sb(allocator);
8376  {
8377  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
8378  json.BeginObject();
8379 
8380  VmaStats stats;
8381  allocator->CalculateStats(&stats);
8382 
8383  json.WriteString("Total");
8384  VmaPrintStatInfo(json, stats.total);
8385 
8386  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
8387  {
8388  json.BeginString("Heap ");
8389  json.ContinueString(heapIndex);
8390  json.EndString();
8391  json.BeginObject();
8392 
8393  json.WriteString("Size");
8394  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
8395 
8396  json.WriteString("Flags");
8397  json.BeginArray(true);
8398  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
8399  {
8400  json.WriteString("DEVICE_LOCAL");
8401  }
8402  json.EndArray();
8403 
8404  if(stats.memoryHeap[heapIndex].blockCount > 0)
8405  {
8406  json.WriteString("Stats");
8407  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
8408  }
8409 
8410  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
8411  {
8412  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
8413  {
8414  json.BeginString("Type ");
8415  json.ContinueString(typeIndex);
8416  json.EndString();
8417 
8418  json.BeginObject();
8419 
8420  json.WriteString("Flags");
8421  json.BeginArray(true);
8422  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
8423  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
8424  {
8425  json.WriteString("DEVICE_LOCAL");
8426  }
8427  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
8428  {
8429  json.WriteString("HOST_VISIBLE");
8430  }
8431  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
8432  {
8433  json.WriteString("HOST_COHERENT");
8434  }
8435  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
8436  {
8437  json.WriteString("HOST_CACHED");
8438  }
8439  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
8440  {
8441  json.WriteString("LAZILY_ALLOCATED");
8442  }
8443  json.EndArray();
8444 
8445  if(stats.memoryType[typeIndex].blockCount > 0)
8446  {
8447  json.WriteString("Stats");
8448  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
8449  }
8450 
8451  json.EndObject();
8452  }
8453  }
8454 
8455  json.EndObject();
8456  }
8457  if(detailedMap == VK_TRUE)
8458  {
8459  allocator->PrintDetailedMap(json);
8460  }
8461 
8462  json.EndObject();
8463  }
8464 
8465  const size_t len = sb.GetLength();
8466  char* const pChars = vma_new_array(allocator, char, len + 1);
8467  if(len > 0)
8468  {
8469  memcpy(pChars, sb.GetData(), len);
8470  }
8471  pChars[len] = '\0';
8472  *ppStatsString = pChars;
8473 }
8474 
8475 void vmaFreeStatsString(
8476  VmaAllocator allocator,
8477  char* pStatsString)
8478 {
8479  if(pStatsString != VMA_NULL)
8480  {
8481  VMA_ASSERT(allocator);
8482  size_t len = strlen(pStatsString);
8483  vma_delete_array(allocator, pStatsString, len + 1);
8484  }
8485 }
8486 
8487 #endif // #if VMA_STATS_STRING_ENABLED
8488 
8489 /*
8490 This function is not protected by any mutex because it just reads immutable data.
8491 */
8492 VkResult vmaFindMemoryTypeIndex(
8493  VmaAllocator allocator,
8494  uint32_t memoryTypeBits,
8495  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8496  uint32_t* pMemoryTypeIndex)
8497 {
8498  VMA_ASSERT(allocator != VK_NULL_HANDLE);
8499  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8500  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8501 
8502  if(pAllocationCreateInfo->memoryTypeBits != 0)
8503  {
8504  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
8505  }
8506 
8507  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
8508  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
8509 
8510  // Convert usage to requiredFlags and preferredFlags.
8511  switch(pAllocationCreateInfo->usage)
8512  {
8513  case VMA_MEMORY_USAGE_UNKNOWN:
8514  break;
8515  case VMA_MEMORY_USAGE_GPU_ONLY:
8516  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8517  break;
8518  case VMA_MEMORY_USAGE_CPU_ONLY:
8519  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
8520  break;
8521  case VMA_MEMORY_USAGE_CPU_TO_GPU:
8522  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8523  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8524  break;
8525  case VMA_MEMORY_USAGE_GPU_TO_CPU:
8526  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8527  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
8528  break;
8529  default:
8530  break;
8531  }
8532 
8533  *pMemoryTypeIndex = UINT32_MAX;
8534  uint32_t minCost = UINT32_MAX;
8535  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
8536  memTypeIndex < allocator->GetMemoryTypeCount();
8537  ++memTypeIndex, memTypeBit <<= 1)
8538  {
8539  // This memory type is acceptable according to memoryTypeBits bitmask.
8540  if((memTypeBit & memoryTypeBits) != 0)
8541  {
8542  const VkMemoryPropertyFlags currFlags =
8543  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
8544  // This memory type contains requiredFlags.
8545  if((requiredFlags & ~currFlags) == 0)
8546  {
8547  // Calculate cost as number of bits from preferredFlags not present in this memory type.
8548  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
8549  // Remember memory type with lowest cost.
8550  if(currCost < minCost)
8551  {
8552  *pMemoryTypeIndex = memTypeIndex;
8553  if(currCost == 0)
8554  {
8555  return VK_SUCCESS;
8556  }
8557  minCost = currCost;
8558  }
8559  }
8560  }
8561  }
8562  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
8563 }
8564 
8565 VkResult vmaFindMemoryTypeIndexForBufferInfo(
8566  VmaAllocator allocator,
8567  const VkBufferCreateInfo* pBufferCreateInfo,
8568  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8569  uint32_t* pMemoryTypeIndex)
8570 {
8571  VMA_ASSERT(allocator != VK_NULL_HANDLE);
8572  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
8573  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8574  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8575 
8576  const VkDevice hDev = allocator->m_hDevice;
8577  VkBuffer hBuffer = VK_NULL_HANDLE;
8578  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
8579  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
8580  if(res == VK_SUCCESS)
8581  {
8582  VkMemoryRequirements memReq = {};
8583  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
8584  hDev, hBuffer, &memReq);
8585 
8586  res = vmaFindMemoryTypeIndex(
8587  allocator,
8588  memReq.memoryTypeBits,
8589  pAllocationCreateInfo,
8590  pMemoryTypeIndex);
8591 
8592  allocator->GetVulkanFunctions().vkDestroyBuffer(
8593  hDev, hBuffer, allocator->GetAllocationCallbacks());
8594  }
8595  return res;
8596 }
8597 
8598 VkResult vmaFindMemoryTypeIndexForImageInfo(
8599  VmaAllocator allocator,
8600  const VkImageCreateInfo* pImageCreateInfo,
8601  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8602  uint32_t* pMemoryTypeIndex)
8603 {
8604  VMA_ASSERT(allocator != VK_NULL_HANDLE);
8605  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
8606  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8607  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8608 
8609  const VkDevice hDev = allocator->m_hDevice;
8610  VkImage hImage = VK_NULL_HANDLE;
8611  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
8612  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
8613  if(res == VK_SUCCESS)
8614  {
8615  VkMemoryRequirements memReq = {};
8616  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
8617  hDev, hImage, &memReq);
8618 
8619  res = vmaFindMemoryTypeIndex(
8620  allocator,
8621  memReq.memoryTypeBits,
8622  pAllocationCreateInfo,
8623  pMemoryTypeIndex);
8624 
8625  allocator->GetVulkanFunctions().vkDestroyImage(
8626  hDev, hImage, allocator->GetAllocationCallbacks());
8627  }
8628  return res;
8629 }
8630 
8631 VkResult vmaCreatePool(
8632  VmaAllocator allocator,
8633  const VmaPoolCreateInfo* pCreateInfo,
8634  VmaPool* pPool)
8635 {
8636  VMA_ASSERT(allocator && pCreateInfo && pPool);
8637 
8638  VMA_DEBUG_LOG("vmaCreatePool");
8639 
8640  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8641 
8642  return allocator->CreatePool(pCreateInfo, pPool);
8643 }
8644 
8645 void vmaDestroyPool(
8646  VmaAllocator allocator,
8647  VmaPool pool)
8648 {
8649  VMA_ASSERT(allocator);
8650 
8651  if(pool == VK_NULL_HANDLE)
8652  {
8653  return;
8654  }
8655 
8656  VMA_DEBUG_LOG("vmaDestroyPool");
8657 
8658  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8659 
8660  allocator->DestroyPool(pool);
8661 }
8662 
8663 void vmaGetPoolStats(
8664  VmaAllocator allocator,
8665  VmaPool pool,
8666  VmaPoolStats* pPoolStats)
8667 {
8668  VMA_ASSERT(allocator && pool && pPoolStats);
8669 
8670  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8671 
8672  allocator->GetPoolStats(pool, pPoolStats);
8673 }
8674 
8675 void vmaMakePoolAllocationsLost(
8676  VmaAllocator allocator,
8677  VmaPool pool,
8678  size_t* pLostAllocationCount)
8679 {
8680  VMA_ASSERT(allocator && pool);
8681 
8682  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8683 
8684  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
8685 }
8686 
8687 VkResult vmaAllocateMemory(
8688  VmaAllocator allocator,
8689  const VkMemoryRequirements* pVkMemoryRequirements,
8690  const VmaAllocationCreateInfo* pCreateInfo,
8691  VmaAllocation* pAllocation,
8692  VmaAllocationInfo* pAllocationInfo)
8693 {
8694  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
8695 
8696  VMA_DEBUG_LOG("vmaAllocateMemory");
8697 
8698  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8699 
8700  VkResult result = allocator->AllocateMemory(
8701  *pVkMemoryRequirements,
8702  false, // requiresDedicatedAllocation
8703  false, // prefersDedicatedAllocation
8704  VK_NULL_HANDLE, // dedicatedBuffer
8705  VK_NULL_HANDLE, // dedicatedImage
8706  *pCreateInfo,
8707  VMA_SUBALLOCATION_TYPE_UNKNOWN,
8708  pAllocation);
8709 
8710  if(pAllocationInfo && result == VK_SUCCESS)
8711  {
8712  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8713  }
8714 
8715  return result;
8716 }
8717 
8718 VkResult vmaAllocateMemoryForBuffer(
8719  VmaAllocator allocator,
8720  VkBuffer buffer,
8721  const VmaAllocationCreateInfo* pCreateInfo,
8722  VmaAllocation* pAllocation,
8723  VmaAllocationInfo* pAllocationInfo)
8724 {
8725  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8726 
8727  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
8728 
8729  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8730 
8731  VkMemoryRequirements vkMemReq = {};
8732  bool requiresDedicatedAllocation = false;
8733  bool prefersDedicatedAllocation = false;
8734  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
8735  requiresDedicatedAllocation,
8736  prefersDedicatedAllocation);
8737 
8738  VkResult result = allocator->AllocateMemory(
8739  vkMemReq,
8740  requiresDedicatedAllocation,
8741  prefersDedicatedAllocation,
8742  buffer, // dedicatedBuffer
8743  VK_NULL_HANDLE, // dedicatedImage
8744  *pCreateInfo,
8745  VMA_SUBALLOCATION_TYPE_BUFFER,
8746  pAllocation);
8747 
8748  if(pAllocationInfo && result == VK_SUCCESS)
8749  {
8750  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8751  }
8752 
8753  return result;
8754 }
8755 
8756 VkResult vmaAllocateMemoryForImage(
8757  VmaAllocator allocator,
8758  VkImage image,
8759  const VmaAllocationCreateInfo* pCreateInfo,
8760  VmaAllocation* pAllocation,
8761  VmaAllocationInfo* pAllocationInfo)
8762 {
8763  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8764 
8765  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
8766 
8767  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8768 
8769  VkResult result = AllocateMemoryForImage(
8770  allocator,
8771  image,
8772  pCreateInfo,
8773  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
8774  pAllocation);
8775 
8776  if(pAllocationInfo && result == VK_SUCCESS)
8777  {
8778  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8779  }
8780 
8781  return result;
8782 }
8783 
8784 void vmaFreeMemory(
8785  VmaAllocator allocator,
8786  VmaAllocation allocation)
8787 {
8788  VMA_ASSERT(allocator && allocation);
8789 
8790  VMA_DEBUG_LOG("vmaFreeMemory");
8791 
8792  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8793 
8794  allocator->FreeMemory(allocation);
8795 }
8796 
8797 void vmaGetAllocationInfo(
8798  VmaAllocator allocator,
8799  VmaAllocation allocation,
8800  VmaAllocationInfo* pAllocationInfo)
8801 {
8802  VMA_ASSERT(allocator && allocation && pAllocationInfo);
8803 
8804  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8805 
8806  allocator->GetAllocationInfo(allocation, pAllocationInfo);
8807 }
8808 
8809 VkBool32 vmaTouchAllocation(
8810  VmaAllocator allocator,
8811  VmaAllocation allocation)
8812 {
8813  VMA_ASSERT(allocator && allocation);
8814 
8815  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8816 
8817  return allocator->TouchAllocation(allocation);
8818 }
8819 
8820 void vmaSetAllocationUserData(
8821  VmaAllocator allocator,
8822  VmaAllocation allocation,
8823  void* pUserData)
8824 {
8825  VMA_ASSERT(allocator && allocation);
8826 
8827  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8828 
8829  allocation->SetUserData(allocator, pUserData);
8830 }
8831 
8832 void vmaCreateLostAllocation(
8833  VmaAllocator allocator,
8834  VmaAllocation* pAllocation)
8835 {
8836  VMA_ASSERT(allocator && pAllocation);
8837 
8838  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
8839 
8840  allocator->CreateLostAllocation(pAllocation);
8841 }
8842 
8843 VkResult vmaMapMemory(
8844  VmaAllocator allocator,
8845  VmaAllocation allocation,
8846  void** ppData)
8847 {
8848  VMA_ASSERT(allocator && allocation && ppData);
8849 
8850  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8851 
8852  return allocator->Map(allocation, ppData);
8853 }
8854 
8855 void vmaUnmapMemory(
8856  VmaAllocator allocator,
8857  VmaAllocation allocation)
8858 {
8859  VMA_ASSERT(allocator && allocation);
8860 
8861  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8862 
8863  allocator->Unmap(allocation);
8864 }
8865 
8866 VkResult vmaDefragment(
8867  VmaAllocator allocator,
8868  VmaAllocation* pAllocations,
8869  size_t allocationCount,
8870  VkBool32* pAllocationsChanged,
8871  const VmaDefragmentationInfo *pDefragmentationInfo,
8872  VmaDefragmentationStats* pDefragmentationStats)
8873 {
8874  VMA_ASSERT(allocator && pAllocations);
8875 
8876  VMA_DEBUG_LOG("vmaDefragment");
8877 
8878  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8879 
8880  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
8881 }
8882 
8883 VkResult vmaCreateBuffer(
8884  VmaAllocator allocator,
8885  const VkBufferCreateInfo* pBufferCreateInfo,
8886  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8887  VkBuffer* pBuffer,
8888  VmaAllocation* pAllocation,
8889  VmaAllocationInfo* pAllocationInfo)
8890 {
8891  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
8892 
8893  VMA_DEBUG_LOG("vmaCreateBuffer");
8894 
8895  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8896 
8897  *pBuffer = VK_NULL_HANDLE;
8898  *pAllocation = VK_NULL_HANDLE;
8899 
8900  // 1. Create VkBuffer.
8901  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
8902  allocator->m_hDevice,
8903  pBufferCreateInfo,
8904  allocator->GetAllocationCallbacks(),
8905  pBuffer);
8906  if(res >= 0)
8907  {
8908  // 2. vkGetBufferMemoryRequirements.
8909  VkMemoryRequirements vkMemReq = {};
8910  bool requiresDedicatedAllocation = false;
8911  bool prefersDedicatedAllocation = false;
8912  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
8913  requiresDedicatedAllocation, prefersDedicatedAllocation);
8914 
8915  // Make sure alignment requirements for specific buffer usages reported
8916  // in Physical Device Properties are included in alignment reported by memory requirements.
8917  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
8918  {
8919  VMA_ASSERT(vkMemReq.alignment %
8920  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
8921  }
8922  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
8923  {
8924  VMA_ASSERT(vkMemReq.alignment %
8925  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
8926  }
8927  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
8928  {
8929  VMA_ASSERT(vkMemReq.alignment %
8930  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
8931  }
8932 
8933  // 3. Allocate memory using allocator.
8934  res = allocator->AllocateMemory(
8935  vkMemReq,
8936  requiresDedicatedAllocation,
8937  prefersDedicatedAllocation,
8938  *pBuffer, // dedicatedBuffer
8939  VK_NULL_HANDLE, // dedicatedImage
8940  *pAllocationCreateInfo,
8941  VMA_SUBALLOCATION_TYPE_BUFFER,
8942  pAllocation);
8943  if(res >= 0)
8944  {
8945  // 3. Bind buffer with memory.
8946  res = (*allocator->GetVulkanFunctions().vkBindBufferMemory)(
8947  allocator->m_hDevice,
8948  *pBuffer,
8949  (*pAllocation)->GetMemory(),
8950  (*pAllocation)->GetOffset());
8951  if(res >= 0)
8952  {
8953  // All steps succeeded.
8954  if(pAllocationInfo != VMA_NULL)
8955  {
8956  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8957  }
8958  return VK_SUCCESS;
8959  }
8960  allocator->FreeMemory(*pAllocation);
8961  *pAllocation = VK_NULL_HANDLE;
8962  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8963  *pBuffer = VK_NULL_HANDLE;
8964  return res;
8965  }
8966  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8967  *pBuffer = VK_NULL_HANDLE;
8968  return res;
8969  }
8970  return res;
8971 }
8972 
8973 void vmaDestroyBuffer(
8974  VmaAllocator allocator,
8975  VkBuffer buffer,
8976  VmaAllocation allocation)
8977 {
8978  if(buffer != VK_NULL_HANDLE)
8979  {
8980  VMA_ASSERT(allocator);
8981 
8982  VMA_DEBUG_LOG("vmaDestroyBuffer");
8983 
8984  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8985 
8986  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
8987 
8988  allocator->FreeMemory(allocation);
8989  }
8990 }
8991 
8992 VkResult vmaCreateImage(
8993  VmaAllocator allocator,
8994  const VkImageCreateInfo* pImageCreateInfo,
8995  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8996  VkImage* pImage,
8997  VmaAllocation* pAllocation,
8998  VmaAllocationInfo* pAllocationInfo)
8999 {
9000  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
9001 
9002  VMA_DEBUG_LOG("vmaCreateImage");
9003 
9004  VMA_DEBUG_GLOBAL_MUTEX_LOCK
9005 
9006  *pImage = VK_NULL_HANDLE;
9007  *pAllocation = VK_NULL_HANDLE;
9008 
9009  // 1. Create VkImage.
9010  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
9011  allocator->m_hDevice,
9012  pImageCreateInfo,
9013  allocator->GetAllocationCallbacks(),
9014  pImage);
9015  if(res >= 0)
9016  {
9017  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
9018  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
9019  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
9020 
9021  // 2. Allocate memory using allocator.
9022  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
9023  if(res >= 0)
9024  {
9025  // 3. Bind image with memory.
9026  res = (*allocator->GetVulkanFunctions().vkBindImageMemory)(
9027  allocator->m_hDevice,
9028  *pImage,
9029  (*pAllocation)->GetMemory(),
9030  (*pAllocation)->GetOffset());
9031  if(res >= 0)
9032  {
9033  // All steps succeeded.
9034  if(pAllocationInfo != VMA_NULL)
9035  {
9036  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
9037  }
9038  return VK_SUCCESS;
9039  }
9040  allocator->FreeMemory(*pAllocation);
9041  *pAllocation = VK_NULL_HANDLE;
9042  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
9043  *pImage = VK_NULL_HANDLE;
9044  return res;
9045  }
9046  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
9047  *pImage = VK_NULL_HANDLE;
9048  return res;
9049  }
9050  return res;
9051 }
9052 
9053 void vmaDestroyImage(
9054  VmaAllocator allocator,
9055  VkImage image,
9056  VmaAllocation allocation)
9057 {
9058  if(image != VK_NULL_HANDLE)
9059  {
9060  VMA_ASSERT(allocator);
9061 
9062  VMA_DEBUG_LOG("vmaDestroyImage");
9063 
9064  VMA_DEBUG_GLOBAL_MUTEX_LOCK
9065 
9066  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
9067 
9068  allocator->FreeMemory(allocation);
9069  }
9070 }
9071 
9072 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:1013
+
VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:1267
vmaUnmapMemory
void vmaUnmapMemory(VmaAllocator allocator, VmaAllocation allocation)
Unmaps memory represented by given allocation, mapped previously using vmaMapMemory().
-
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:1030
+
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:1038
vmaDefragment
VkResult vmaDefragment(VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
Compacts memory by moving allocations.
-
VmaVulkanFunctions::vkCreateBuffer
PFN_vkCreateBuffer vkCreateBuffer
Definition: vk_mem_alloc.h:1015
+
VmaVulkanFunctions::vkCreateBuffer
PFN_vkCreateBuffer vkCreateBuffer
Definition: vk_mem_alloc.h:1023
vmaFreeStatsString
void vmaFreeStatsString(VmaAllocator allocator, char *pStatsString)
VmaStats
struct VmaStats VmaStats
General statistics from current state of Allocator.
-
VMA_MEMORY_USAGE_GPU_ONLY
Definition: vk_mem_alloc.h:1216
-
VmaVulkanFunctions::vkMapMemory
PFN_vkMapMemory vkMapMemory
Definition: vk_mem_alloc.h:1009
-
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:1565
-
VmaAllocatorCreateInfo::flags
VmaAllocatorCreateFlags flags
Flags for created allocator. Use VmaAllocatorCreateFlagBits enum.
Definition: vk_mem_alloc.h:1027
-
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:1764
-
VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
Definition: vk_mem_alloc.h:1435
+
VMA_MEMORY_USAGE_GPU_ONLY
Definition: vk_mem_alloc.h:1224
+
VmaVulkanFunctions::vkMapMemory
PFN_vkMapMemory vkMapMemory
Definition: vk_mem_alloc.h:1017
+
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:1573
+
VmaAllocatorCreateInfo::flags
VmaAllocatorCreateFlags flags
Flags for created allocator. Use VmaAllocatorCreateFlagBits enum.
Definition: vk_mem_alloc.h:1035
+
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:1772
+
VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
Definition: vk_mem_alloc.h:1443
vmaMakePoolAllocationsLost
void vmaMakePoolAllocationsLost(VmaAllocator allocator, VmaPool pool, size_t *pLostAllocationCount)
Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInf...
-
VmaPoolStats::size
VkDeviceSize size
Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
Definition: vk_mem_alloc.h:1489
-
VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
Definition: vk_mem_alloc.h:1296
-
VmaAllocatorCreateFlags
VkFlags VmaAllocatorCreateFlags
Definition: vk_mem_alloc.h:998
-
VmaAllocationCreateInfo::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a memory type chosen for an allocation.
Definition: vk_mem_alloc.h:1334
-
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:1243
-
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom CPU memory allocation callbacks. Optional.
Definition: vk_mem_alloc.h:1039
+
VmaPoolStats::size
VkDeviceSize size
Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
Definition: vk_mem_alloc.h:1497
+
VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
Definition: vk_mem_alloc.h:1304
+
VmaAllocatorCreateFlags
VkFlags VmaAllocatorCreateFlags
Definition: vk_mem_alloc.h:1006
+
VmaAllocationCreateInfo::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a memory type chosen for an allocation.
Definition: vk_mem_alloc.h:1342
+
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:1251
+
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom CPU memory allocation callbacks. Optional.
Definition: vk_mem_alloc.h:1047
vmaCalculateStats
void vmaCalculateStats(VmaAllocator allocator, VmaStats *pStats)
Retrieves statistics from current state of the Allocator.
-
VmaAllocatorCreateInfo::pVulkanFunctions
const VmaVulkanFunctions * pVulkanFunctions
Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
Definition: vk_mem_alloc.h:1092
-
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:1024
+
VmaAllocatorCreateInfo::pVulkanFunctions
const VmaVulkanFunctions * pVulkanFunctions
Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
Definition: vk_mem_alloc.h:1100
+
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:1032
vmaDestroyAllocator
void vmaDestroyAllocator(VmaAllocator allocator)
Destroys allocator object.
-
VmaAllocationCreateFlagBits
VmaAllocationCreateFlagBits
Flags to be passed as VmaAllocationCreateInfo::flags.
Definition: vk_mem_alloc.h:1247
+
VmaAllocationCreateFlagBits
VmaAllocationCreateFlagBits
Flags to be passed as VmaAllocationCreateInfo::flags.
Definition: vk_mem_alloc.h:1255
vmaGetAllocationInfo
void vmaGetAllocationInfo(VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
Returns current information about specified allocation and atomically marks it as used in current fra...
-
VmaStatInfo::allocationSizeMax
VkDeviceSize allocationSizeMax
Definition: vk_mem_alloc.h:1157
-
VmaVulkanFunctions::vkBindImageMemory
PFN_vkBindImageMemory vkBindImageMemory
Definition: vk_mem_alloc.h:1012
-
VmaStatInfo::unusedBytes
VkDeviceSize unusedBytes
Total number of bytes occupied by unused ranges.
Definition: vk_mem_alloc.h:1156
-
VmaVulkanFunctions::vkGetImageMemoryRequirements2KHR
PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR
Definition: vk_mem_alloc.h:1020
-
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:1768
+
VmaStatInfo::allocationSizeMax
VkDeviceSize allocationSizeMax
Definition: vk_mem_alloc.h:1165
+
VmaVulkanFunctions::vkBindImageMemory
PFN_vkBindImageMemory vkBindImageMemory
Definition: vk_mem_alloc.h:1020
+
VmaStatInfo::unusedBytes
VkDeviceSize unusedBytes
Total number of bytes occupied by unused ranges.
Definition: vk_mem_alloc.h:1164
+
VmaVulkanFunctions::vkGetImageMemoryRequirements2KHR
PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR
Definition: vk_mem_alloc.h:1028
+
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:1776
vmaFreeMemory
void vmaFreeMemory(VmaAllocator allocator, VmaAllocation allocation)
Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
-
VmaAllocatorCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:1056
-
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:1166
-
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:1776
-
VmaAllocationCreateInfo::flags
VmaAllocationCreateFlags flags
Use VmaAllocationCreateFlagBits enum.
Definition: vk_mem_alloc.h:1318
-
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1759
-
VmaVulkanFunctions::vkGetBufferMemoryRequirements
PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
Definition: vk_mem_alloc.h:1013
-
PFN_vmaAllocateDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called after successful vkAllocateMemory.
Definition: vk_mem_alloc.h:940
-
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:1033
-
VmaPoolCreateInfo
Describes parameter of created VmaPool.
Definition: vk_mem_alloc.h:1443
-
VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1437
-
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:1575
+
VmaAllocatorCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:1064
+
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:1174
+
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:1784
+
VmaAllocationCreateInfo::flags
VmaAllocationCreateFlags flags
Use VmaAllocationCreateFlagBits enum.
Definition: vk_mem_alloc.h:1326
+
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1767
+
VmaVulkanFunctions::vkGetBufferMemoryRequirements
PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
Definition: vk_mem_alloc.h:1021
+
PFN_vmaAllocateDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called after successful vkAllocateMemory.
Definition: vk_mem_alloc.h:948
+
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:1041
+
VmaPoolCreateInfo
Describes parameter of created VmaPool.
Definition: vk_mem_alloc.h:1451
+
VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1445
+
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:1583
vmaGetMemoryTypeProperties
void vmaGetMemoryTypeProperties(VmaAllocator allocator, uint32_t memoryTypeIndex, VkMemoryPropertyFlags *pFlags)
Given Memory Type Index, returns Property Flags of this memory type.
-
VmaVulkanFunctions::vkUnmapMemory
PFN_vkUnmapMemory vkUnmapMemory
Definition: vk_mem_alloc.h:1010
-
VmaAllocationCreateInfo::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:1355
-
VmaPoolCreateInfo::minBlockCount
size_t minBlockCount
Minimum number of blocks to be always allocated in this pool, even if they stay empty.
Definition: vk_mem_alloc.h:1459
-
VmaPoolStats::allocationCount
size_t allocationCount
Number of VmaAllocation objects created from this pool that were not destroyed or lost...
Definition: vk_mem_alloc.h:1495
+
VmaVulkanFunctions::vkUnmapMemory
PFN_vkUnmapMemory vkUnmapMemory
Definition: vk_mem_alloc.h:1018
+
VmaAllocationCreateInfo::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:1363
+
VmaPoolCreateInfo::minBlockCount
size_t minBlockCount
Minimum number of blocks to be always allocated in this pool, even if they stay empty.
Definition: vk_mem_alloc.h:1467
+
VmaPoolStats::allocationCount
size_t allocationCount
Number of VmaAllocation objects created from this pool that were not destroyed or lost...
Definition: vk_mem_alloc.h:1503
VmaVulkanFunctions
struct VmaVulkanFunctions VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
-
VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:996
-
VmaPoolCreateInfo::memoryTypeIndex
uint32_t memoryTypeIndex
Vulkan memory type index to allocate this pool from.
Definition: vk_mem_alloc.h:1446
+
VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1004
+
VmaPoolCreateInfo::memoryTypeIndex
uint32_t memoryTypeIndex
Vulkan memory type index to allocate this pool from.
Definition: vk_mem_alloc.h:1454
vmaFindMemoryTypeIndex
VkResult vmaFindMemoryTypeIndex(VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
Helps to find memoryTypeIndex, given memoryTypeBits and VmaAllocationCreateInfo.
-
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:1194
+
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:1202
VmaAllocationInfo
struct VmaAllocationInfo VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
-
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:1754
+
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:1762
VmaPoolCreateInfo
struct VmaPoolCreateInfo VmaPoolCreateInfo
Describes parameter of created VmaPool.
vmaDestroyPool
void vmaDestroyPool(VmaAllocator allocator, VmaPool pool)
Destroys VmaPool object and frees Vulkan device memory.
-
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:1772
-
VMA_MEMORY_USAGE_CPU_TO_GPU
Definition: vk_mem_alloc.h:1233
-
VmaAllocationCreateInfo::memoryTypeBits
uint32_t memoryTypeBits
Bitmask containing one bit set for every memory type acceptable for this allocation.
Definition: vk_mem_alloc.h:1342
-
VmaVulkanFunctions::vkBindBufferMemory
PFN_vkBindBufferMemory vkBindBufferMemory
Definition: vk_mem_alloc.h:1011
+
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:1780
+
VMA_MEMORY_USAGE_CPU_TO_GPU
Definition: vk_mem_alloc.h:1241
+
VmaAllocationCreateInfo::memoryTypeBits
uint32_t memoryTypeBits
Bitmask containing one bit set for every memory type acceptable for this allocation.
Definition: vk_mem_alloc.h:1350
+
VmaVulkanFunctions::vkBindBufferMemory
PFN_vkBindBufferMemory vkBindBufferMemory
Definition: vk_mem_alloc.h:1019
vmaGetPoolStats
void vmaGetPoolStats(VmaAllocator allocator, VmaPool pool, VmaPoolStats *pPoolStats)
Retrieves statistics of existing VmaPool object.
VmaDefragmentationInfo
struct VmaDefragmentationInfo VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
-
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:1162
-
PFN_vmaFreeDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called before vkFreeMemory.
Definition: vk_mem_alloc.h:946
+
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:1170
+
PFN_vmaFreeDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called before vkFreeMemory.
Definition: vk_mem_alloc.h:954
vmaSetAllocationUserData
void vmaSetAllocationUserData(VmaAllocator allocator, VmaAllocation allocation, void *pUserData)
Sets pUserData in given allocation to new value.
vmaCreatePool
VkResult vmaCreatePool(VmaAllocator allocator, const VmaPoolCreateInfo *pCreateInfo, VmaPool *pPool)
Allocates Vulkan device memory and creates VmaPool object.
-
VmaAllocatorCreateFlagBits
VmaAllocatorCreateFlagBits
Flags for created VmaAllocator.
Definition: vk_mem_alloc.h:967
+
VmaAllocatorCreateFlagBits
VmaAllocatorCreateFlagBits
Flags for created VmaAllocator.
Definition: vk_mem_alloc.h:975
VmaStatInfo
struct VmaStatInfo VmaStatInfo
Calculated statistics of memory usage in entire allocator.
-
VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
Definition: vk_mem_alloc.h:972
-
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:1774
+
VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
Definition: vk_mem_alloc.h:980
+
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:1782
vmaCreateLostAllocation
void vmaCreateLostAllocation(VmaAllocator allocator, VmaAllocation *pAllocation)
Creates new allocation that is in lost state from the beginning.
-
VmaAllocationCreateInfo::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:1329
-
VmaPoolStats::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Size of the largest continuous free memory region.
Definition: vk_mem_alloc.h:1505
+
VmaAllocationCreateInfo::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:1337
+
VmaPoolStats::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Size of the largest continuous free memory region.
Definition: vk_mem_alloc.h:1513
vmaBuildStatsString
void vmaBuildStatsString(VmaAllocator allocator, char **ppStatsString, VkBool32 detailedMap)
Builds and returns statistics as string in JSON format.
-
VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
Definition: vk_mem_alloc.h:1006
-
VmaStatInfo
Calculated statistics of memory usage in entire allocator.
Definition: vk_mem_alloc.h:1145
-
VmaPoolCreateInfo::blockSize
VkDeviceSize blockSize
Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
Definition: vk_mem_alloc.h:1454
-
VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
Definition: vk_mem_alloc.h:959
+
VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
Definition: vk_mem_alloc.h:1014
+
VmaStatInfo
Calculated statistics of memory usage in entire allocator.
Definition: vk_mem_alloc.h:1153
+
VmaPoolCreateInfo::blockSize
VkDeviceSize blockSize
Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
Definition: vk_mem_alloc.h:1462
+
VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
Definition: vk_mem_alloc.h:967
vmaCreateBuffer
VkResult vmaCreateBuffer(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
-
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
Definition: vk_mem_alloc.h:1303
-
VmaStatInfo::unusedRangeSizeMin
VkDeviceSize unusedRangeSizeMin
Definition: vk_mem_alloc.h:1158
-
VmaDeviceMemoryCallbacks::pfnFree
PFN_vmaFreeDeviceMemoryFunction pfnFree
Optional, can be null.
Definition: vk_mem_alloc.h:963
-
VmaPoolCreateInfo::flags
VmaPoolCreateFlags flags
Use combination of VmaPoolCreateFlagBits.
Definition: vk_mem_alloc.h:1449
-
VMA_MEMORY_USAGE_GPU_TO_CPU
Definition: vk_mem_alloc.h:1242
+
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
Definition: vk_mem_alloc.h:1311
+
VmaStatInfo::unusedRangeSizeMin
VkDeviceSize unusedRangeSizeMin
Definition: vk_mem_alloc.h:1166
+
VmaDeviceMemoryCallbacks::pfnFree
PFN_vmaFreeDeviceMemoryFunction pfnFree
Optional, can be null.
Definition: vk_mem_alloc.h:971
+
VmaPoolCreateInfo::flags
VmaPoolCreateFlags flags
Use combination of VmaPoolCreateFlagBits.
Definition: vk_mem_alloc.h:1457
+
VMA_MEMORY_USAGE_GPU_TO_CPU
Definition: vk_mem_alloc.h:1250
VmaPoolStats
struct VmaPoolStats VmaPoolStats
Describes parameter of existing VmaPool.
vmaCreateImage
VkResult vmaCreateImage(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
Function similar to vmaCreateBuffer().
-
VmaAllocationCreateInfo::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:1324
-
VmaAllocationCreateInfo
Definition: vk_mem_alloc.h:1315
+
VmaAllocationCreateInfo::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:1332
+
VmaAllocationCreateInfo
Definition: vk_mem_alloc.h:1323
vmaFindMemoryTypeIndexForImageInfo
VkResult vmaFindMemoryTypeIndexForImageInfo(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
Helps to find memoryTypeIndex, given VkImageCreateInfo and VmaAllocationCreateInfo.
-
VmaStatInfo::blockCount
uint32_t blockCount
Number of VkDeviceMemory Vulkan memory blocks allocated.
Definition: vk_mem_alloc.h:1148
-
VmaVulkanFunctions::vkFreeMemory
PFN_vkFreeMemory vkFreeMemory
Definition: vk_mem_alloc.h:1008
-
VmaPoolCreateInfo::maxBlockCount
size_t maxBlockCount
Maximum number of blocks that can be allocated in this pool. Optional.
Definition: vk_mem_alloc.h:1467
-
VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
Informative callbacks for vkAllocateMemory, vkFreeMemory. Optional.
Definition: vk_mem_alloc.h:1042
-
VmaPoolStats::unusedRangeCount
size_t unusedRangeCount
Number of continuous memory ranges in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1498
-
VmaAllocationCreateFlags
VkFlags VmaAllocationCreateFlags
Definition: vk_mem_alloc.h:1313
-
VmaAllocationCreateInfo::pool
VmaPool pool
Pool that this allocation should be created in.
Definition: vk_mem_alloc.h:1348
+
VmaStatInfo::blockCount
uint32_t blockCount
Number of VkDeviceMemory Vulkan memory blocks allocated.
Definition: vk_mem_alloc.h:1156
+
VmaVulkanFunctions::vkFreeMemory
PFN_vkFreeMemory vkFreeMemory
Definition: vk_mem_alloc.h:1016
+
VmaPoolCreateInfo::maxBlockCount
size_t maxBlockCount
Maximum number of blocks that can be allocated in this pool. Optional.
Definition: vk_mem_alloc.h:1475
+
VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
Informative callbacks for vkAllocateMemory, vkFreeMemory. Optional.
Definition: vk_mem_alloc.h:1050
+
VmaPoolStats::unusedRangeCount
size_t unusedRangeCount
Number of continuous memory ranges in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1506
+
VmaAllocationCreateFlags
VkFlags VmaAllocationCreateFlags
Definition: vk_mem_alloc.h:1321
+
VmaAllocationCreateInfo::pool
VmaPool pool
Pool that this allocation should be created in.
Definition: vk_mem_alloc.h:1356
vmaGetMemoryProperties
void vmaGetMemoryProperties(VmaAllocator allocator, const VkPhysicalDeviceMemoryProperties **ppPhysicalDeviceMemoryProperties)
-
VmaAllocatorCreateInfo::pHeapSizeLimit
const VkDeviceSize * pHeapSizeLimit
Either null or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
Definition: vk_mem_alloc.h:1080
-
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:1164
-
VMA_ALLOCATION_CREATE_MAPPED_BIT
Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
Definition: vk_mem_alloc.h:1283
-
VmaStatInfo::allocationSizeMin
VkDeviceSize allocationSizeMin
Definition: vk_mem_alloc.h:1157
+
VmaAllocatorCreateInfo::pHeapSizeLimit
const VkDeviceSize * pHeapSizeLimit
Either null or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
Definition: vk_mem_alloc.h:1088
+
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:1172
+
VMA_ALLOCATION_CREATE_MAPPED_BIT
Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
Definition: vk_mem_alloc.h:1291
+
VmaStatInfo::allocationSizeMin
VkDeviceSize allocationSizeMin
Definition: vk_mem_alloc.h:1165
vmaFindMemoryTypeIndexForBufferInfo
VkResult vmaFindMemoryTypeIndexForBufferInfo(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
Helps to find memoryTypeIndex, given VkBufferCreateInfo and VmaAllocationCreateInfo.
-
VmaVulkanFunctions::vkCreateImage
PFN_vkCreateImage vkCreateImage
Definition: vk_mem_alloc.h:1017
-
VmaDeviceMemoryCallbacks::pfnAllocate
PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
Optional, can be null.
Definition: vk_mem_alloc.h:961
-
VmaVulkanFunctions::vkDestroyBuffer
PFN_vkDestroyBuffer vkDestroyBuffer
Definition: vk_mem_alloc.h:1016
+
VmaVulkanFunctions::vkCreateImage
PFN_vkCreateImage vkCreateImage
Definition: vk_mem_alloc.h:1025
+
VmaDeviceMemoryCallbacks::pfnAllocate
PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
Optional, can be null.
Definition: vk_mem_alloc.h:969
+
VmaVulkanFunctions::vkDestroyBuffer
PFN_vkDestroyBuffer vkDestroyBuffer
Definition: vk_mem_alloc.h:1024
vmaMapMemory
VkResult vmaMapMemory(VmaAllocator allocator, VmaAllocation allocation, void **ppData)
Maps memory represented by given allocation and returns pointer to it.
-
VmaPoolCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:1481
+
VmaPoolCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:1489
vmaAllocateMemoryForImage
VkResult vmaAllocateMemoryForImage(VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
Function similar to vmaAllocateMemoryForBuffer().
VmaAllocatorCreateInfo
struct VmaAllocatorCreateInfo VmaAllocatorCreateInfo
Description of a Allocator to be created.
-
VmaAllocationInfo::pUserData
void * pUserData
Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
Definition: vk_mem_alloc.h:1589
-
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from large heaps > 1 GiB...
Definition: vk_mem_alloc.h:1036
-
VmaStatInfo::allocationSizeAvg
VkDeviceSize allocationSizeAvg
Definition: vk_mem_alloc.h:1157
-
VmaStatInfo::usedBytes
VkDeviceSize usedBytes
Total number of bytes occupied by all allocations.
Definition: vk_mem_alloc.h:1154
+
VmaAllocationInfo::pUserData
void * pUserData
Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
Definition: vk_mem_alloc.h:1597
+
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from large heaps > 1 GiB...
Definition: vk_mem_alloc.h:1044
+
VmaStatInfo::allocationSizeAvg
VkDeviceSize allocationSizeAvg
Definition: vk_mem_alloc.h:1165
+
VmaStatInfo::usedBytes
VkDeviceSize usedBytes
Total number of bytes occupied by all allocations.
Definition: vk_mem_alloc.h:1162
VmaDeviceMemoryCallbacks
struct VmaDeviceMemoryCallbacks VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
-
VmaPoolStats
Describes parameter of existing VmaPool.
Definition: vk_mem_alloc.h:1486
-
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:1570
-
VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1311
-
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1770
-
VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
Definition: vk_mem_alloc.h:1004
+
VmaPoolStats
Describes parameter of existing VmaPool.
Definition: vk_mem_alloc.h:1494
+
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:1578
+
VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1319
+
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1778
+
VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
Definition: vk_mem_alloc.h:1012
vmaCreateAllocator
VkResult vmaCreateAllocator(const VmaAllocatorCreateInfo *pCreateInfo, VmaAllocator *pAllocator)
Creates Allocator object.
-
VmaVulkanFunctions::vkGetBufferMemoryRequirements2KHR
PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR
Definition: vk_mem_alloc.h:1019
-
VmaStatInfo::unusedRangeCount
uint32_t unusedRangeCount
Number of free ranges of memory between allocations.
Definition: vk_mem_alloc.h:1152
-
VMA_MEMORY_USAGE_UNKNOWN
Definition: vk_mem_alloc.h:1199
-
VmaPoolCreateFlags
VkFlags VmaPoolCreateFlags
Definition: vk_mem_alloc.h:1439
+
VmaVulkanFunctions::vkGetBufferMemoryRequirements2KHR
PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR
Definition: vk_mem_alloc.h:1027
+
VmaStatInfo::unusedRangeCount
uint32_t unusedRangeCount
Number of free ranges of memory between allocations.
Definition: vk_mem_alloc.h:1160
+
VMA_MEMORY_USAGE_UNKNOWN
Definition: vk_mem_alloc.h:1207
+
VmaPoolCreateFlags
VkFlags VmaPoolCreateFlags
Definition: vk_mem_alloc.h:1447
vmaGetPhysicalDeviceProperties
void vmaGetPhysicalDeviceProperties(VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
-
VmaStatInfo::allocationCount
uint32_t allocationCount
Number of VmaAllocation allocation objects allocated.
Definition: vk_mem_alloc.h:1150
-
VmaVulkanFunctions::vkGetImageMemoryRequirements
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
Definition: vk_mem_alloc.h:1014
-
VmaVulkanFunctions::vkDestroyImage
PFN_vkDestroyImage vkDestroyImage
Definition: vk_mem_alloc.h:1018
-
VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:1270
-
VMA_MEMORY_USAGE_CPU_ONLY
Definition: vk_mem_alloc.h:1226
-
VmaAllocationInfo::pMappedData
void * pMappedData
Pointer to the beginning of this allocation as mapped data.
Definition: vk_mem_alloc.h:1584
+
VmaStatInfo::allocationCount
uint32_t allocationCount
Number of VmaAllocation allocation objects allocated.
Definition: vk_mem_alloc.h:1158
+
VmaVulkanFunctions::vkGetImageMemoryRequirements
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
Definition: vk_mem_alloc.h:1022
+
VmaVulkanFunctions::vkDestroyImage
PFN_vkDestroyImage vkDestroyImage
Definition: vk_mem_alloc.h:1026
+
VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:1278
+
VMA_MEMORY_USAGE_CPU_ONLY
Definition: vk_mem_alloc.h:1234
+
VmaAllocationInfo::pMappedData
void * pMappedData
Pointer to the beginning of this allocation as mapped data.
Definition: vk_mem_alloc.h:1592
vmaDestroyImage
void vmaDestroyImage(VmaAllocator allocator, VkImage image, VmaAllocation allocation)
Destroys Vulkan image and frees allocated memory.
-
VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT
Enables usage of VK_KHR_dedicated_allocation extension.
Definition: vk_mem_alloc.h:994
+
VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT
Enables usage of VK_KHR_dedicated_allocation extension.
Definition: vk_mem_alloc.h:1002
VmaDefragmentationStats
struct VmaDefragmentationStats VmaDefragmentationStats
Statistics returned by function vmaDefragment().
-
VmaVulkanFunctions::vkAllocateMemory
PFN_vkAllocateMemory vkAllocateMemory
Definition: vk_mem_alloc.h:1007
-
VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
Definition: vk_mem_alloc.h:1551
+
VmaVulkanFunctions::vkAllocateMemory
PFN_vkAllocateMemory vkAllocateMemory
Definition: vk_mem_alloc.h:1015
+
VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
Definition: vk_mem_alloc.h:1559
vmaAllocateMemory
VkResult vmaAllocateMemory(VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
General purpose memory allocation.
vmaSetCurrentFrameIndex
void vmaSetCurrentFrameIndex(VmaAllocator allocator, uint32_t frameIndex)
Sets index of the current frame.
VmaAllocationCreateInfo
struct VmaAllocationCreateInfo VmaAllocationCreateInfo
vmaAllocateMemoryForBuffer
VkResult vmaAllocateMemoryForBuffer(VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
-
VmaPoolCreateFlagBits
VmaPoolCreateFlagBits
Flags to be passed as VmaPoolCreateInfo::flags.
Definition: vk_mem_alloc.h:1417
-
VmaStatInfo::unusedRangeSizeAvg
VkDeviceSize unusedRangeSizeAvg
Definition: vk_mem_alloc.h:1158
+
VmaPoolCreateFlagBits
VmaPoolCreateFlagBits
Flags to be passed as VmaPoolCreateInfo::flags.
Definition: vk_mem_alloc.h:1425
+
VmaStatInfo::unusedRangeSizeAvg
VkDeviceSize unusedRangeSizeAvg
Definition: vk_mem_alloc.h:1166
vmaTouchAllocation
VkBool32 vmaTouchAllocation(VmaAllocator allocator, VmaAllocation allocation)
Returns VK_TRUE if allocation is not lost and atomically marks it as used in current frame...
-
VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT
Definition: vk_mem_alloc.h:1309
-
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:1165
+
VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT
Definition: vk_mem_alloc.h:1317
+
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:1173
vmaDestroyBuffer
void vmaDestroyBuffer(VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation)
Destroys Vulkan buffer and frees allocated memory.
-
VmaPoolStats::unusedSize
VkDeviceSize unusedSize
Total number of bytes in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1492
-
VmaStatInfo::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Definition: vk_mem_alloc.h:1158
-
VmaAllocationInfo::memoryType
uint32_t memoryType
Memory type index that this allocation was allocated from.
Definition: vk_mem_alloc.h:1556
+
VmaPoolStats::unusedSize
VkDeviceSize unusedSize
Total number of bytes in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1500
+
VmaStatInfo::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Definition: vk_mem_alloc.h:1166
+
VmaAllocationInfo::memoryType
uint32_t memoryType
Memory type index that this allocation was allocated from.
Definition: vk_mem_alloc.h:1564
diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 210f450..d77b201 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -673,6 +673,14 @@ about overlapping regions of memory bound to different kinds of buffers and images. This is still valid as long as you implement proper handling of lost allocations (like in the example above) and don't use them. +You can create an allocation that is already in lost state from the beginning using function +vmaCreateLostAllocation(). It may be useful if you need a "dummy" allocation that is not null. + +You can call function vmaMakePoolAllocationsLost() to set all eligible allocations +in a specified custom pool to lost state. +Allocations that have been "touched" in current frame or VmaPoolCreateInfo::frameInUseCount frames back +cannot become lost. + \page statistics Statistics